in xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiCatalogPartitionSyncTool.java [144:186]
public boolean syncPartitions(InternalTable table, CatalogTableIdentifier tableIdentifier) {
Map<String, String> lastCommitTimeSyncedProperties =
catalogClient.getTableProperties(
tableIdentifier,
Arrays.asList(LAST_COMMIT_TIME_SYNC, LAST_COMMIT_COMPLETION_TIME_SYNC));
Option<String> lastCommitTimeSynced =
Option.ofNullable(lastCommitTimeSyncedProperties.get(LAST_COMMIT_TIME_SYNC));
Option<String> lastCommitCompletionTimeSynced =
Option.ofNullable(lastCommitTimeSyncedProperties.get(LAST_COMMIT_COMPLETION_TIME_SYNC));
HoodieTableMetaClient metaClient = getMetaClient(table.getBasePath());
if (!lastCommitTimeSynced.isPresent()
|| metaClient.getActiveTimeline().isBeforeTimelineStarts(lastCommitTimeSynced.get())) {
// If the last commit time synced is before the start of the active timeline,
// the Hive sync falls back to list all partitions on storage, instead of
// reading active and archived timelines for written partitions.
log.info(
"Sync all partitions given the last commit time synced is empty or "
+ "before the start of the active timeline. Listing all partitions in "
+ table.getBasePath());
return syncAllPartitions(metaClient, table, tableIdentifier);
} else {
List<String> writtenPartitionsSince =
getWrittenPartitionsSince(
metaClient,
Option.ofNullable(lastCommitTimeSynced.get()),
Option.ofNullable(lastCommitCompletionTimeSynced.get()));
log.info("Storage partitions scan complete. Found " + writtenPartitionsSince.size());
// Sync the partitions if needed
// find dropped partitions, if any, in the latest commit
Set<String> droppedPartitions =
getDroppedPartitionsSince(
metaClient,
Option.ofNullable(lastCommitTimeSynced.get()),
Option.of(lastCommitCompletionTimeSynced.get()));
boolean partitionsChanged =
syncPartitions(metaClient, tableIdentifier, writtenPartitionsSince, droppedPartitions);
if (partitionsChanged) {
updateLastCommitTimeSynced(metaClient, tableIdentifier);
}
return partitionsChanged;
}
}