public boolean syncPartitions()

in xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiCatalogPartitionSyncTool.java [144:186]


  public boolean syncPartitions(InternalTable table, CatalogTableIdentifier tableIdentifier) {
    Map<String, String> lastCommitTimeSyncedProperties =
        catalogClient.getTableProperties(
            tableIdentifier,
            Arrays.asList(LAST_COMMIT_TIME_SYNC, LAST_COMMIT_COMPLETION_TIME_SYNC));
    Option<String> lastCommitTimeSynced =
        Option.ofNullable(lastCommitTimeSyncedProperties.get(LAST_COMMIT_TIME_SYNC));
    Option<String> lastCommitCompletionTimeSynced =
        Option.ofNullable(lastCommitTimeSyncedProperties.get(LAST_COMMIT_COMPLETION_TIME_SYNC));
    HoodieTableMetaClient metaClient = getMetaClient(table.getBasePath());
    if (!lastCommitTimeSynced.isPresent()
        || metaClient.getActiveTimeline().isBeforeTimelineStarts(lastCommitTimeSynced.get())) {
      // If the last commit time synced is before the start of the active timeline,
      // the Hive sync falls back to list all partitions on storage, instead of
      // reading active and archived timelines for written partitions.
      log.info(
          "Sync all partitions given the last commit time synced is empty or "
              + "before the start of the active timeline. Listing all partitions in "
              + table.getBasePath());
      return syncAllPartitions(metaClient, table, tableIdentifier);
    } else {
      List<String> writtenPartitionsSince =
          getWrittenPartitionsSince(
              metaClient,
              Option.ofNullable(lastCommitTimeSynced.get()),
              Option.ofNullable(lastCommitCompletionTimeSynced.get()));
      log.info("Storage partitions scan complete. Found " + writtenPartitionsSince.size());

      // Sync the partitions if needed
      // find dropped partitions, if any, in the latest commit
      Set<String> droppedPartitions =
          getDroppedPartitionsSince(
              metaClient,
              Option.ofNullable(lastCommitTimeSynced.get()),
              Option.of(lastCommitCompletionTimeSynced.get()));
      boolean partitionsChanged =
          syncPartitions(metaClient, tableIdentifier, writtenPartitionsSince, droppedPartitions);
      if (partitionsChanged) {
        updateLastCommitTimeSynced(metaClient, tableIdentifier);
      }
      return partitionsChanged;
    }
  }