in xtable-core/src/main/java/org/apache/xtable/hudi/catalog/HudiCatalogPartitionSyncTool.java [366:413]
private List<CatalogPartitionEvent> getPartitionEvents(
HoodieTableMetaClient metaClient,
List<CatalogPartition> allPartitionsInCatalog,
List<String> allPartitionsOnStorage) {
Map<String, String> paths = getPartitionValuesToPathMapping(allPartitionsInCatalog);
Set<String> partitionsToDrop = new HashSet<>(paths.keySet());
List<CatalogPartitionEvent> events = new ArrayList<>();
for (String storagePartition : allPartitionsOnStorage) {
Path storagePartitionPath =
FSUtils.getPartitionPath(metaClient.getBasePathV2(), storagePartition);
String fullStoragePartitionPath =
Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
List<String> storagePartitionValues =
partitionValuesExtractor.extractPartitionValuesInPath(storagePartition);
if (!storagePartitionValues.isEmpty()) {
String storageValue = String.join(", ", storagePartitionValues);
// Remove partitions that exist on storage from the `partitionsToDrop` set,
// so the remaining partitions that exist in the catalog should be dropped
partitionsToDrop.remove(storageValue);
if (!paths.containsKey(storageValue)) {
events.add(CatalogPartitionEvent.newPartitionAddEvent(storagePartition));
} else if (!paths.get(storageValue).equals(fullStoragePartitionPath)) {
events.add(CatalogPartitionEvent.newPartitionUpdateEvent(storagePartition));
}
}
}
partitionsToDrop.forEach(
storageValue -> {
String storagePath = paths.get(storageValue);
try {
String relativePath =
FSUtils.getRelativePartitionPath(
metaClient.getBasePathV2(), new CachingPath(storagePath));
events.add(CatalogPartitionEvent.newPartitionDropEvent(relativePath));
} catch (IllegalArgumentException e) {
log.error(
"Cannot parse the path stored in the catalog, ignoring it for "
+ "generating DROP partition event: \""
+ storagePath
+ "\".",
e);
}
});
return events;
}