in hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java [1262:1452]
long generateDiffReport(
final String jobId,
final Table<String, OmKeyInfo> fsTable,
final Table<String, OmKeyInfo> tsTable,
final Table<String, OmDirectoryInfo> fsDirTable,
final Table<String, OmDirectoryInfo> tsDirTable,
final PersistentMap<byte[], Boolean> objectIdToIsDirMap,
final PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
final PersistentMap<byte[], byte[]> newObjIdToKeyMap,
final String volumeName,
final String bucketName,
final String fromSnapshotName,
final String toSnapshotName,
final boolean isFSOBucket,
final Optional<Map<Long, Path>> oldParentIdPathMap,
final Optional<Map<Long, Path>> newParentIdPathMap,
final Map<String, String> tablePrefix) {
LOG.info("Starting diff report generation for jobId: {}.", jobId);
ColumnFamilyHandle deleteDiffColumnFamily = null;
ColumnFamilyHandle renameDiffColumnFamily = null;
ColumnFamilyHandle createDiffColumnFamily = null;
ColumnFamilyHandle modifyDiffColumnFamily = null;
// JobId is prepended to column family name to make it unique for request.
try {
deleteDiffColumnFamily =
createColumnFamily(jobId + DELETE_DIFF_TABLE_SUFFIX);
renameDiffColumnFamily =
createColumnFamily(jobId + RENAME_DIFF_TABLE_SUFFIX);
createDiffColumnFamily =
createColumnFamily(jobId + CREATE_DIFF_TABLE_SUFFIX);
modifyDiffColumnFamily =
createColumnFamily(jobId + MODIFY_DIFF_TABLE_SUFFIX);
// Keep byte array instead of storing as DiffReportEntry to avoid
// unnecessary serialization and deserialization.
final PersistentList<byte[]> deleteDiffs =
createDiffReportPersistentList(deleteDiffColumnFamily);
final PersistentList<byte[]> renameDiffs =
createDiffReportPersistentList(renameDiffColumnFamily);
final PersistentList<byte[]> createDiffs =
createDiffReportPersistentList(createDiffColumnFamily);
final PersistentList<byte[]> modifyDiffs =
createDiffReportPersistentList(modifyDiffColumnFamily);
try (ClosableIterator<Map.Entry<byte[], Boolean>>
iterator = objectIdToIsDirMap.iterator()) {
// This counter is used, so that we can check every 100 elements
// if the job is cancelled and snapshots are still active.
int counter = 0;
while (iterator.hasNext()) {
if (counter % 100 == 0 &&
!areDiffJobAndSnapshotsActive(volumeName, bucketName,
fromSnapshotName, toSnapshotName)) {
return -1L;
}
Map.Entry<byte[], Boolean> nextEntry = iterator.next();
byte[] id = nextEntry.getKey();
boolean isDirectoryObject = nextEntry.getValue();
/*
* This key can be
* -> Created after the old snapshot was taken, which means it will be
* missing in oldKeyTable and present in newKeyTable.
* -> Deleted after the old snapshot was taken, which means it will be
* present in oldKeyTable and missing in newKeyTable.
* -> Modified after the old snapshot was taken, which means it will
* be present in oldKeyTable and present in newKeyTable with same
* Object ID but with different metadata.
* -> Renamed after the old snapshot was taken, which means it will be
* present in oldKeyTable and present in newKeyTable but with
* different name and same Object ID.
*/
byte[] oldKeyName = oldObjIdToKeyMap.get(id);
byte[] newKeyName = newObjIdToKeyMap.get(id);
if (oldKeyName == null && newKeyName == null) {
// This cannot happen.
throw new IllegalStateException(
"Old and new key name both are null");
} else if (oldKeyName == null) { // Key Created.
String key = resolveBucketRelativePath(isFSOBucket,
newParentIdPathMap, newKeyName, true);
if (key != null) {
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(CREATE, key);
createDiffs.add(codecRegistry.asRawData(entry));
}
} else if (newKeyName == null) { // Key Deleted.
String key = resolveBucketRelativePath(isFSOBucket,
oldParentIdPathMap, oldKeyName, false);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DELETE, key);
deleteDiffs.add(codecRegistry.asRawData(entry));
} else if (isDirectoryObject &&
Arrays.equals(oldKeyName, newKeyName)) {
String key = resolveBucketRelativePath(isFSOBucket,
newParentIdPathMap, newKeyName, true);
if (key != null) {
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(MODIFY, key);
modifyDiffs.add(codecRegistry.asRawData(entry));
}
} else {
String keyPrefix = getTablePrefix(tablePrefix,
(isDirectoryObject ? fsDirTable : fsTable).getName());
String oldKey = resolveBucketRelativePath(isFSOBucket,
oldParentIdPathMap, oldKeyName, false);
String newKey = resolveBucketRelativePath(isFSOBucket,
newParentIdPathMap, newKeyName, true);
if (newKey == null) {
deleteDiffs.add(codecRegistry.asRawData(SnapshotDiffReportOzone
.getDiffReportEntry(DELETE, oldKey)));
} else {
// Check if block location is same or not. If it is not same,
// key must have been overridden as well.
boolean isObjectModified = isObjectModified(
keyPrefix + codecRegistry.asObject(oldKeyName, String.class),
keyPrefix + codecRegistry.asObject(newKeyName, String.class),
isDirectoryObject ? fsDirTable : fsTable,
isDirectoryObject ? tsDirTable : tsTable);
if (isObjectModified) {
// Here, oldKey name is returned as modified. Modified key name
// is based on base snapshot (from snapshot).
modifyDiffs.add(codecRegistry.asRawData(SnapshotDiffReportOzone
.getDiffReportEntry(MODIFY, oldKey)));
}
if (!isObjectModified || !Arrays.equals(oldKeyName, newKeyName)) {
renameDiffs.add(codecRegistry.asRawData(
SnapshotDiffReportOzone.getDiffReportEntry(RENAME, oldKey,
newKey)));
}
}
}
counter++;
}
}
/*
* The order in which snap-diff should be applied
*
* 1. Delete diffs
* 2. Rename diffs
* 3. Create diffs
* 4. Modified diffs
*
* Consider the following scenario
*
* 1. File "A" is created.
* 2. File "B" is created.
* 3. File "C" is created.
* Snapshot "1" is taken.
*
* Case 1:
* 1. File "A" is deleted.
* 2. File "B" is renamed to "A".
* Snapshot "2" is taken.
*
* Snapshot diff should be applied in the following order:
* 1. Delete "A"
* 2. Rename "B" to "A"
*
*
* Case 2:
* 1. File "B" is renamed to "C".
* 2. File "B" is created.
* Snapshot "2" is taken.
*
* Snapshot diff should be applied in the following order:
* 1. Rename "B" to "C"
* 2. Create "B"
*
*/
long index = 0;
index = addToReport(jobId, index, deleteDiffs);
index = addToReport(jobId, index, renameDiffs);
index = addToReport(jobId, index, createDiffs);
return addToReport(jobId, index, modifyDiffs);
} catch (RocksDBException | IOException e) {
// TODO: [SNAPSHOT] Fail gracefully.
throw new RuntimeException(e);
} finally {
dropAndCloseColumnFamilyHandle(deleteDiffColumnFamily);
dropAndCloseColumnFamilyHandle(renameDiffColumnFamily);
dropAndCloseColumnFamilyHandle(createDiffColumnFamily);
dropAndCloseColumnFamilyHandle(modifyDiffColumnFamily);
}
}