in hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java [991:1129]
public ListKeysResult listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix, int maxKeys)
throws IOException {
long startNanos = Time.monotonicNowNanos();
List<OmKeyInfo> result = new ArrayList<>();
if (maxKeys <= 0) {
return new ListKeysResult(result, false);
}
if (Strings.isNullOrEmpty(volumeName)) {
throw new OMException("Volume name is required.",
ResultCodes.VOLUME_NOT_FOUND);
}
if (Strings.isNullOrEmpty(bucketName)) {
throw new OMException("Bucket name is required.",
ResultCodes.BUCKET_NOT_FOUND);
}
String bucketNameBytes = getBucketKey(volumeName, bucketName);
if (getBucketTable().get(bucketNameBytes) == null) {
throw new OMException("Bucket " + bucketName + " not found.",
ResultCodes.BUCKET_NOT_FOUND);
}
String seekKey;
boolean skipStartKey = false;
if (StringUtil.isNotBlank(startKey)) {
// Seek to the specified key.
seekKey = getOzoneKey(volumeName, bucketName, startKey);
skipStartKey = true;
} else {
// This allows us to seek directly to the first key with the right prefix.
seekKey = getOzoneKey(volumeName, bucketName,
StringUtil.isNotBlank(keyPrefix) ? keyPrefix : OM_KEY_PREFIX);
}
String seekPrefix;
if (StringUtil.isNotBlank(keyPrefix)) {
seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix);
} else {
seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX;
}
int currentCount = 0;
TreeMap<String, OmKeyInfo> cacheKeyMap = new TreeMap<>();
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> iterator =
keyTable.cacheIterator();
//TODO: We can avoid this iteration if table cache has stored entries in
// treemap. Currently HashMap is used in Cache. HashMap get operation is an
// constant time operation, where as for treeMap get is log(n).
// So if we move to treemap, the get operation will be affected. As get
// is frequent operation on table. So, for now in list we iterate cache map
// and construct treeMap which match with keyPrefix and are greater than or
// equal to startKey. Later we can revisit this, if list operation
// is becoming slow.
while (iterator.hasNext()) {
Map.Entry< CacheKey<String>, CacheValue<OmKeyInfo>> entry =
iterator.next();
String key = entry.getKey().getCacheKey();
OmKeyInfo omKeyInfo = entry.getValue().getCacheValue();
// Making sure that entry in cache is not for delete key request.
if (omKeyInfo != null
&& key.startsWith(seekPrefix)
&& key.compareTo(seekKey) >= 0) {
cacheKeyMap.put(key, omKeyInfo);
}
}
long readFromRDbStartNs, readFromRDbStopNs = 0;
// Get maxKeys from DB if it has.
try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>>
keyIter = getKeyTable(getBucketLayout()).iterator()) {
readFromRDbStartNs = Time.monotonicNowNanos();
KeyValue< String, OmKeyInfo > kv;
keyIter.seek(seekKey);
// we need to iterate maxKeys + 1 here because if skipStartKey is true,
// we should skip that entry and return the result.
while (currentCount < maxKeys + 1 && keyIter.hasNext()) {
kv = keyIter.next();
if (kv != null && kv.getKey().startsWith(seekPrefix)) {
// Entry should not be marked for delete, consider only those
// entries.
CacheValue<OmKeyInfo> cacheValue =
keyTable.getCacheValue(new CacheKey<>(kv.getKey()));
if (cacheValue == null || cacheValue.getCacheValue() != null) {
cacheKeyMap.put(kv.getKey(), kv.getValue());
currentCount++;
}
} else {
// The SeekPrefix does not match any more, we can break out of the
// loop.
break;
}
}
readFromRDbStopNs = Time.monotonicNowNanos();
}
boolean isTruncated = cacheKeyMap.size() > maxKeys;
if (perfMetrics != null) {
long keyCount;
if (isTruncated) {
keyCount = maxKeys;
} else {
keyCount = cacheKeyMap.size();
}
perfMetrics.setListKeysAveragePagination(keyCount);
float opsPerSec =
keyCount / ((Time.monotonicNowNanos() - startNanos) / 1000000000.0f);
perfMetrics.setListKeysOpsPerSec(opsPerSec);
perfMetrics.addListKeysReadFromRocksDbLatencyNs(readFromRDbStopNs - readFromRDbStartNs);
}
// Finally DB entries and cache entries are merged, then return the count
// of maxKeys from the sorted map.
currentCount = 0;
for (Map.Entry<String, OmKeyInfo> cacheKey : cacheKeyMap.entrySet()) {
if (cacheKey.getKey().equals(seekKey) && skipStartKey) {
continue;
}
result.add(cacheKey.getValue());
currentCount++;
if (currentCount == maxKeys) {
break;
}
}
// Clear map and set.
cacheKeyMap.clear();
return new ListKeysResult(result, isTruncated);
}