in storm-webapp/src/main/java/org/apache/storm/daemon/logviewer/utils/DirectoryCleaner.java [90:180]
public DeletionMeta deleteOldestWhileTooLarge(List<Path> dirs,
long quota, boolean forPerDir, Set<Path> activeDirs) throws IOException {
long totalSize = 0;
for (Path dir : dirs) {
try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
for (Path path : stream) {
totalSize += Files.size(path);
}
}
}
LOG.debug("totalSize: {} quota: {}", totalSize, quota);
long toDeleteSize = totalSize - quota;
if (toDeleteSize <= 0) {
return DeletionMeta.EMPTY;
}
int deletedFiles = 0;
long deletedSize = 0;
// the oldest pq_size files in this directory will be placed in PQ, with the newest at the root
PriorityQueue<Pair<Path, FileTime>> pq = new PriorityQueue<>(PQ_SIZE,
Comparator.comparing((Pair<Path, FileTime> p) -> p.getRight()).reversed());
int round = 0;
final Set<Path> excluded = new HashSet<>();
while (toDeleteSize > 0) {
LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round);
for (Path dir : dirs) {
try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
for (Path path : stream) {
if (!excluded.contains(path)) {
if (isFileEligibleToSkipDelete(forPerDir, activeDirs, dir, path)) {
excluded.add(path);
} else {
Pair<Path, FileTime> p = Pair.of(path, Files.getLastModifiedTime(path));
if (pq.size() < PQ_SIZE) {
pq.offer(p);
} else if (p.getRight().toMillis() < pq.peek().getRight().toMillis()) {
pq.poll();
pq.offer(p);
}
}
}
}
}
}
if (!pq.isEmpty()) {
// need to reverse the order of elements in PQ to delete files from oldest to newest
Stack<Pair<Path, FileTime>> stack = new Stack<>();
while (!pq.isEmpty()) {
stack.push(pq.poll());
}
while (!stack.isEmpty() && toDeleteSize > 0) {
Pair<Path, FileTime> pair = stack.pop();
Path file = pair.getLeft();
final String canonicalPath = file.toAbsolutePath().normalize().toString();
final long fileSize = Files.size(file);
final long lastModified = pair.getRight().toMillis();
//Original implementation doesn't actually check if delete succeeded or not.
try {
Utils.forceDelete(file.toString());
LOG.info("Delete file: {}, size: {}, lastModified: {}", canonicalPath, fileSize, lastModified);
toDeleteSize -= fileSize;
deletedSize += fileSize;
deletedFiles++;
} catch (IOException e) {
excluded.add(file);
}
}
pq.clear();
round++;
if (round >= MAX_ROUNDS) {
if (forPerDir) {
LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in "
+ "a single directory : {}, will delete the rest files in next interval.",
MAX_ROUNDS, dirs.get(0).toAbsolutePath().normalize());
} else {
LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, "
+ "will delete the rest files in next interval.", MAX_ROUNDS);
}
break;
}
} else {
LOG.warn("No more files able to delete this round, but {} is over quota by {} MB",
forPerDir ? "this directory" : "root directory", toDeleteSize * 1e-6);
LOG.warn("No more files eligible to be deleted this round, but {} is over {} quota by {} MB",
forPerDir ? "worker directory: " + dirs.get(0).toAbsolutePath().normalize() : "log root directory",
forPerDir ? "per-worker" : "global", toDeleteSize * 1e-6);
break; // No entries left to delete
}
}
return new DeletionMeta(deletedSize, deletedFiles);
}