in tokenizers/java/src/main/java/javatokenizer/Extractor.java [51:82]
public static void ExtractForFolder(File projectFolder, File outputFolder, boolean onlyIdentifiers, File baseFolder) {
Iterator<File> allFiles = FileUtils.iterateFiles(projectFolder, new String[] {"java"}, true);
try {
FileOutputStream output = new FileOutputStream(Paths.get(outputFolder.toPath().toString(), projectFolder.getName() + ".jsonl.gz").toFile());
Gson gson = new GsonBuilder().create();
try {
Writer writer = new OutputStreamWriter(new GZIPOutputStream(output), "UTF-8");
Iterable<File> fileIter = ()-> allFiles;
StreamSupport.stream(
fileIter.spliterator(), true).map(f-> TokenizeFile(f, onlyIdentifiers, baseFolder))
.map(t->gson.toJson(t)).filter(g->g!=null).sequential().forEach(g->{
try{
writer.write(g);
writer.write('\n');
} catch (IOException ioe) {
// really?
ioe.printStackTrace();
}
});
writer.close();
} catch(Exception e) {
System.out.println("Error for project " + projectFolder + ": " + e);
e.printStackTrace();
} finally {
output.close();
}
} catch (IOException e) {
System.out.println("Error for project " + projectFolder + ": " + e);
e.printStackTrace();
}
}