in tokenizers/javascript/parser.js [36:55]
function extractForFolder(dir, identifiersOnly, outpath, baseDir) {
var all_files = listAllFilesRecursive(dir, '.js', []);
var all_json = []
for (var i in all_files) {
console.log('Opening ' + all_files[i]);
var filecontent = fs.readFileSync(all_files[i]).toString();
try {
var tokens = getTokens(filecontent, identifiersOnly);
var jsonStr = JSON.stringify({'filename': path.relative(baseDir, all_files[i]), 'tokens': tokens});
all_json.push(jsonStr);
} catch (err) {
console.warn('Error when extracting '+ all_files[i] + ' : ' + err);
}
}
var gzipped = new Buffer(all_json.join('\n'));
fs.writeFileSync(outpath, zlib.gzipSync(gzipped));
}