in tokenizers/CsharpTokenizer/CsharpTokenizer/Program.cs [29:44]
public static void ExtractForProjectFolder(string projectDir, string outputDir, bool onlyIdentifiers, string baseDir)
{
var allFiles = Directory.EnumerateFiles(projectDir, "*.cs", SearchOption.AllDirectories);
var projectDirName = Path.GetFileName(projectDir);
using (var fileStream = File.Create(Path.Combine(outputDir, projectDirName + ".jsonl.gz")))
using (var gzipStream = new GZipStream(fileStream, CompressionMode.Compress, false))
using (var textStream = new StreamWriter(gzipStream))
{
foreach (var fileJson in allFiles.AsParallel().Select(f => GetJsonForFile(f, onlyIdentifiers, baseDir)))
{
textStream.WriteLine(fileJson);
}
}
}