in src/Test.FunctionalTests.BinSkim.Driver/BaselineTests.cs [79:170]
private void RunRules(StringBuilder sb, string inputFileName)
{
string fileName = Path.GetFileName(inputFileName);
string actualDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
string expectedDirectory;
if (PlatformSpecificHelpers.RunningOnWindows())
{
expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");
}
else
{
expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "NonWindowsExpected");
}
if (!Directory.Exists(actualDirectory))
{
Directory.CreateDirectory(actualDirectory);
}
string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
string actualFileName = Path.Combine(actualDirectory, fileName + ".sarif");
var command = new AnalyzeCommand();
var options = new AnalyzeOptions
{
Force = true,
Recurse = false,
PrettyPrint = true,
DataToInsert = new[] { OptionallyEmittedData.Hashes },
DataToRemove = new[] { OptionallyEmittedData.NondeterministicProperties },
OutputFilePath = actualFileName,
ConfigurationFilePath = "default",
SarifOutputVersion = Sarif.SarifVersion.Current,
TargetFileSpecifiers = new string[] { inputFileName },
Traces = Array.Empty<string>(),
Level = new List<FailureLevel> { FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note, FailureLevel.None },
Kind = new List<ResultKind> { ResultKind.Fail, ResultKind.NotApplicable, ResultKind.Pass },
};
int result = command.Run(options);
// Note that we don't ensure a success code. That is because we
// are running end-to-end tests for valid and invalid files
var settings = new JsonSerializerSettings()
{
Formatting = Newtonsoft.Json.Formatting.Indented
};
string expectedText = File.ReadAllText(expectedFileName);
string actualText = File.ReadAllText(actualFileName);
// Replace repository root absolute path with Z:\ for machine and enlistment independence
string repoRoot = Path.GetFullPath(Path.Combine(actualDirectory, "..", "..", "..", ".."));
actualText = actualText.Replace(repoRoot.Replace(@"\", @"\\"), @"Z:");
actualText = actualText.Replace(repoRoot.Replace(@"\", @"/"), @"Z:");
// Remove stack traces as they can change due to inlining differences by configuration and runtime.
actualText = Regex.Replace(actualText, @"\\r\\n at [^""]+", "");
actualText = actualText.Replace(@"""Sarif""", @"""BinSkim""");
actualText = actualText.Replace(@" ""fileVersion"": ""15.0.0""," + Environment.NewLine, string.Empty);
actualText = Regex.Replace(actualText, @"\s*""fullName""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""semanticVersion""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @" ""id""[^,]+,\s+""tool""", @" ""tool""", RegexOptions.Multiline);
// Write back the normalized actual text so that the diff command given on failure shows what was actually compared.
Encoding utf8encoding = new UTF8Encoding(true);
using (var textWriter = new StreamWriter(actualFileName, false, utf8encoding))
{
textWriter.Write(actualText);
}
// Make sure we can successfully deserialize what was just generated
SarifLog expectedLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
expectedText,
settings.Formatting,
out expectedText);
SarifLog actualLog = JsonConvert.DeserializeObject<SarifLog>(actualText, settings);
var visitor = new ResultDiffingVisitor(expectedLog);
if (!visitor.Diff(actualLog.Runs[0].Results))
{
string errorMessage = "The output of the tool did not match for input {0}.";
sb.AppendLine(string.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
sb.AppendLine("Check differences with:");
sb.AppendLine(this.GenerateDiffCommand(expectedFileName, actualFileName));
}
}