in tools/perf-automation/Azure.Sdk.Tools.PerfAutomation/Program.cs [246:416]
private static async Task RunPackageVersion(
Options options,
string service,
string project,
string primaryPackage,
IDictionary<string, string> packageVersions,
IEnumerable<TestInfo> tests,
string outputJson,
string outputCsv,
string outputTxt,
string outputMd,
List<Result> results)
{
var language = options.Language;
var languageVersion = options.LanguageVersion;
_languages[language].WorkingDirectory = options.RepoRoot;
try
{
Console.WriteLine($"SetupAsync({project}, {languageVersion}, " +
$"{JsonSerializer.Serialize(packageVersions)})");
Console.WriteLine();
string setupOutput = null;
string setupError = null;
object context = null;
string setupException = null;
try
{
(setupOutput, setupError, context) = await _languages[language].SetupAsync(
project, languageVersion, primaryPackage, packageVersions, options.Debug);
}
catch (Exception e)
{
setupException = e.ToString();
Console.WriteLine(e);
Console.WriteLine();
}
foreach (var test in tests)
{
IEnumerable<string> selectedArguments;
if (!options.NoAsync && !options.NoSync)
{
selectedArguments = test.Arguments.SelectMany(a => new string[] { a, a + " --sync" });
}
else if (!options.NoSync)
{
selectedArguments = test.Arguments.Select(a => a + " --sync");
}
else if (!options.NoAsync)
{
selectedArguments = test.Arguments;
}
else
{
throw new InvalidOperationException("Cannot set both --no-sync and --no-async");
}
foreach (var arguments in selectedArguments)
{
var allArguments = arguments;
if (options.Insecure)
{
allArguments += " --insecure";
}
if (options.TestProxies != null && options.TestProxies.Any())
{
allArguments += $" --test-proxies {String.Join(';', options.TestProxies)}";
}
if (options.TestProxy != null)
{
allArguments += $" --test-proxy {options.TestProxy}";
}
var result = new Result
{
Service = service,
Test = test.Test,
Start = DateTime.Now,
Language = language,
LanguageVersion = languageVersion,
Project = project,
LanguageTestName = test.Class,
Arguments = allArguments,
PrimaryPackage = primaryPackage,
PackageVersions = packageVersions,
SetupStandardOutput = setupOutput,
SetupStandardError = setupError,
SetupException = setupException,
};
results.Add(result);
await WriteResults(outputJson, outputCsv, outputTxt, outputMd, results);
if (setupException == null)
{
for (var i = 0; i < options.Iterations; i++)
{
IterationResult iterationResult;
try
{
Console.WriteLine($"RunAsync({project}, {languageVersion}, " +
$"{test.Class}, {allArguments}, {context}, {options.Profile}, {options.ProfilerOptions})");
Console.WriteLine();
iterationResult = await _languages[language].RunAsync(
project,
languageVersion,
primaryPackage,
packageVersions,
test.Class,
allArguments,
options.Profile,
options.ProfilerOptions,
context);
}
catch (Exception e)
{
iterationResult = new IterationResult
{
OperationsPerSecond = double.MinValue,
Exception = e.ToString(),
};
Console.WriteLine(e);
Console.WriteLine();
}
// Replace non-finite values with minvalue, since non-finite values
// are not JSON serializable
if (!double.IsFinite(iterationResult.OperationsPerSecond))
{
iterationResult.OperationsPerSecond = double.MinValue;
}
result.Iterations.Add(iterationResult);
await WriteResults(outputJson, outputCsv, outputTxt, outputMd, results);
}
}
result.End = DateTime.Now;
}
}
}
finally
{
if (!options.NoCleanup)
{
Console.WriteLine($"CleanupAsync({project})");
Console.WriteLine();
try
{
await _languages[language].CleanupAsync(project);
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine();
}
}
}
}