in Sources/SwiftBenchmarkTools/DriverUtils.swift [105:258]
init(_ registeredBenchmarks: [BenchmarkInfo]) {
struct PartialTestConfig {
var delim: String?
var tags, skipTags: Set<BenchmarkCategory>?
var numSamples: UInt?
var numIters: UInt?
var quantile: UInt?
var timeUnit: String?
var delta: Bool?
var afterRunSleep: UInt32?
var sampleTime: Double?
var verbose: Bool?
var logMemory: Bool?
var action: TestAction?
var tests: [String]?
}
// Custom value type parsers
func tags(tags: String) throws -> Set<BenchmarkCategory> {
// We support specifying multiple tags by splitting on comma, i.e.:
// --tags=Array,Dictionary
// --skip-tags=Array,Set,unstable,skip
Set(
try tags.split(separator: ",").map(String.init).map {
try checked({ BenchmarkCategory(rawValue: $0) }, $0)
}
)
}
func finiteDouble(value: String) -> Double? {
Double(value).flatMap { $0.isFinite ? $0 : nil }
}
// Configure the command line argument parser
let p = ArgumentParser(into: PartialTestConfig())
p.addArgument(
"--num-samples", \.numSamples,
help: "number of samples to take per benchmark;\n" +
"default: 1 or auto-scaled to measure for\n" +
"`sample-time` if num-iters is also specified\n",
parser: { UInt($0) }
)
p.addArgument(
"--num-iters", \.numIters,
help: "number of iterations averaged in the sample;\n" +
"default: auto-scaled to measure for `sample-time`",
parser: { UInt($0) }
)
p.addArgument(
"--quantile", \.quantile,
help: "report quantiles instead of normal dist. stats;\n" +
"use 4 to get a five-number summary with quartiles,\n" +
"10 (deciles), 20 (ventiles), 100 (percentiles), etc.",
parser: { UInt($0) }
)
p.addArgument(
"--time-unit", \.timeUnit,
help: "time unit to be used for reported measurements;\n" +
"supported values: ns, us, ms; default: ns",
parser: { $0 }
)
p.addArgument(
"--delta", \.delta, defaultValue: true,
help: "report quantiles with delta encoding"
)
p.addArgument(
"--sample-time", \.sampleTime,
help: "duration of test measurement in seconds\ndefault: 1",
parser: finiteDouble
)
p.addArgument(
"--verbose", \.verbose, defaultValue: true,
help: "increase output verbosity"
)
p.addArgument(
"--memory", \.logMemory, defaultValue: true,
help: "log the change in maximum resident set size (MAX_RSS)"
)
p.addArgument(
"--delim", \.delim,
help: "value delimiter used for log output; default: ,",
parser: { $0 }
)
p.addArgument(
"--tags", \PartialTestConfig.tags,
help: "run tests matching all the specified categories",
parser: tags
)
p.addArgument(
"--skip-tags", \PartialTestConfig.skipTags, defaultValue: [],
help: "don't run tests matching any of the specified\n" +
"categories; default: unstable,skip",
parser: tags
)
p.addArgument(
"--sleep", \.afterRunSleep,
help: "number of seconds to sleep after benchmarking",
parser: { UInt32($0) }
)
p.addArgument(
"--list", \.action, defaultValue: .listTests,
help: "don't run the tests, just log the list of test \n" +
"numbers, names and tags (respects specified filters)"
)
p.addArgument(nil, \.tests) // positional arguments
let c = p.parse()
// Configure from the command line arguments, filling in the defaults.
self.delim = c.delim ?? ","
self.sampleTime = c.sampleTime ?? 1.0
self.numIters = c.numIters.map { Int($0) }
self.numSamples = c.numSamples.map { Int($0) }
self.quantile = c.quantile.map { Int($0) }
self.timeUnit = c.timeUnit.map { TimeUnit($0) } ?? TimeUnit.nanoseconds
self.delta = c.delta ?? false
self.verbose = c.verbose ?? false
self.logMemory = c.logMemory ?? false
self.afterRunSleep = c.afterRunSleep
self.action = c.action ?? .run
self.tests = TestConfig.filterTests(
registeredBenchmarks,
specifiedTests: Set(c.tests ?? []),
tags: c.tags ?? [],
skipTags: c.skipTags ?? [.unstable, .skip]
)
if self.logMemory, self.tests.count > 1 {
print(
"""
warning: The memory usage of a test, reported as the change in MAX_RSS,
is based on measuring the peak memory used by the whole process.
These results are meaningful only when running a single test,
not in the batch mode!
""")
}
// We always prepare the configuration string and call the print to have
// the same memory usage baseline between verbose and normal mode.
let testList = self.tests.map { $0.1.name }.joined(separator: ", ")
let configuration = """
--- CONFIG ---
NumSamples: \(numSamples ?? 0)
Verbose: \(verbose)
LogMemory: \(logMemory)
SampleTime: \(sampleTime)
NumIters: \(numIters ?? 0)
Quantile: \(quantile ?? 0)
TimeUnit: \(timeUnit)
Delimiter: \(String(reflecting: delim))
Tests Filter: \(c.tests ?? [])
Tests to run: \(testList)
--- DATA ---\n
"""
print(self.verbose ? configuration : "", terminator: "")
}