void StartTest()

in benchmarks/rnnt/ootb/inference/loadgen/loadgen.cc [1515:1609]

• 81 lines of code
• 8 McCabe index (conditional complexity)

void StartTest(SystemUnderTest* sut, QuerySampleLibrary* qsl,
               const TestSettings& requested_settings,
               const LogSettings& log_settings) {
  GlobalLogger().StartIOThread();

  const std::string test_date_time = CurrentDateTimeISO8601();

  loadgen::LogOutputs log_outputs(log_settings.log_output, test_date_time);
  if (!log_outputs.CheckOutputs()) {
    return;
  }

  GlobalLogger().StartLogging(&log_outputs.summary_out, &log_outputs.detail_out,
                              &log_outputs.accuracy_out,
                              log_settings.log_output.copy_detail_to_stdout,
                              log_settings.log_output.copy_summary_to_stdout);

  if (log_settings.enable_trace) {
    GlobalLogger().StartNewTrace(&log_outputs.trace_out, PerfClock::now());
  }

  LogLoadgenVersion();
  LogDetail([sut, qsl, test_date_time](AsyncDetail& detail) {
#if USE_NEW_LOGGING_FORMAT
    MLPERF_LOG(detail, "test_datetime", test_date_time);
    MLPERF_LOG(detail, "sut_name", sut->Name());
    MLPERF_LOG(detail, "qsl_name", qsl->Name());
    MLPERF_LOG(detail, "qsl_reported_total_count", qsl->TotalSampleCount());
    MLPERF_LOG(detail, "qsl_reported_performance_count",
               qsl->PerformanceSampleCount());
#else
    detail("Date + time of test: ", test_date_time);
    detail("System Under Test (SUT) name: ", sut->Name());
    detail("Query Sample Library (QSL) name: ", qsl->Name());
    detail("QSL total size: ", qsl->TotalSampleCount());
    detail("QSL performance size*: ", qsl->PerformanceSampleCount());
    detail("*TestSettings (performance_sample_count_override) can override");
    detail("*Refer to Effective Settings for actual value");
#endif
  });

  TestSettings test_settings = requested_settings;
  // Look for Audit Config file to override TestSettings during audit
  const std::string audit_config_filename = "audit.config";
  if (FileExists(audit_config_filename)) {
    LogDetail([](AsyncDetail& detail) {
#if USE_NEW_LOGGING_FORMAT
      MLPERF_LOG_WARNING(detail, "warning_generic_message",
                         "Found Audit Config file (audit.config)."
                         " Overriding TestSettings from audit.config file.");
#else
      detail(
          "Found Audit Config file (audit.config)."
          " Overriding TestSettings from audit.config file.");
#endif
    });
    std::string audit_scenario = loadgen::ToString(test_settings.scenario);
    // Remove Spaces from the string
    RemoveValue(&audit_scenario, ' ');
    const std::string generic_model = "*";
    test_settings.FromConfig(audit_config_filename, generic_model,
                             audit_scenario);
  }

  loadgen::TestSettingsInternal sanitized_settings(
      test_settings, qsl->PerformanceSampleCount());
  sanitized_settings.LogAllSettings();

  auto run_funcs = loadgen::RunFunctions::Get(sanitized_settings.scenario);

  loadgen::SequenceGen sequence_gen;
  switch (sanitized_settings.mode) {
    case TestMode::SubmissionRun:
      run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen);
      run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen);
      break;
    case TestMode::AccuracyOnly:
      run_funcs.accuracy(sut, qsl, sanitized_settings, &sequence_gen);
      break;
    case TestMode::PerformanceOnly:
      run_funcs.performance(sut, qsl, sanitized_settings, &sequence_gen);
      break;
    case TestMode::FindPeakPerformance:
      run_funcs.find_peak_performance(sut, qsl, sanitized_settings,
                                      &sequence_gen);
      break;
  }

  loadgen::IssueQueryController::GetInstance().EndThreads();

  // Stop tracing after logging so all logs are captured in the trace.
  GlobalLogger().StopLogging();
  GlobalLogger().StopTracing();
  GlobalLogger().StopIOThread();
}