async run()

in packages/core/src/suites.js [19:60]
37 lines of code
5 McCabe index (conditional complexity)

  async run() {
    const { task, model_id, options } = this.config;

    const { result: pipe, time: setupTime } = await time(() =>
      pipeline(task, model_id, {
        ...DEFAULT_MODEL_OPTIONS,
        ...options,
      }),
    );

    const stats = {};
    for (const test of this.tests) {
      const { inputs, expected, test_function, num_runs } = test;

      const times = [];
      const numRuns = DEFAULT_NUM_WARMUP_RUNS + (num_runs ?? DEFAULT_NUM_RUNS);
      for (let i = 0; i < numRuns; ++i) {
        const { result, time: executionTime } = await time(() =>
          pipe(...inputs),
        );
        const { pass, message } = (test_function ?? toBeCloseToNested)(
          result,
          expected,
        );
        if (!pass) {
          console.log(result);
          console.log(expected);
          throw new Error(message());
        }
        if (i >= DEFAULT_NUM_WARMUP_RUNS) times.push(executionTime);
      }
      stats[test.name] = computeStatistics(times);
    }

    const { time: disposeTime } = await time(() => pipe.dispose());

    return {
      setupTime,
      stats,
      disposeTime,
    };
  }