def run_oaievalset()

in ultravox/inference/run_vllm_inference.py [0:0]

• 28 lines of code
• 2 McCabe index (conditional complexity)

def run_oaievalset(args: InferenceArgs, log_dir: str):
    env = os.environ.copy()
    env["EVALS_THREADS"] = "256"
    env["ULTRAVOX_API_KEY"] = "..."  # API key is not neede for local evaluation

    command = [
        "oaievalset",
        "--record_dir",
        log_dir,
        "generation/direct/vllm",
        args.evalset,
        "--registry_path",
        "ultravox/inference/oaieval_registry",
    ]

    # Run the evaluation set
    subprocess.run(command, check=True, env=env)

    # Extract the results from the log directory
    subprocess.run(
        [
            "python",
            "-m",
            "evals.elsuite.audio.make_table",
            "--out_dir",
            log_dir,
            "--log_dir",
            log_dir,
        ],
        check=True,
    )

    df = pd.read_csv(os.path.join(log_dir, "results.csv"))

    return df