def launch_eval()

in optimum/onnxruntime/runs/__init__.py [0:0]

• 31 lines of code
• 2 McCabe index (conditional complexity)

    def launch_eval(self):
        kwargs = self.task_processor.get_pipeline_kwargs()

        # transformers pipelines are smart enought to detect whether the tokenizer or feature_extractor is needed
        ort_pipeline = _optimum_pipeline(
            task=self.task,
            model=self.ort_model,
            tokenizer=self.preprocessor,
            feature_extractor=self.preprocessor,
            accelerator="ort",
            **kwargs,
        )

        transformers_pipeline = _transformers_pipeline(
            task=self.task,
            model=self.torch_model,
            tokenizer=self.preprocessor,
            feature_extractor=self.preprocessor,
            **kwargs,
        )

        eval_dataset = self.get_eval_dataset()

        print("Running evaluation...")
        baseline_metrics_dict = self.task_processor.run_evaluation(
            eval_dataset, transformers_pipeline, self.metric_names
        )
        optimized_metrics_dict = self.task_processor.run_evaluation(eval_dataset, ort_pipeline, self.metric_names)

        baseline_metrics_dict.pop("total_time_in_seconds", None)
        baseline_metrics_dict.pop("samples_per_second", None)
        baseline_metrics_dict.pop("latency_in_seconds", None)

        optimized_metrics_dict.pop("total_time_in_seconds", None)
        optimized_metrics_dict.pop("samples_per_second", None)
        optimized_metrics_dict.pop("latency_in_seconds", None)

        self.return_body["evaluation"]["others"]["baseline"].update(baseline_metrics_dict)
        self.return_body["evaluation"]["others"]["optimized"].update(optimized_metrics_dict)