def grid_search()

in genai-on-vertex-ai/gemini/evals_playbook/utils/evals_playbook.py [0:0]

• 14 lines of code
• 6 McCabe index (conditional complexity)

    def grid_search(self, task_id, experiment_run_ids, opt_metrics, opt_params):
        """
        Performs grid search on the evaluation results and returns the best parameter combinations for each metric.

        Args:
            task_id: The specific task ID to filter the results.
            experiment_run_ids: List of experiment run IDs to include in the grid search.
            opt_metrics: List of metrics to optimize (e.g., ["ROUGE_1", "BLEU"]).
            opt_params: List of parameters to consider in the grid search (e.g., ["prompt_template", "temperature"]).

        Returns:
            A dictionary where keys are the optimization metrics and values are the corresponding best parameter combinations.
        """

        # Get 
        grid_df = (self.compare_eval_runs(experiment_run_ids)).T

        # Filter the grid_df based on the task_id
        filtered_df = grid_df[grid_df['task_id'] == task_id]

        # Initialize a dictionary to store the best parameter combinations for each metric
        best_params = {}

        for metric in opt_metrics:
            # Convert metric name to the corresponding column name in grid_df
            metric_mean_col = metric.lower().replace("_", "_") + "/mean" 
            metric_std_col = metric.lower().replace("_", "_") + "/std"

            # Find the row with the highest value for the given metric
            best_row = filtered_df.loc[filtered_df[metric_mean_col].idxmax()]

            # Extract the values of the optimization parameters and the std from the best row
            best_params[metric] = {
                "params": {param: best_row[param] for param in opt_params},
                "metric_mean": best_row[metric_mean_col],
                "metric_std": best_row[metric_std_col]
            }

        return best_params