def load_datasets()

in src/lighteval/tasks/lighteval_task.py [0:0]

• 29 lines of code
• 5 McCabe index (conditional complexity)

    def load_datasets(tasks: dict[str, "LightevalTask"], dataset_loading_processes: int = 1) -> None:
        """
        Load datasets from the HuggingFace Hub for the given tasks.

        Args:
            tasks (list): A list of tasks.
            dataset_loading_processes (int, optional): number of processes to use for dataset loading. Defaults to 1.

        Returns:
            None
        """

        if dataset_loading_processes <= 1:
            datasets = [
                download_dataset_worker(
                    task.dataset_path,
                    task.dataset_config_name,
                    task.trust_dataset,
                    task.dataset_filter,
                    task.dataset_revision,
                )
                for task in tasks.values()
            ]
        else:
            with Pool(processes=dataset_loading_processes) as pool:
                datasets = pool.starmap(
                    download_dataset_worker,
                    [
                        (
                            task.dataset_path,
                            task.dataset_config_name,
                            task.trust_dataset,
                            task.dataset_filter,
                            task.dataset_revision,
                        )
                        for task in tasks.values()
                    ],
                )

        for task, dataset in zip(tasks, datasets):
            tasks[task].dataset = dataset