def requires()

in leaderboard/tasks.py [0:0]

• 21 lines of code
• 11 McCabe index (conditional complexity)

    def requires(self):
        folds = ["dev"]
        if Path(conf["squad"]["submission_predictions"]["test"]).exists():
            folds.append("test")

        for fold in folds:
            for irt_type in IRT_TYPES:
                for evaluation in IRT_EVALUATION_TYPES:
                    if "no_irt" in irt_type or "no_irt" in evaluation:
                        continue

                    # Only need a trained model for the CAT sampling stability
                    # experiment
                    if evaluation == "subject_heldout":
                        if irt_type == "2PL" and fold == "dev":
                            yield PyroTrainIRT(
                                evaluation=evaluation,
                                irt_type=irt_type,
                                device=self.device,
                                fold=fold,
                            )
                    else:
                        yield EvaluateIRT(
                            evaluation=evaluation, irt_type=irt_type, device=self.device, fold=fold,
                        )