leaderboard/cat_sampling_stability.py [35:109]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    def sample(self, n_items: int):
        pass


class RandomSampler(Sampler):
    def sample(self, n_items: int):
        return random.sample(self._item_ids, n_items)


def create_sort_key(param_key: str):
    def sort_key(item_stats):
        if param_key == "high_disc":
            return item_stats.disc
        elif param_key == "low_disc":
            return -item_stats.disc
        elif param_key == "zero_disc":
            return -abs(item_stats.disc)
        elif param_key == "high_diff":
            return item_stats.diff
        elif param_key == "low_diff":
            return -item_stats.diff
        elif param_key == "zero_diff":
            return -abs(item_stats.diff)
        elif param_key == "high_disc_diff":
            return item_stats.disc + item_stats.diff
        else:
            raise ValueError("Invalid sort key")

    return sort_key


class IrtSampler(Sampler):
    def __init__(self, *, fold: str, param_key: str, irt_model: str):
        super().__init__(fold)
        self._param_key = param_key
        self._irt_model = IrtParsed.from_irt_file(
            Path(conf["irt"]["squad"][fold]["pyro"][irt_model]["full"]) / "parameters.json"
        )

    def sample(self, n_items: int):
        item_stats = {}
        for item_id in self._item_ids:
            item_stats[item_id] = self._irt_model.example_stats[item_id]

        sorted_items = sorted(
            item_stats.values(), key=create_sort_key(self._param_key), reverse=True
        )
        return [item.example_id for item in sorted_items[:n_items]]


def prob_2pl(*, skill: float, disc: float, diff: float) -> float:
    exponent = -disc * (skill - diff)
    return 1 / (1 + np.exp(exponent))


CACHE_PREDICTIONS = {}


def cached_leaderboard_predictions(fold: str) -> LeaderboardPredictions:
    if fold in CACHE_PREDICTIONS:
        return CACHE_PREDICTIONS[fold]
    else:
        preds = LeaderboardPredictions.parse_file(conf["squad"]["submission_predictions"][fold])
        CACHE_PREDICTIONS[fold] = preds
        return preds


CACHE_IRT = {}


def cached_irt(irt_type: str) -> IrtParsed:
    if irt_type in CACHE_IRT:
        return CACHE_IRT[irt_type]
    else:
        irt_model = IrtParsed.from_irt_file(
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



leaderboard/sampling_stability.py [34:108]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    def sample(self, n_items: int):
        pass


class RandomSampler(Sampler):
    def sample(self, n_items: int):
        return random.sample(self._item_ids, n_items)


def create_sort_key(param_key: str):
    def sort_key(item_stats):
        if param_key == "high_disc":
            return item_stats.disc
        elif param_key == "low_disc":
            return -item_stats.disc
        elif param_key == "zero_disc":
            return -abs(item_stats.disc)
        elif param_key == "high_diff":
            return item_stats.diff
        elif param_key == "low_diff":
            return -item_stats.diff
        elif param_key == "zero_diff":
            return -abs(item_stats.diff)
        elif param_key == "high_disc_diff":
            return item_stats.disc + item_stats.diff
        else:
            raise ValueError("Invalid sort key")

    return sort_key


class IrtSampler(Sampler):
    def __init__(self, *, fold: str, param_key: str, irt_model: str):
        super().__init__(fold)
        self._param_key = param_key
        self._irt_model = IrtParsed.from_irt_file(
            Path(conf["irt"]["squad"][fold]["pyro"][irt_model]["full"]) / "parameters.json"
        )

    def sample(self, n_items: int):
        item_stats = {}
        for item_id in self._item_ids:
            item_stats[item_id] = self._irt_model.example_stats[item_id]

        sorted_items = sorted(
            item_stats.values(), key=create_sort_key(self._param_key), reverse=True
        )
        return [item.example_id for item in sorted_items[:n_items]]


def prob_2pl(*, skill: float, disc: float, diff: float) -> float:
    exponent = -disc * (skill - diff)
    return 1 / (1 + np.exp(exponent))


CACHE_PREDICTIONS = {}


def cached_leaderboard_predictions(fold: str) -> LeaderboardPredictions:
    if fold in CACHE_PREDICTIONS:
        return CACHE_PREDICTIONS[fold]
    else:
        preds = LeaderboardPredictions.parse_file(conf["squad"]["submission_predictions"][fold])
        CACHE_PREDICTIONS[fold] = preds
        return preds


CACHE_IRT = {}


def cached_irt(irt_type: str) -> IrtParsed:
    if irt_type in CACHE_IRT:
        return CACHE_IRT[irt_type]
    else:
        irt_model = IrtParsed.from_irt_file(
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -