leaderboard/cat_sampling_stability.py [280:410]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return { "info_df": info_df, "irt_df": irt_df, "random_df": random_df, } def create_random_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): for trial_id in range(10): random_items = self._random_sampler.sample(trial_size) random_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, random_items) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_random_score": random_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": trial_id, "dev_random_to_test": corr.loc["dev_random_score"].test_score, } ) random_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return random_df def create_irt_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): high_disc_items = self._high_disc_sampler.sample(trial_size) low_disc_items = self._low_disc_sampler.sample(trial_size) zero_disc_items = self._zero_disc_sampler.sample(trial_size) high_diff_items = self._high_diff_sampler.sample(trial_size) low_diff_items = self._low_diff_sampler.sample(trial_size) zero_diff_items = self._zero_diff_sampler.sample(trial_size) high_disc_diff_items = self._high_disc_diff_sampler.sample(trial_size) high_disc_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_disc_items ) low_disc_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, low_disc_items) zero_disc_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, zero_disc_items ) high_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_diff_items ) low_diff_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, low_diff_items) zero_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, zero_diff_items ) high_disc_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_disc_diff_items ) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_high_disc_score": high_disc_dev_ranking[sid], "dev_low_disc_score": low_disc_dev_ranking[sid], "dev_zero_disc_score": zero_disc_dev_ranking[sid], "dev_high_diff_score": high_diff_dev_ranking[sid], "dev_low_diff_score": low_diff_dev_ranking[sid], "dev_zero_diff_score": zero_diff_dev_ranking[sid], "dev_high_disc_diff_score": high_disc_diff_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": 0, "dev_high_disc_to_test": corr.loc["dev_high_disc_score"].test_score, #'dev_low_disc_to_test': corr.loc['dev_low_disc_score'].test_score, #'dev_zero_disc_to_test': corr.loc['dev_zero_disc_score'].test_score, "dev_high_diff_to_test": corr.loc["dev_high_diff_score"].test_score, #'dev_low_diff_to_test': corr.loc['dev_low_diff_score'].test_score, #'dev_zero_diff_to_test': corr.loc['dev_zero_diff_score'].test_score, "dev_high_disc_diff_to_test": corr.loc["dev_high_disc_diff_score"].test_score, } ) irt_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return irt_df def create_info_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): info_items = self._info_sampler.sample(trial_size) info_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, info_items) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_info_score": info_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": 0, "dev_info_to_test": corr.loc["dev_info_score"].test_score, } ) info_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return info_df @sampling_app.command() def run(max_size: int = 6000, step_size: int = 25, n_trials: int = 10): simulation = Simulation(max_size=max_size, step_size=step_size, n_trials=n_trials) simulation.run() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - leaderboard/sampling_stability.py [263:393]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - return { "info_df": info_df, "irt_df": irt_df, "random_df": random_df, } def create_random_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): for trial_id in range(10): random_items = self._random_sampler.sample(trial_size) random_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, random_items) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_random_score": random_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": trial_id, "dev_random_to_test": corr.loc["dev_random_score"].test_score, } ) random_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return random_df def create_irt_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): high_disc_items = self._high_disc_sampler.sample(trial_size) low_disc_items = self._low_disc_sampler.sample(trial_size) zero_disc_items = self._zero_disc_sampler.sample(trial_size) high_diff_items = self._high_diff_sampler.sample(trial_size) low_diff_items = self._low_diff_sampler.sample(trial_size) zero_diff_items = self._zero_diff_sampler.sample(trial_size) high_disc_diff_items = self._high_disc_diff_sampler.sample(trial_size) high_disc_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_disc_items ) low_disc_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, low_disc_items) zero_disc_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, zero_disc_items ) high_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_diff_items ) low_diff_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, low_diff_items) zero_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, zero_diff_items ) high_disc_diff_dev_ranking = self._dev_classic_ranker.rank( self._dev_subjects, high_disc_diff_items ) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_high_disc_score": high_disc_dev_ranking[sid], "dev_low_disc_score": low_disc_dev_ranking[sid], "dev_zero_disc_score": zero_disc_dev_ranking[sid], "dev_high_diff_score": high_diff_dev_ranking[sid], "dev_low_diff_score": low_diff_dev_ranking[sid], "dev_zero_diff_score": zero_diff_dev_ranking[sid], "dev_high_disc_diff_score": high_disc_diff_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": 0, "dev_high_disc_to_test": corr.loc["dev_high_disc_score"].test_score, #'dev_low_disc_to_test': corr.loc['dev_low_disc_score'].test_score, #'dev_zero_disc_to_test': corr.loc['dev_zero_disc_score'].test_score, "dev_high_diff_to_test": corr.loc["dev_high_diff_score"].test_score, #'dev_low_diff_to_test': corr.loc['dev_low_diff_score'].test_score, #'dev_zero_diff_to_test': corr.loc['dev_zero_diff_score'].test_score, "dev_high_disc_diff_to_test": corr.loc["dev_high_disc_diff_score"].test_score, } ) irt_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return irt_df def create_info_df(self): rows = [] for trial_size in track(range(self._step_size, self._n_dev_questions, self._step_size)): info_items = self._info_sampler.sample(trial_size) info_dev_ranking = self._dev_classic_ranker.rank(self._dev_subjects, info_items) rank_rows = [] for sid in self._dev_subjects: rank_rows.append( { "dev_id": sid, "dev_info_score": info_dev_ranking[sid], "test_score": self._remapped_test_ranking[sid], } ) rank_df = pd.DataFrame(rank_rows) corr = rank_df.corr(method="kendall") rows.append( { "trial_size": trial_size, "trial_id": 0, "dev_info_to_test": corr.loc["dev_info_score"].test_score, } ) info_df = pd.DataFrame(rows).melt(id_vars=["trial_size", "trial_id"]) return info_df @sampling_app.command() def run(max_size: int = 6000, step_size: int = 25, n_trials: int = 10): simulation = Simulation(max_size=max_size, step_size=step_size, n_trials=n_trials) simulation.run() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -