in leaderboard/visualize.py [0:0]
def main():
st.header("Welcome to the SQuAD 2.0 IRT Leaderboard!")
st.markdown(
"""
On this page, you can inspect the IRT parameters
and predictions of each 1D IRT model in our paper.
In order, we show the list of subjects, items,
and a subject-item response inspector.
On the left column, there are controls for each table
which let you sort by:
* Each IRT parameter
* SQuAD evaluation metrics
* In ascending or descending order
* To view the next page, click the -/+ buttons
"""
)
st.sidebar.title("IRT Viewer")
st.sidebar.markdown(DESC)
st.sidebar.subheader("IRT Model Selection")
irt_model = st.sidebar.selectbox("IRT Model", ["3PL", "2PL", "1PL"])
st.sidebar.latex(r"\theta=\text{ability/skill}, \beta=\text{difficulty}")
st.sidebar.latex(r"\gamma=\text{discriminability}, \lambda=\text{feasibility}")
st.sidebar.latex(irt_expression[irt_model])
model_df, item_df = load_data(irt_model)
st.sidebar.subheader("Subject Filters")
subject_sort_col_primary = st.sidebar.selectbox(
"Sort By (Subject)", ["skill", "test_em", "test_f1", "dev_em", "dev_f1", "created"],
)
subject_ascending = st.sidebar.checkbox("Ascending (Subject)", value=False)
subject_offset = st.sidebar.number_input(
"Subject Offset (Size: %d)" % len(model_df),
min_value=0,
max_value=int(len(item_df)) - STEP,
value=0,
step=STEP,
)
st.sidebar.subheader("Item Filters")
item_sort_col_primary = st.sidebar.selectbox("Primary Sort By (Item)", ITEM_SORT_COLS)
item_ascending = st.sidebar.checkbox("Ascending (Item)", value=True)
item_abs = st.sidebar.checkbox("Absolute Value", value=False)
if item_abs:
item_sort_key = np.abs
else:
item_sort_key = None
item_offset = st.sidebar.number_input(
"Item Offset (Size: %d)" % len(item_df),
min_value=0,
max_value=int(len(item_df)) - STEP,
value=0,
step=STEP,
)
st.header("Subjects")
st.subheader("This table shows all the subjects (SQuAD Models)")
st.table(
model_df.sort_values(subject_sort_col_primary, ascending=subject_ascending).iloc[
subject_offset : subject_offset + STEP
]
)
st.header("Items")
st.subheader("The table, chart, and inspector show each item (question)")
id_to_question = load_cached_squad()
all_item_ids = list(id_to_question)
item_accuracy, item_f1 = get_item_accuracy()
left, right = st.beta_columns(2)
irt_params = get_irt_model(irt_model)
with left:
st.subheader("Hover over points to see more about it")
st.write(irt_chart(irt_model))
with right:
st.subheader("Choose an Item ID to View")
inspect_item_id = st.selectbox("Choose an Item ID:", all_item_ids)
chosen_item = id_to_question[inspect_item_id].copy()
chosen_item["mean_exact_match"] = item_accuracy[inspect_item_id]
chosen_item["mean_f1"] = item_f1[inspect_item_id]
chosen_item["difficulty"] = irt_params.example_stats[inspect_item_id].diff
chosen_item["discriminability"] = irt_params.example_stats[inspect_item_id].disc
chosen_item["feasibility"] = irt_params.example_stats[inspect_item_id].lambda_
st.write(chosen_item)
st.table(
item_df.sort_values(
[item_sort_col_primary], ascending=item_ascending, key=item_sort_key
).iloc[item_offset : item_offset + STEP]
)
predictions = get_predictions()
all_subject_ids = model_df.dev_id.tolist()
st.sidebar.subheader("Subject-Item Response Inspector")
st.header("Subject-Item Response Inspector")
st.subheader("Here you can inspect IRT prediction for item-subject pairs")
item_id = st.sidebar.selectbox("Item ID", all_item_ids)
subject_id = st.sidebar.selectbox("Subject Dev ID", all_subject_ids)
st.write(id_to_question[item_id])
subject_probs = load_subject_probs(irt_model, subject_id)
prob = subject_probs[item_id]
correct = predictions.scored_predictions[subject_id]["exact_match"][item_id]
guess = load_subject_answer(subject_id, item_id)
if guess == "":
guess = "No Answer: Subject Abstains"
st.write(
{
"subject_id": subject_id,
"item_id": item_id,
"ability": irt_params.model_stats[subject_id].skill,
"difficulty": irt_params.example_stats[item_id].diff,
"discriminability": irt_params.example_stats[item_id].disc,
"feasibility": irt_params.example_stats[item_id].lambda_,
"p_correct": prob,
"subject_guess": guess,
"subject_answer_correct": correct,
}
)
st.sidebar.subheader("Subject-Item Filters")
subject_item_sort_col_primary = st.sidebar.selectbox(
"Sort By (Subject-Item)",
[
"irt_prob",
# "ability", "diff", "disc", "feasibility",
"abs(irt_prob - score)",
],
)
subject_item_ascending = st.sidebar.checkbox("Ascending", value=True)
subject_item_abs = st.sidebar.checkbox("Absolute Value (Subject-Item)", value=False)
if subject_item_abs:
subject_item_sort_key = np.abs
else:
subject_item_sort_key = None
subject_item_df = load_single_subject_df(irt_model, subject_id)
subject_item_offset = st.sidebar.number_input(
"Offset (Size: %d) Subject-Item" % len(subject_item_df),
min_value=0,
max_value=int(len(subject_item_df)) - STEP,
value=0,
step=STEP,
)
st.header("Subject-Item Responses")
st.table(
subject_item_df.sort_values(
[subject_item_sort_col_primary],
ascending=subject_item_ascending,
key=subject_item_sort_key,
).iloc[subject_item_offset : subject_item_offset + STEP]
)