def bless_directionality_setup()

in hypernymysuite/evaluation.py [0:0]

• 24 lines of code
• 1 McCabe index (conditional complexity)

def bless_directionality_setup(model):
    """
    Asks a model whether (x, y) > (y, x) for a number of positive hypernymy examples.
    """
    # load up the data
    ds = Dataset(os.path.join(DATA_DIR, "bless.tsv"), model.vocab)

    # only keep the positive pairs
    hypos = ds.hypos[ds.y]
    hypers = ds.hypers[ds.y]

    forward_predictions = model.predict_many(hypos, hypers)
    reverse_predictions = model.predict_many(hypers, hypos)

    # Fold masks
    m_val = ds.val_mask[ds.y]
    m_test = ds.test_mask[ds.y]

    # Fold, in-vocab masks
    oov = ds.oov_mask[ds.y]
    mi_val = m_val & ~oov
    mi_test = m_test & ~oov

    # final input
    # Check that the original directionality is correct
    yhat = forward_predictions > reverse_predictions

    return {
        "acc_val": np.mean(yhat[m_val]),
        "acc_test": np.mean(yhat[m_test]),
        "acc_all": np.mean(yhat),
        "acc_val_inv": np.mean(yhat[mi_val]),
        "acc_test_inv": np.mean(yhat[mi_test]),
        "acc_all_inv": np.mean(yhat[~oov]),
        "num_val": int(sum(m_val)),
        "num_test": int(sum(m_test)),
        "num_oov_all": int(sum(oov)),
        "pct_oov_all": np.mean(oov),
    }