def evaluate_scoring_method_on_qats()

in tseval/qats.py [0:0]

• 18 lines of code
• 1 McCabe index (conditional complexity)

def evaluate_scoring_method_on_qats(aspect, scoring_method):
    '''Evaluates a scoring method with signature method(complex_sentence, simple_sentence)'''
    pipeline = Pipeline([('raw_scoring', FunctionPredictor(row_vectorize(scoring_method)))])
    train_sentences, train_labels = get_qats_train_data(aspect=aspect)
    pred_train_labels = pipeline.predict(train_sentences)
    train_score, p_value, conf_int_low, conf_int_high = pearsonr_with_confidence_interval(train_labels,
                                                                                          pred_train_labels)
    test_sentences, test_labels = get_qats_test_data(aspect=aspect)
    test_score = pearson(test_labels, pipeline.predict(test_sentences))
    return {
        'team': scoring_method.__name__,
        'valid_pearson': train_score,
        'valid_p': p_value,
        # 95% Confidence interval on pearson correlation: r in [valid_conf_int_low, valid_conf_int_high]
        'valid_conf_int_low': conf_int_low,
        'valid_conf_int_high': conf_int_high,
        'pearson': test_score,
    }