def evaluate_classification_pipeline_on_qats()

in tseval/qats.py [0:0]

• 14 lines of code
• 1 McCabe index (conditional complexity)

def evaluate_classification_pipeline_on_qats(aspect, pipeline, method_name):
    train_sentences, train_labels = get_qats_train_data(aspect=aspect)
    valid_scores = cross_val_score(pipeline, train_sentences, train_labels, cv=10, scoring='f1_weighted')
    valid_score, cross_val_confidence_interval = mean_with_confidence_interval(valid_scores, 0.95)
    pipeline.fit(train_sentences, train_labels)
    test_sentences, test_labels = get_qats_test_data(aspect=aspect)
    test_score = f1_score(test_labels, pipeline.predict(test_sentences), average='weighted')
    return {
        'team': method_name,
        'valid_weighted_f_score': valid_score,
        # 95% confidence interval of the cross validation mean: valid_score +- condience_interval
        'valid_conf_int': cross_val_confidence_interval,
        'weighted_f_score': test_score,
    }