def validation_metric_vw()

in scripts/vw-hyperopt.py [0:0]

• 31 lines of code
• 11 McCabe index (conditional complexity)

    def validation_metric_vw(self):
        y_pred_holdout = self.get_y_pred_holdout()

        if self.outer_loss_function == "logistic":
            if self.labels_clf_count > 2:
                y_pred_holdout_proba = y_pred_holdout
            else:
                y_pred_holdout_proba = [1.0 / (1 + exp(-i)) for i in y_pred_holdout]

            loss = log_loss(self.y_true_holdout, y_pred_holdout_proba)

        elif self.outer_loss_function == "squared":
            loss = mean_squared_error(self.y_true_holdout, y_pred_holdout)

        elif self.outer_loss_function == "hinge":
            loss = hinge_loss(self.y_true_holdout, y_pred_holdout)

        elif self.outer_loss_function == "pr-auc":
            loss = -average_precision_score(self.y_true_holdout, y_pred_holdout)

        elif self.outer_loss_function == "roc-auc":
            y_pred_holdout_proba = [1.0 / (1 + exp(-i)) for i in y_pred_holdout]
            fpr, tpr, _ = roc_curve(self.y_true_holdout, y_pred_holdout_proba)
            loss = -auc(fpr, tpr)

        elif self.outer_loss_function == "quantile":  # Minimum at Median
            tau = 0.5
            loss = np.mean(
                [
                    max(tau * (true - pred), (tau - 1) * (true - pred))
                    for true, pred in zip(self.y_true_holdout, y_pred_holdout)
                ]
            )

        else:
            raise KeyError("Invalide outer loss function")

        self.logger.info("parameter suffix: %s" % self.param_suffix)
        self.logger.info("loss value: %.6f" % loss)

        return loss