def get_value_status()

in lnt/server/reporting/analysis.py [0:0]
37 lines of code
21 McCabe index (conditional complexity)

    def get_value_status(self, confidence_interval=2.576,
                         value_precision=MIN_VALUE_PRECISION,
                         ignore_small=True):
        if self.current is None or self.previous is None:
            return None

        # Don't report value errors for tests which fail, or which just started
        # passing.
        #
        # FIXME: One bug here is that we risk losing performance data on tests
        # which flop to failure then back. What would be nice to do here is to
        # find the last value in a passing run, or to move to using proper keyed
        # reference runs.
        if self.failed:
            return UNCHANGED_FAIL
        elif self.prev_failed:
            return UNCHANGED_PASS 

        # Always ignore percentage changes below 1%, for now, we just don't have
        # enough time to investigate that level of stuff.
        if ignore_small and abs(self.pct_delta) < .01:
            return UNCHANGED_PASS

        # Always ignore changes with small deltas. There is no mathematical
        # basis for this, it should be obviated by appropriate statistical
        # checks, but practical evidence indicates what we currently have isn't
        # good enough (for reasons I do not yet understand).
        if ignore_small and abs(self.delta) < .01:
            return UNCHANGED_PASS

        # Ignore tests whose delta is too small relative to the precision we can
        # sample at; otherwise quantization means that we can't measure the
        # standard deviation with enough accuracy.
        if abs(self.delta) <= 2 * value_precision * confidence_interval:
            return UNCHANGED_PASS

        # Use Mann-Whitney U test to test null hypothesis that result is
        # unchanged.
        if len(self.samples) >= 4 and len(self.prev_samples) >= 4:
            same = stats.mannwhitneyu(self.samples, self.prev_samples,
                                      self.confidence_lv)
            if same:
                return UNCHANGED_PASS

        # If we have a comparison window, then measure using a symmetic
        # confidence interval.
        if self.stddev is not None:
            is_significant = abs(self.delta) > (self.stddev *
                                                confidence_interval)

            # If the delta is significant, return
            if is_significant:
                if self.delta < 0:
                    return REGRESSED if self.bigger_is_better else IMPROVED
                else:
                    return IMPROVED if self.bigger_is_better else REGRESSED
            else:
                return UNCHANGED_PASS

        # Otherwise, report any changes above 0.2%, which is a rough
        # approximation for the smallest change we expect "could" be measured
        # accurately.
        if not ignore_small or abs(self.pct_delta) >= .002:
            if self.pct_delta < 0:
                return REGRESSED if self.bigger_is_better else IMPROVED
            else:
                return IMPROVED if self.bigger_is_better else REGRESSED
        else:
            return UNCHANGED_PASS