testsuite/driver/runtests.py

#!/usr/bin/env python3

#
# (c) Simon Marlow 2002
#

import argparse
import signal
import sys
import os
import io
import shutil
import tempfile
import time
import re
import traceback

# We don't actually need subprocess in runtests.py, but:
# * We do need it in testlibs.py
# * We can't import testlibs.py until after we have imported ctypes
# * If we import ctypes before subprocess on cygwin, then sys.exit(0)
#   says "Aborted" and we fail with exit code 134.
# So we import it here first, so that the testsuite doesn't appear to fail.
import subprocess

from testutil import getStdout, Watcher, str_warn, str_info
from testglobals import getConfig, ghc_env, getTestRun, TestOptions, brokens
from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty
from junit import junit
import cpu_features

# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'
ghc_env['TERM'] = 'vt100'

global config
config = getConfig() # get it from testglobals

def signal_handler(signal, frame):
        stopNow()

# -----------------------------------------------------------------------------
# cmd-line options

parser = argparse.ArgumentParser(description="GHC's testsuite driver")
perf_group = parser.add_mutually_exclusive_group()

parser.add_argument("-e", action='append', help="A string to execute from the command line.")
parser.add_argument("--config-file", action="append", help="config file")
parser.add_argument("--config", action='append', help="config field")
parser.add_argument("--rootdir", action='append', help="root of tree containing tests (default: .)")
parser.add_argument("--metrics-file", help="file in which to save (append) the performance test metrics. If omitted, git notes will be used.")
parser.add_argument("--summary-file", help="file in which to save the (human-readable) summary")
parser.add_argument("--no-print-summary", action="store_true", help="should we print the summary?")
parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
parser.add_argument("--way", action="append", help="just this way")
parser.add_argument("--skipway", action="append", help="skip this way")
parser.add_argument("--threads", type=int, help="threads to run simultaneously")
parser.add_argument("--verbose", type=int, choices=[0,1,2,3,4,5], help="verbose (Values 0 through 5 accepted)")
parser.add_argument("--junit", type=argparse.FileType('wb'), help="output testsuite summary in JUnit format")
parser.add_argument("--test-env", default='local', help="Override default chosen test-env.")
perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")

args = parser.parse_args()

if args.e:
    for e in args.e:
        exec(e)

if args.config_file:
    for arg in args.config_file:
        exec(open(arg).read())

if args.config:
    for arg in args.config:
        field, value = arg.split('=', 1)
        setattr(config, field, value)

all_ways = config.run_ways+config.compile_ways+config.other_ways

if args.rootdir:
    config.rootdirs = args.rootdir

config.metrics_file = args.metrics_file
hasMetricsFile = bool(config.metrics_file)
config.summary_file = args.summary_file
config.no_print_summary = args.no_print_summary

if args.only:
    config.only = args.only
    config.run_only_some_tests = True

if args.way:
    for way in args.way:
        if way not in all_ways:
            print('WARNING: Unknown WAY %s in --way' % way)
        else:
            config.cmdline_ways += [way]
            if way in config.other_ways:
                config.run_ways += [way]
                config.compile_ways += [way]

if args.skipway:
    for way in args.skipway:
        if way not in all_ways:
            print('WARNING: Unknown WAY %s in --skipway' % way)

    config.other_ways = [w for w in config.other_ways if w not in args.skipway]
    config.run_ways = [w for w in config.run_ways if w not in args.skipway]
    config.compile_ways = [w for w in config.compile_ways if w not in args.skipway]

if args.threads:
    config.threads = args.threads
    config.use_threads = True

if args.verbose is not None:
    config.verbose = args.verbose

# Note force skip perf tests: skip if this is not a git repo (estimated with inside_git_repo)
# and no metrics file is given. In this case there is no way to read the previous commit's
# perf test results, nor a way to store new perf test results.
forceSkipPerfTests = not hasMetricsFile and not inside_git_repo()
config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
config.only_perf_tests = args.only_perf_tests

if args.test_env:
    config.test_env = args.test_env

config.cygwin = False
config.msys = False

if windows:
    h = os.popen('uname -s', 'r')
    v = h.read()
    h.close()
    if v.startswith("CYGWIN"):
        config.cygwin = True
    elif v.startswith("MINGW") or v.startswith("MSYS"):
# msys gives "MINGW32"
# msys2 gives "MINGW_NT-6.2" or "MSYS_NT-6.3"
        config.msys = True
    else:
        raise Exception("Can't detect Windows terminal type")

# Try to use UTF8
if windows:
    import ctypes
    # Windows and mingw* Python provide windll, msys2 python provides cdll.
    if hasattr(ctypes, 'WinDLL'):
        mydll = ctypes.WinDLL
    else:
        mydll = ctypes.CDLL

    # This actually leaves the terminal in codepage 65001 (UTF8) even
    # after python terminates. We ought really remember the old codepage
    # and set it back.
    kernel32 = mydll('kernel32.dll')
    if kernel32.SetConsoleCP(65001) == 0:
        raise Exception("Failure calling SetConsoleCP(65001)")
    if kernel32.SetConsoleOutputCP(65001) == 0:
        raise Exception("Failure calling SetConsoleOutputCP(65001)")

    # register the interrupt handler
    signal.signal(signal.SIGINT, signal_handler)
else:
    # Try and find a utf8 locale to use
    # First see if we already have a UTF8 locale
    h = os.popen('locale | grep LC_CTYPE | grep -i utf', 'r')
    v = h.read()
    h.close()
    if v == '':
        # We don't, so now see if 'locale -a' works
        h = os.popen('locale -a | grep -F .', 'r')
        v = h.read()
        h.close()
        if v != '':
            # If it does then use the first utf8 locale that is available
            h = os.popen('locale -a | grep -i "utf8\|utf-8" 2>/dev/null', 'r')
            v = h.readline().strip()
            h.close()
            if v != '':
                os.environ['LC_ALL'] = v
                ghc_env['LC_ALL'] = v
                print("setting LC_ALL to", v)
            else:
                print('WARNING: No UTF8 locale found.')
                print('You may get some spurious test failures.')

# This has to come after arg parsing as the args can change the compiler
get_compiler_info()

# Can't import this earlier as we need to know if threading will be
# enabled or not
from testlib import *

def format_path(path):
    if windows:
        if os.pathsep == ':':
            # If using msys2 python instead of mingw we have to change the drive
            # letter representation. Otherwise it thinks we're adding two env
            # variables E and /Foo when we add E:/Foo.
            path = re.sub('([a-zA-Z]):', '/\\1', path)
        if config.cygwin:
            # On cygwin we can't put "c:\foo" in $PATH, as : is a
            # field separator. So convert to /cygdrive/c/foo instead.
            # Other pythons use ; as the separator, so no problem.
            path = re.sub('([a-zA-Z]):', '/cygdrive/\\1', path)
            path = re.sub('\\\\', '/', path)
    return path

# On Windows we need to set $PATH to include the paths to all the DLLs
# in order for the dynamic library tests to work.
if windows or darwin:
    pkginfo = getStdout([config.ghc_pkg, 'dump'])
    topdir = config.libdir
    if windows:
        mingw = os.path.abspath(os.path.join(topdir, '../mingw/bin'))
        mingw = format_path(mingw)
        ghc_env['PATH'] = os.pathsep.join([ghc_env.get("PATH", ""), mingw])
    for line in pkginfo.split('\n'):
        if line.startswith('library-dirs:'):
            path = line.rstrip()
            path = re.sub('^library-dirs: ', '', path)
            # Use string.replace instead of re.sub, because re.sub
            # interprets backslashes in the replacement string as
            # escape sequences.
            path = path.replace('$topdir', topdir)
            if path.startswith('"'):
                path = re.sub('^"(.*)"$', '\\1', path)
                path = re.sub('\\\\(.)', '\\1', path)
            if windows:
                path = format_path(path)
                ghc_env['PATH'] = os.pathsep.join([path, ghc_env.get("PATH", "")])
            else:
                # darwin
                ghc_env['DYLD_LIBRARY_PATH'] = os.pathsep.join([path, ghc_env.get("DYLD_LIBRARY_PATH", "")])

testopts_local.x = TestOptions()

# if timeout == -1 then we try to calculate a sensible value
if config.timeout == -1:
    config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))

print('Timeout is ' + str(config.timeout))
print('Known ways: ' + ', '.join(config.other_ways))
print('Run ways: ' + ', '.join(config.run_ways))
print('Compile ways: ' + ', '.join(config.compile_ways))

# Try get allowed performance changes from the git commit.
try:
    config.allowed_perf_changes = Perf.get_allowed_perf_changes()
except subprocess.CalledProcessError:
    print('Failed to get allowed metric changes from the HEAD git commit message.')

print('Allowing performance changes in: ' + ', '.join(config.allowed_perf_changes.keys()))

# -----------------------------------------------------------------------------
# The main dude

if config.rootdirs == []:
    config.rootdirs = ['.']

t_files = list(findTFiles(config.rootdirs))

print('Found', len(t_files), '.T files...')

t = getTestRun()

# Avoid cmd.exe built-in 'date' command on Windows
t.start_time = time.localtime()

print('Beginning test run at', time.strftime("%c %Z",t.start_time))

# For reference
try:
    print('Detected CPU features: ', cpu_features.get_cpu_features())
except Exception as e:
    print('Failed to detect CPU features: ', e)

sys.stdout.flush()
# we output text, which cannot be unbuffered
sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w")

if config.local:
    tempdir = ''
else:
    # See note [Running tests in /tmp]
    tempdir = tempfile.mkdtemp('', 'ghctest-')

    # opts.testdir should be quoted when used, to make sure the testsuite
    # keeps working when it contains backward slashes, for example from
    # using os.path.join. Windows native and mingw* python
    # (/mingw64/bin/python) set `os.path.sep = '\\'`, while msys2 python
    # (/bin/python, /usr/bin/python or /usr/local/bin/python) sets
    # `os.path.sep = '/'`.
    # To catch usage of unquoted opts.testdir early, insert some spaces into
    # tempdir.
    tempdir = os.path.join(tempdir, 'test   spaces')

def cleanup_and_exit(exitcode):
    if config.cleanup and tempdir:
        shutil.rmtree(tempdir, ignore_errors=True)
    exit(exitcode)

# First collect all the tests to be run
t_files_ok = True
for file in t_files:
    if_verbose(2, '====> Scanning %s' % file)
    newTestDir(tempdir, os.path.dirname(file))
    try:
        with io.open(file, encoding='utf8') as f:
            src = f.read()

        exec(src)
    except Exception as e:
        traceback.print_exc()
        framework_fail(file, '', str(e))
        t_files_ok = False

for name in config.only:
    if t_files_ok:
        # See Note [Mutating config.only]
        framework_fail(name, '', 'test not found')
    else:
        # Let user fix .T file errors before reporting on unfound tests.
        # The reason the test can not be found is likely because of those
        # .T file errors.
        pass

if config.list_broken:
    print('')
    print('Broken tests:')
    print(' '.join(map (lambda bdn: '#' + str(bdn[0]) + '(' + bdn[1] + '/' + bdn[2] + ')', brokens)))
    print('')

    if t.framework_failures:
        print('WARNING:', len(t.framework_failures), 'framework failures!')
        print('')
else:
    # completion watcher
    watcher = Watcher(len(parallelTests))

    # Now run all the tests
    try:
        for oneTest in parallelTests:
            if stopping():
                break
            oneTest(watcher)

        # wait for parallel tests to finish
        if not stopping():
            watcher.wait()

        # Run the following tests purely sequential
        config.use_threads = False
        for oneTest in aloneTests:
            if stopping():
                break
            oneTest(watcher)
    except KeyboardInterrupt:
        pass

    # flush everything before we continue
    sys.stdout.flush()

    # Warn if had to force skip perf tests (see Note force skip perf tests).
    spacing = "       "
    if forceSkipPerfTests and not args.skip_perf_tests:
        print()
        print(str_warn('Skipping All Performance Tests') + ' `git` exited with non-zero exit code.')
        print(spacing + 'Git is required because performance test results are compared with ancestor git commits\' results (stored with git notes).')
        print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')

    # Warn of new metrics.
    new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
    if any(new_metrics):
        if inside_git_repo():
            reason = 'a baseline (expected value) cannot be recovered from' + \
                ' previous git commits. This may be due to HEAD having' + \
                ' new tests or having expected changes, the presence of' + \
                ' expected changes since the last run of the tests, and/or' + \
                ' the latest test run being too old.'
            fix = 'If the tests exist on the previous' + \
                ' commit (And are configured to run with the same ways),' + \
                ' then check out that commit and run the tests to generate' + \
                ' the missing metrics. Alternatively, a baseline may be' + \
                ' recovered from ci results once fetched:\n\n' + \
                spacing + 'git fetch ' + \
                  'https://gitlab.haskell.org/ghc/ghc-performance-notes.git' + \
                  ' refs/notes/perf:refs/notes/' + Perf.CiNamespace
        else:
            reason = "this is not a git repo so the previous git commit's" + \
                     " metrics cannot be loaded from git notes:"
            fix = ""
        print()
        print(str_warn('Missing Baseline Metrics') + \
                ' these metrics trivially pass because ' + reason)
        print(spacing + (' ').join(set([metric.test for metric in new_metrics])))
        if fix != "":
            print()
            print(fix)

    # Inform of how to accept metric changes.
    if (len(t.unexpected_stat_failures) > 0):
        print()
        print(str_info("Some stats have changed") + " If this is expected, " + \
            "allow changes by appending the git commit message with this:")
        print('-' * 25)
        print(Perf.allow_changes_string(t.metrics))
        print('-' * 25)

    summary(t, sys.stdout, config.no_print_summary, True)

    # Write perf stats if any exist or if a metrics file is specified.
    stats = [stat for (_, stat) in t.metrics]
    if hasMetricsFile:
        print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
        with open(config.metrics_file, 'a') as file:
            file.write("\n" + Perf.format_perf_stat(stats))
    elif inside_git_repo() and any(stats):
        if is_worktree_dirty():
            print()
            print(str_warn('Performance Metrics NOT Saved') + \
                ' working tree is dirty. Commit changes or use ' + \
                '--metrics-file to save metrics to a file.')
        else:
            Perf.append_perf_stat(stats)

    # Write summary
    if config.summary_file:
        with open(config.summary_file, 'w') as file:
            summary(t, file)

    if args.junit:
        junit(t).write(args.junit)

if len(t.unexpected_failures) > 0 or \
   len(t.unexpected_stat_failures) > 0 or \
   len(t.unexpected_passes) > 0 or \
   len(t.framework_failures) > 0:
    exitcode = 1
else:
    exitcode = 0

cleanup_and_exit(exitcode)

# Note [Running tests in /tmp]
#
# Use LOCAL=0 to run tests in /tmp, to catch tests that use files from
# the source directory without copying them to the test directory first.
#
# As an example, take a run_command test with a Makefile containing
# `$(TEST_HC) ../Foo.hs`. GHC will now create the output files Foo.o and
# Foo.hi in the source directory. There are 2 problems with this:
# * Output files in the source directory won't get cleaned up automatically.
# * Two tests might (over)write the same output file.
#
# Tests that only fail when run concurrently with other tests are the
# worst, so we try to catch them early by enabling LOCAL=0 in validate.
#
# Adding -outputdir='.' to TEST_HC_OPTS would help a bit, but it requires
# making changes to quite a few tests. The problem is that
# `$(TEST_HC) ../Foo.hs -outputdir=.` with Foo.hs containing
# `module Main where` does not produce Foo.o, as it would without
# -outputdir, but Main.o. See [1].
#
# Using -outputdir='.' is not foolproof anyway, since it does not change
# the destination of the final executable (Foo.exe).
#
# Another hardening method that could be tried is to `chmod -w` the
# source directory.
#
# By default we set LOCAL=1, because it makes it easier to inspect the
# test directory while working on a new test.
#
# [1]
# https://downloads.haskell.org/~ghc/8.0.1/docs/html/users_guide/separate_compilation.html#output-files