automation/tests.py

#!/usr/bin/env python3
"""
Run application-services tests

This script provides an interface to run our test/linting/formating scripts

The first argument specifies the operating mode.  This is either a particular
type of test (rust-tests, swift-lint, etc.), or it's "changes" which is for devs
to test their changes.


Changes Mode:
  - Runs tests/clippy against the changes in your branch as determined by:
    - `git merge-base` to find the last commit from the main branch
    - `git diff` to find changed files from that commit
  - Prioritizes executing the tests in a reasonable amount of time.
  - Runs linting and formatting (clippy, rustfmt, etc).
  - Lets rustfmt fix any issues it finds.  It's recommended to
    add or commit code before running the tests so that you can inspect any
    changes with git diff.

Other Modes:
    - rust-tests
    - rust-min-version-tests
    - rust-clippy
    - rust-fmt
    - ktlint
    - swiftlint
    - swiftformat
    - nss-bindings
    - gradle
    - ios-tests
    - python-tests
"""

import argparse
import json
import os
import platform
import shlex
import subprocess
import sys
import traceback
from enum import Enum
from pathlib import Path

PROJECT_ROOT = Path(__file__).parent.parent
AUTOMATION_DIR = PROJECT_ROOT / "automation"
COMPONENTS_DIR = PROJECT_ROOT / "components"
GRADLE = PROJECT_ROOT / "gradlew"
# Ensure this is a proper path, so we can execute it without searching $PATH.
GRADLE = GRADLE.resolve()


IGNORE_PATHS = set(
    [
        # let's not run tests just for dependency changes
        "megazords/full/DEPENDENCIES.md",
        "megazords/full/android/dependency-licenses.xml",
        "megazords/ios-rust/DEPENDENCIES.md",
    ]
)


def blue_text(text):
    if not sys.stdout.isatty():
        return text
    return f"\033[96m{text}\033[0m"


def yellow_text(text):
    if not sys.stdout.isatty():
        return text
    return f"\033[93m{text}\033[0m"


def get_output(cmdline, **kwargs):
    output = subprocess.check_output(cmdline, **kwargs).decode("utf8")
    return output


def run_command(cmdline, **kwargs):
    print(yellow_text(" ".join(shlex.quote(str(part)) for part in cmdline)))
    subprocess.check_call(cmdline, **kwargs)


def path_is_relative_to(path, other):
    """
    Implementation of Path.is_relative_to() which was only added in python 3.9
    """
    return str(path.resolve()).startswith(str(other.resolve()))


def parse_args():
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument("mode", help="Testing mode", metavar="MODE")
    parser.add_argument(
        "--base-branch", dest="base_branch", default="main", help="git base branch"
    )
    return parser.parse_args()


def on_darwin():
    return platform.system() == "Darwin"


def get_default_target():
    return get_output(
        [PROJECT_ROOT / "taskcluster" / "scripts" / "detect-target.sh", "--use-rustc"]
    )


def should_run_rust_tests(package, min_version):
    # There are no tests in examples/ packages, so don't waste time on them.
    if "examples" in package.manifest_path.parts:
        return False
    # Skip the error support trybuild tests when running the min version tests, since we keep the
    # trybuild output files pinned to the latest rust version.
    if min_version and package.name == "error-support-tests":
        return False
    return True


class BranchChanges:
    """
    Tracks which files have been changed in this branch
    """

    def __init__(self, base_branch):
        # Calculate the merge base, this is the last commit from the base
        # branch that's present in this branch.
        self.merge_base = get_output(
            [
                "git",
                "merge-base",
                "HEAD",
                base_branch,
            ]
        ).strip()
        # Use the merge base to calculate which files have changed from the
        # base branch.
        raw_paths = get_output(
            [
                "git",
                "diff",
                "--name-only",
                self.merge_base,
            ]
        ).split("\n")
        raw_paths = [p for p in raw_paths if p not in IGNORE_PATHS]
        self.paths = [PROJECT_ROOT.joinpath(p) for p in raw_paths]

    @staticmethod
    def has_unstanged_changes():
        output = get_output(["git", "status", "--porcelain=v1"])
        return any(line and line[1] == "M" for line in output.split("\n"))


class RustPackage:
    """
    A rust package that we want to test and run clippy on
    """

    def __init__(self, cargo_metadata):
        self.cargo_metadata = cargo_metadata
        self.name = cargo_metadata["name"]
        # Use manifest path to select the package in cargo.  This works better
        # when using the --no-default-features flag
        self.manifest_path = Path(cargo_metadata["manifest_path"])
        self.directory = self.manifest_path.parent

    def has_default_features(self):
        return bool(self.cargo_metadata.get("features").get("default"))

    def has_features(self):
        return bool(self.cargo_metadata.get("features"))

    def has_changes(self, branch_changes):
        return any(path_is_relative_to(p, self.directory) for p in branch_changes.paths)


class RustFeatures(Enum):
    """
    A set of features to for testing/clippy

    Rust tests and clippy can be affected by which features are enabled.
    Therefore we want to use different combinations of features when running
    tests/clippy.
    """

    DEFAULT = "default features"
    ALL = "all features"
    NONE = "no features"

    def label(self):
        return self.value

    def cmdline_args(self):
        if self == RustFeatures.DEFAULT:
            return []
        elif self == RustFeatures.ALL:
            return ["--all-features"]
        elif self == RustFeatures.NONE:
            return ["--no-default-features"]


def calc_rust_items(branch_changes=None, default_features_only=False):
    """
    Calculate which items we want to test and run clippy on

    Args:
        branch_changes: only yield items for rust packages that have changes
        default_features_only: only test with the default features

    Returns: list of (RustPackage, RustFeatures) items
    """
    json_data = json.loads(
        get_output(
            [
                "cargo",
                "metadata",
                "--no-deps",
                "--format-version",
                "1",
            ]
        )
    )

    packages = [RustPackage(p) for p in json_data["packages"]]

    if branch_changes:
        packages = [p for p in packages if p.has_changes(branch_changes)]

    for p in packages:
        yield p, RustFeatures.DEFAULT

    if default_features_only:
        return

    for p in packages:
        if p.has_features():
            yield p, RustFeatures.ALL
    for p in packages:
        if p.has_default_features():
            yield p, RustFeatures.NONE


def calc_non_workspace_rust_items(branch_changes=None, default_features_only=False):
    """
    Calculate which items are not in our default workspace, but we might want to
    do certain things with.

    Returns the same as calc_rust_items
    """
    for path in ["testing/sync-test/Cargo.toml"]:
        json_data = json.loads(
            get_output(
                [
                    "cargo",
                    "metadata",
                    "--no-deps",
                    "--format-version",
                    "1",
                    "--manifest-path",
                    path,
                ]
            )
        )

        packages = [RustPackage(p) for p in json_data["packages"]]

        if branch_changes:
            packages = [p for p in packages if p.has_changes(branch_changes)]

        for p in packages:
            yield p, RustFeatures.DEFAULT

        if default_features_only:
            return

        for p in packages:
            if p.has_features():
                yield p, RustFeatures.ALL
        for p in packages:
            if p.has_default_features():
                yield p, RustFeatures.NONE


# Define a couple functions to avoid this clippy issue:
# https://github.com/rust-lang/rust-clippy/issues/4612h
#
# The safest way to avoid the issue is running cargo clean.  For changes mode
# we use the faster method of touching the changed files so only they get
# rebuilt.


def cargo_clean():
    """
    Force cargo to rebuild rust files
    """
    run_command(["cargo", "clean"])


def touch_changed_paths(branch_changes):
    """
    Quick version of force_rebuild() for change mode

    This version just touches any changed paths, which causes them to be
    rebuilt, but leaves the rest of the files alone.
    """
    for path in branch_changes.paths:
        if path.exists():
            path.touch()


def print_rust_environment():
    print(f"platform: {platform.uname()}")
    print("rustc version: {}".format(get_output(["rustc", "--version"]).strip()))
    print("cargo version: {}".format(get_output(["cargo", "--version"]).strip()))
    print("rustfmt version: {}".format(get_output(["rustfmt", "--version"]).strip()))
    print("GCC version: {}".format(get_output(["gcc", "--version"]).split("\n")[0]))
    print()


def calc_rust_env(package, features):
    if features == RustFeatures.ALL:
        # nss-sys's --features handling is broken.  Workaround it by using a
        # custom --cfg.  This shouldn't be this way!
        return {**os.environ, "RUSTFLAGS": "--cfg __appsvc_ci_hack"}
    else:
        return None


def run_rust_test(package, features):
    run_command(
        [
            "cargo",
            "test",
            "--manifest-path",
            package.manifest_path,
        ]
        + features.cmdline_args(),
        env=calc_rust_env(package, features),
    )


def run_nss_bindings_test():
    run_command(
        [
            "cargo",
            "run",
            "-p",
            "systest",
        ]
    )


def run_clippy(package, features):
    run_command(
        [
            "cargo",
            "clippy",
            "--all-targets",
            "--manifest-path",
            package.manifest_path,
        ]
        + features.cmdline_args()
        + ["--", "-D", "warnings"],
        env=calc_rust_env(package, features),
    )


def run_ktlint():
    run_command([GRADLE, "ktlint", "detekt"])


def run_swiftlint():
    if on_darwin():
        run_command(["swiftlint", "--strict"])
    elif not docker_installed():
        print("WARNING: On non-Darwin hosts, docker is required to run swiftlint")
        print("WARNING: skipping swiftlint on non-Darwin host")
    else:
        cwd = os.getcwd()

        run_command(
            [
                "docker",
                "run",
                "-it",
                "--rm",
                "-v",
                f"{cwd}:{cwd}",
                "-w",
                cwd,
                "ghcr.io/realm/swiftlint:latest",
                "swiftlint",
                "--strict",
            ]
        )


def run_gradle_tests():
    run_command([GRADLE, "test"])


def run_ios_tests():
    if on_darwin():
        run_command([AUTOMATION_DIR / "run_ios_tests.sh"])
    else:
        print("WARNING: skipping iOS tests on non-Darwin host")


def run_python_tests():
    target = get_default_target()
    run_command(
        [
            PROJECT_ROOT / "taskcluster/scripts/server-megazord-build.py",
            "cirrus",
            target,
        ]
    )
    run_command(
        [
            PROJECT_ROOT / "taskcluster/scripts/server-megazord-build.py",
            "nimbus-experimenter",
            target,
        ]
    )


def cargo_fmt(package=None, fix_issues=False):
    cmdline = ["cargo", "fmt"]
    if package:
        cmdline.extend(["--manifest-path", package.manifest_path])
    else:
        cmdline.append("--all")
    if not fix_issues:
        cmdline.extend(["--", "--check"])
    run_command(cmdline)


def swift_format():
    swift_format_args = [
        "megazords",
        "--exclude",
        "**/Generated",
        "--exclude",
        "megazords/ios-rust/Sources/MozillaRustComponentsWrapper/Nimbus/Utils",
        "--lint",
        "--swiftversion",
        "5",
    ]
    if on_darwin():
        run_command(["swiftformat", *swift_format_args])
    elif not docker_installed():
        print("WARNING: On non-Darwin hosts, docker is required to run swiftformat")
        print("WARNING: skipping swiftformat on non-Darwin host")
    else:
        cwd = os.getcwd()

        run_command(
            [
                "docker",
                "run",
                "-it",
                "--rm",
                "-v",
                f"{cwd}:{cwd}",
                "-w",
                cwd,
                "ghcr.io/nicklockwood/swiftformat:latest",
                *swift_format_args,
            ]
        )


def check_for_fmt_changes(branch_changes):
    print()
    if branch_changes.has_unstanged_changes():
        print("cargo fmt made changes.  Make sure to check and commit them.")
    else:
        print("All checks passed!")


class Step:
    """
    Represents a single step of the testing process
    """

    def __init__(self, name, func, *args, **kwargs):
        self.name = name
        self.func = func
        self.args = args
        self.kwargs = kwargs

    def run(self):
        print()
        print(blue_text(f"Running {self.name}"))
        try:
            self.func(*self.args, **self.kwargs)
        except subprocess.CalledProcessError:
            exit_with_error(1, f"Error while running {self.name}")
        except Exception:
            exit_with_error(
                2,
                f"Unexpected exception while running {self.name}",
                print_exception=True,
            )


def calc_steps(args):
    """
    Calculate the steps needed to run the tests

    Yields a list of (name, func) items.
    """
    if args.mode == "changes":
        # changes mode is complicated enough that it's split off into its own
        # function
        for step in calc_steps_change_mode(args):
            yield step
    elif args.mode == "rust-tests":
        print_rust_environment()
        yield Step("cargo clean", cargo_clean)
        for package, features in calc_rust_items():
            if should_run_rust_tests(package, False):
                yield Step(
                    f"tests for {package.name} ({features.label()})",
                    run_rust_test,
                    package,
                    features,
                )
    elif args.mode == "rust-min-version-tests":
        print_rust_environment()
        yield Step("cargo clean", cargo_clean)
        for package, features in calc_rust_items():
            if should_run_rust_tests(package, True):
                yield Step(
                    f"tests for {package.name} ({features.label()})",
                    run_rust_test,
                    package,
                    features,
                )
    elif args.mode == "rust-clippy":
        print_rust_environment()
        yield Step("cargo clean", cargo_clean)
        for package, features in calc_rust_items():
            yield Step(
                f"clippy for {package.name} ({features.label()})",
                run_clippy,
                package,
                features,
            )
        # non-workspace items aren't tested, but we do run clippy on them to
        # make sure they don't go stale.
        for package, features in calc_non_workspace_rust_items():
            yield Step(
                f"clippy for {package.name} ({features.label()})",
                run_clippy,
                package,
                features,
            )
    elif args.mode == "rust-fmt":
        print_rust_environment()
        yield Step("cargo fmt", cargo_fmt)
    elif args.mode == "ktlint":
        yield Step("ktlint", run_ktlint)
    elif args.mode == "swiftlint":
        yield Step("swiftlint", run_swiftlint)
    elif args.mode == "swiftformat":
        yield Step("swiftformat", swift_format)
    elif args.mode == "nss-bindings":
        print_rust_environment()
        yield Step("NSS bindings test", run_nss_bindings_test)
    elif args.mode == "gradle":
        yield Step("gradle tests", run_gradle_tests)
    elif args.mode == "ios-tests":
        yield Step("ios tests", run_ios_tests)
    elif args.mode == "python-tests":
        yield Step("python tests", run_python_tests)
    else:
        print(f"Invalid mode: {args.mode}")
        sys.exit(1)


def calc_steps_change_mode(args):
    """
    Calculate the steps needed for change mode
    """
    print_rust_environment()
    branch_changes = BranchChanges(args.base_branch)
    rust_items = list(calc_rust_items(branch_changes, default_features_only=True))
    rust_packages = list(set(package for package, _ in rust_items))

    if not rust_items:
        print("no changes found.")
        return

    if branch_changes.has_unstanged_changes():
        subprocess.run(["git", "status"], check=False)
        print()
        print("WARNING: unstaged changes in your branch:")
        print(
            "Consider git add or git commit to stage them since this "
            "script will run cargo fmt"
        )
        print("Continue (Y/N)?")
        if input().lower() != "y":
            sys.exit(0)

    yield Step("touch changed paths", touch_changed_paths, branch_changes)
    for package, features in rust_items:
        yield Step(
            f"tests for {package.name} ({features.label()})",
            run_rust_test,
            package,
            features,
        )
    for package, features in rust_items:
        yield Step(
            f"clippy for {package.name} ({features.label()})",
            run_clippy,
            package,
            features,
        )
    for package in rust_packages:
        yield Step(f"rustfmt for {package.name}", cargo_fmt, package, fix_issues=True)
    yield Step("Check for changes", check_for_fmt_changes, branch_changes)


def main():
    args = parse_args()
    os.chdir(PROJECT_ROOT)
    for step in calc_steps(args):
        step.run()


def exit_with_error(code, text, print_exception=False):
    print()
    print("-" * 78)
    print()
    print(text)
    if print_exception:
        traceback.print_exc()
    sys.exit(code)


def docker_installed():
    result = subprocess.run(
        ["docker"],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
        check=False,
    )

    return result.returncode == 0


if __name__ == "__main__":
    main()