# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import errno import os import shutil import subprocess import sys HERE = os.path.abspath(os.path.dirname(__file__)) ROOT = os.path.normpath(os.path.join(HERE, "..", "..")) CREATE_VIRTUALENV = os.path.join(ROOT, "testing", "create-virtualenv") SITECUSTOMIZE = b""" import os if os.environ.get('CODE_COVERAGE', False): import uuid import coverage covpath = os.path.join(os.environ['COVERAGE_DIR'], 'data', 'coverage.%s' % uuid.uuid1()) cov = coverage.Coverage(data_file=covpath, auto_data=True, branch=True) cov._warn_no_data = False cov._warn_unimported_source = False cov.start() """ def create_virtualenv(name=None, python="python"): path = os.path.join(ROOT, "venv") env = dict(os.environ) env["PYTHON_VERSION"] = python if name: path = os.path.join(path, name) try: os.makedirs(os.path.dirname(path)) except OSError as e: if e.errno != errno.EEXIST: raise if os.name == "nt": bin_dir = os.path.join(path, "Scripts") pip = os.path.join(bin_dir, "pip.exe") python = os.path.join(bin_dir, python + ".exe") activate = os.path.join(bin_dir, "activate") else: bin_dir = os.path.join(path, "bin") pip = os.path.join(bin_dir, "pip") python = os.path.join(bin_dir, python) activate = os.path.join(bin_dir, "activate") res = { "path": path, "bin_dir": bin_dir, "pip": pip, "python": python, "activate": activate, "activate_this": os.path.join(bin_dir, "activate_this.py"), } env["ROOT"] = ROOT env["VENV"] = path if not os.path.exists(res["pip"]): subprocess.check_call([CREATE_VIRTUALENV, path], env=env) # Install a sitecustomize.py that starts code coverage if an environment # variable is set. with open(os.path.join(bin_dir, "sitecustomize.py"), "wb") as fh: fh.write(SITECUSTOMIZE) return res def activate_virtualenv(venv): """Activate a virtualenv in the current Python process.""" with open(venv["activate_this"]) as f: exec(f.read(), dict(__file__=venv["activate_this"])) def process_pip_requirements(venv, requirements): args = [ venv["pip"], "install", "--upgrade", "--require-hashes", "-r", os.path.join(ROOT, requirements), ] subprocess.check_call(args) def install_editable(venv, relpath, extra_env=None): args = [ venv["pip"], "install", "--no-deps", "--editable", os.path.join(ROOT, relpath), ] env = dict(os.environ) env.update(extra_env or {}) subprocess.check_call(args, env=env) def install_mercurials(venv, hg="hg"): """Install supported Mercurial versions in a central location.""" VERSIONS = [ "6.1.4", "6.2.3", "6.3.2", "6.4.3", "6.5.2", "6.6.3", "6.7.4", "6.8.2", "6.9.5", ] hg_dir = os.path.join("/app", "venv", "hg") mercurials = os.path.join(venv["path"], "mercurials") # Setting HGRCPATH to an empty value stops the global and user hgrc from # being loaded. These could interfere with behavior we expect from # vanilla Mercurial. hg_env = dict(os.environ) hg_env["HGRCPATH"] = "" # Ensure a Mercurial clone is present and up to date. if not os.path.isdir(hg_dir): print("cloning Mercurial repository to %s" % hg_dir) subprocess.check_call( [hg, "clone", "https://repo.mercurial-scm.org/hg", hg_dir], cwd="/", env=hg_env, ) subprocess.check_call([hg, "pull"], cwd=hg_dir, env=hg_env) try: os.makedirs(mercurials) except OSError as e: if e.errno != errno.EEXIST: raise # Remove old versions. for p in os.listdir(mercurials): if p in (".", ".."): continue if p not in VERSIONS: print("removing old, unsupported Mercurial version: %s" % p) shutil.rmtree(os.path.join(mercurials, p)) for v in VERSIONS: dest = os.path.join(mercurials, v) if os.path.exists(dest): continue print("installing Mercurial %s to %s" % (v, dest)) try: subprocess.check_output( [hg, "update", v], cwd=hg_dir, env=hg_env, stderr=subprocess.STDOUT ) # We don't care about support files, which only slow down # installation. So install-bin is a suitable target. subprocess.check_output( [ "make", "install-bin", "PREFIX=%s" % dest, "PYTHON=%s" % venv["python"], ], cwd=hg_dir, env=hg_env, stderr=subprocess.STDOUT, ) subprocess.check_output( [hg, "--config", "extensions.purge=", "purge", "--all"], cwd=hg_dir, env=hg_env, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: print("error installing: %s" % e.output) raise Exception("could not install Mercurial") def docker_client(): """Attempt to obtain a Docker client. Returns a client on success. None on failure. """ from .docker import ( Docker, params_from_env, ) docker_url, tls = params_from_env(os.environ) d = Docker(docker_url, tls=tls) return d if d.is_alive() else None def create_docs(): """Create environment used for building docs.""" venv = create_virtualenv("docs") process_pip_requirements(venv, "docs-requirements.txt") install_editable(venv, "hghooks") install_editable(venv, "pylib/Bugsy") install_editable(venv, "pylib/mozhg") install_editable(venv, "pylib/mozhginfo") install_editable(venv, "pylib/mozautomation") install_editable(venv, "testing") return venv def create_hgdev(): """Create an environment used for hacking on Mercurial extensions.""" venv = create_virtualenv("hgdev") reqs = "testing/requirements-hgdev.txt" process_pip_requirements(venv, reqs) install_editable(venv, "hghooks") install_editable(venv, "pylib/Bugsy") install_editable(venv, "pylib/mozhg") install_editable(venv, "pylib/mozhginfo") install_editable(venv, "pylib/mozautomation") install_editable(venv, "testing") install_mercurials(venv, hg=os.path.join(venv["bin_dir"], "hg")) return venv def install_cinnabar(dest=None): """Install git-cinnabar""" if not dest: dest = os.path.join(ROOT, "venv", "git-cinnabar") if not os.path.exists(dest): subprocess.check_call( [ "git", "clone", "--branch", "release", "https://github.com/glandium/git-cinnabar.git", dest, ] ) subprocess.check_call(["git", "pull"], cwd=dest) subprocess.check_call( [ "make", "-j4", "helper", "NO_OPENSSL=1", "NO_GETTEXT=1", ], cwd=dest, ) def create_global(): """Create the global test environment virtualenv This functions the same as ./create-test-environment """ from .hgmo import ( HgCluster, ) # No `name` parameter since this will be the top-level venv venv_py3 = create_virtualenv(name="py3", python="python3") # Install third-party dependencies process_pip_requirements(venv_py3, "test-requirements-3.txt") # Install editable packages editables = { "hgserver/hgmolib", "pylib/Bugsy", "pylib/mozansible", "pylib/mozhg", "pylib/mozhginfo", "pylib/mozautomation", "pylib/vcsreplicator", "hghooks", "testing", } for package in editables: install_editable(venv_py3, package) install_mercurials(venv_py3) cinnabar_dest = os.path.join(venv_py3["path"], "git-cinnabar") install_cinnabar(dest=cinnabar_dest) if os.getenv("NO_DOCKER"): print("Not building Docker images because NO_DOCKER is set") else: print("Building Docker images.") print("This could take a while and may consume a lot of internet bandwidth.") print( "If you don't want Docker images, it is safe to hit CTRL+c to abort this." ) try: HgCluster.build() except subprocess.CalledProcessError: print("You will not be able to run tests that require Docker.") print( "Please see https://docs.docker.com/installation/ for how to install Docker." ) print("When Docker is installed, re-run this script") print( "To avoid re-building the Docker images next time, set the `NO_DOCKER` " "environment variable to any value." ) sys.exit(1) return venv_py3 if __name__ == "__main__": import sys if sys.argv[1] != "install-mercurials": sys.exit(1) venv = { "path": os.path.join("/app", "venv"), "python": os.path.join("/app", "venv", "bin", "python"), } install_mercurials(venv, hg="hg") sys.exit(0)