build/fbcode_builder/fbcode_builder.py

#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.

"""

This is a small DSL to describe builds of Facebook's open-source projects
that are published to Github from a single internal repo, including projects
that depend on folly, wangle, proxygen, fbthrift, etc.

This file defines the interface of the DSL, and common utilieis, but you
will have to instantiate a specific builder, with specific options, in
order to get work done -- see e.g. make_docker_context.py.

== Design notes ==

Goals:

 - A simple declarative language for what needs to be checked out & built,
   how, in what order.

 - The same specification should work for external continuous integration
   builds (e.g. Travis + Docker) and for internal VM-based continuous
   integration builds.

 - One should be able to build without root, and to install to a prefix.

Non-goals:

 - General usefulness. The only point of this is to make it easier to build
   and test Facebook's open-source services.

Ideas for the future -- these may not be very good :)

 - Especially on Ubuntu 14.04 the current initial setup is inefficient:
   we add PPAs after having installed a bunch of packages -- this prompts
   reinstalls of large amounts of code.  We also `apt-get update` a few
   times.

 - A "shell script" builder. Like DockerFBCodeBuilder, but outputs a
   shell script that runs outside of a container. Or maybe even
   synchronously executes the shell commands, `make`-style.

 - A "Makefile" generator. That might make iterating on builds even quicker
   than what you can currently get with Docker build caching.

 - Generate a rebuild script that can be run e.g. inside the built Docker
   container by tagging certain steps with list-inheriting Python objects:
     * do change directories
     * do NOT `git clone` -- if we want to update code this should be a
       separate script that e.g. runs rebase on top of specific targets
       across all the repos.
     * do NOT install software (most / all setup can be skipped)
     * do NOT `autoreconf` or `configure`
     * do `make` and `cmake`

 - If we get non-Debian OSes, part of ccache setup should be factored out.
"""

import os
import re

from shell_quoting import path_join, shell_join, ShellQuoted


def _read_project_github_hashes():
    base_dir = "deps/github_hashes/"  # trailing slash used in regex below
    for dirname, _, files in os.walk(base_dir):
        for filename in files:
            path = os.path.join(dirname, filename)
            with open(path) as f:
                m_proj = re.match("^" + base_dir + "(.*)-rev\.txt$", path)
                if m_proj is None:
                    raise RuntimeError("Not a hash file? {0}".format(path))
                m_hash = re.match("^Subproject commit ([0-9a-f]+)\n$", f.read())
                if m_hash is None:
                    raise RuntimeError("No hash in {0}".format(path))
                yield m_proj.group(1), m_hash.group(1)


class FBCodeBuilder(object):
    def __init__(self, **kwargs):
        self._options_do_not_access = kwargs  # Use .option() instead.
        # This raises upon detecting options that are specified but unused,
        # because otherwise it is very easy to make a typo in option names.
        self.options_used = set()
        # Mark 'projects_dir' used even if the build installs no github
        # projects.  This is needed because driver programs like
        # `shell_builder.py` unconditionally set this for all builds.
        self._github_dir = self.option("projects_dir")
        self._github_hashes = dict(_read_project_github_hashes())

    def __repr__(self):
        return "{0}({1})".format(
            self.__class__.__name__,
            ", ".join(
                "{0}={1}".format(k, repr(v))
                for k, v in self._options_do_not_access.items()
            ),
        )

    def option(self, name, default=None):
        value = self._options_do_not_access.get(name, default)
        if value is None:
            raise RuntimeError("Option {0} is required".format(name))
        self.options_used.add(name)
        return value

    def has_option(self, name):
        return name in self._options_do_not_access

    def add_option(self, name, value):
        if name in self._options_do_not_access:
            raise RuntimeError("Option {0} already set".format(name))
        self._options_do_not_access[name] = value

    #
    # Abstract parts common to every installation flow
    #

    def render(self, steps):
        """

        Converts nested actions to your builder's expected output format.
        Typically takes the output of build().

        """
        res = self._render_impl(steps)  # Implementation-dependent
        # Now that the output is rendered, we expect all options to have
        # been used.
        unused_options = set(self._options_do_not_access)
        unused_options -= self.options_used
        if unused_options:
            raise RuntimeError(
                "Unused options: {0} -- please check if you made a typo "
                "in any of them. Those that are truly not useful should "
                "be not be set so that this typo detection can be useful.".format(
                    unused_options
                )
            )
        return res

    def build(self, steps):
        if not steps:
            raise RuntimeError(
                "Please ensure that the config you are passing " "contains steps"
            )
        return [self.setup(), self.diagnostics()] + steps

    def setup(self):
        "Your builder may want to install packages here."
        raise NotImplementedError

    def diagnostics(self):
        "Log some system diagnostics before/after setup for ease of debugging"
        # The builder's repr is not used in a command to avoid pointlessly
        # invalidating Docker's build cache.
        return self.step(
            "Diagnostics",
            [
                self.comment("Builder {0}".format(repr(self))),
                self.run(ShellQuoted("hostname")),
                self.run(ShellQuoted("cat /etc/issue || echo no /etc/issue")),
                self.run(ShellQuoted("g++ --version || echo g++ not installed")),
                self.run(ShellQuoted("cmake --version || echo cmake not installed")),
            ],
        )

    def step(self, name, actions):
        "A labeled collection of actions or other steps"
        raise NotImplementedError

    def run(self, shell_cmd):
        "Run this bash command"
        raise NotImplementedError

    def set_env(self, key, value):
        'Set the environment "key" to value "value"'
        raise NotImplementedError

    def workdir(self, dir):
        "Create this directory if it does not exist, and change into it"
        raise NotImplementedError

    def copy_local_repo(self, dir, dest_name):
        """
        Copy the local repo at `dir` into this step's `workdir()`, analog of:
          cp -r /path/to/folly folly
        """
        raise NotImplementedError

    def python_deps(self):
        return [
            "wheel",
            "cython==0.28.6",
        ]

    def debian_deps(self):
        return [
            "autoconf-archive",
            "bison",
            "build-essential",
            "cmake",
            "curl",
            "flex",
            "git",
            "gperf",
            "joe",
            "libboost-all-dev",
            "libcap-dev",
            "libdouble-conversion-dev",
            "libevent-dev",
            "libgflags-dev",
            "libgoogle-glog-dev",
            "libkrb5-dev",
            "libpcre3-dev",
            "libpthread-stubs0-dev",
            "libnuma-dev",
            "libsasl2-dev",
            "libsnappy-dev",
            "libsqlite3-dev",
            "libssl-dev",
            "libtool",
            "netcat-openbsd",
            "pkg-config",
            "sudo",
            "unzip",
            "wget",
            "python3-venv",
        ]

    #
    # Specific build helpers
    #

    def install_debian_deps(self):
        actions = [
            self.run(
                ShellQuoted("apt-get update && apt-get install -yq {deps}").format(
                    deps=shell_join(
                        " ", (ShellQuoted(dep) for dep in self.debian_deps())
                    )
                )
            ),
        ]
        gcc_version = self.option("gcc_version")

        # Make the selected GCC the default before building anything
        actions.extend(
            [
                self.run(
                    ShellQuoted("apt-get install -yq {c} {cpp}").format(
                        c=ShellQuoted("gcc-{v}").format(v=gcc_version),
                        cpp=ShellQuoted("g++-{v}").format(v=gcc_version),
                    )
                ),
                self.run(
                    ShellQuoted(
                        "update-alternatives --install /usr/bin/gcc gcc {c} 40 "
                        "--slave /usr/bin/g++ g++ {cpp}"
                    ).format(
                        c=ShellQuoted("/usr/bin/gcc-{v}").format(v=gcc_version),
                        cpp=ShellQuoted("/usr/bin/g++-{v}").format(v=gcc_version),
                    )
                ),
                self.run(ShellQuoted("update-alternatives --config gcc")),
            ]
        )

        actions.extend(self.debian_ccache_setup_steps())

        return self.step("Install packages for Debian-based OS", actions)

    def create_python_venv(self):
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions.append(
                self.run(
                    ShellQuoted("python3 -m venv {p}").format(
                        p=path_join(self.option("prefix"), "venv")
                    )
                )
            )
        return actions

    def python_venv(self):
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions.append(
                ShellQuoted("source {p}").format(
                    p=path_join(self.option("prefix"), "venv", "bin", "activate")
                )
            )

            actions.append(
                self.run(
                    ShellQuoted("python3 -m pip install {deps}").format(
                        deps=shell_join(
                            " ", (ShellQuoted(dep) for dep in self.python_deps())
                        )
                    )
                )
            )
        return actions

    def enable_rust_toolchain(self, toolchain="stable", is_bootstrap=True):
        choices = set(["stable", "beta", "nightly"])

        assert toolchain in choices, (
            "while enabling rust toolchain: {} is not in {}"
        ).format(toolchain, choices)

        rust_toolchain_opt = (toolchain, is_bootstrap)
        prev_opt = self.option("rust_toolchain", rust_toolchain_opt)
        assert prev_opt == rust_toolchain_opt, (
            "while enabling rust toolchain: previous toolchain already set to"
            " {}, but trying to set it to {} now"
        ).format(prev_opt, rust_toolchain_opt)

        self.add_option("rust_toolchain", rust_toolchain_opt)

    def rust_toolchain(self):
        actions = []
        if self.option("rust_toolchain", False):
            (toolchain, is_bootstrap) = self.option("rust_toolchain")
            rust_dir = path_join(self.option("prefix"), "rust")
            actions = [
                self.set_env("CARGO_HOME", rust_dir),
                self.set_env("RUSTUP_HOME", rust_dir),
                self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
                self.run(
                    ShellQuoted(
                        "curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
                        " | sh -s --"
                        "   --default-toolchain={r} "
                        "   --profile=minimal"
                        "   --no-modify-path"
                        "   -y"
                    ).format(p=rust_dir, r=toolchain)
                ),
                self.set_env(
                    "PATH",
                    ShellQuoted("{p}:$PATH").format(p=path_join(rust_dir, "bin")),
                ),
                self.run(ShellQuoted("rustup update")),
                self.run(ShellQuoted("rustc --version")),
                self.run(ShellQuoted("rustup --version")),
                self.run(ShellQuoted("cargo --version")),
            ]
        return actions

    def debian_ccache_setup_steps(self):
        return []  # It's ok to ship a renderer without ccache support.

    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option(
            "{0}:git_hash".format(project),
            # Any repo that has a hash in deps/github_hashes defaults to
            # that, with the goal of making builds maximally consistent.
            self._github_hashes.get(project, ""),
        )
        maybe_change_branch = (
            [
                self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)),
            ]
            if git_hash
            else []
        )

        local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
        return self.step(
            "Check out {0}, workdir {1}".format(project, path),
            [
                self.workdir(self._github_dir),
                self.run(
                    ShellQuoted("git clone {opts} https://github.com/{p}").format(
                        p=project,
                        opts=ShellQuoted(
                            self.option("{}:git_clone_opts".format(project), "")
                        ),
                    )
                )
                if not local_repo_dir
                else self.copy_local_repo(local_repo_dir, os.path.basename(project)),
                self.workdir(
                    path_join(self._github_dir, os.path.basename(project), path),
                ),
            ]
            + maybe_change_branch,
        )

    def fb_github_project_workdir(self, project_and_path, github_org="facebook"):
        "This helper lets Facebook-internal CI special-cases FB projects"
        project, path = project_and_path.split("/", 1)
        return self.github_project_workdir(github_org + "/" + project, path)

    def _make_vars(self, make_vars):
        return shell_join(
            " ",
            (
                ShellQuoted("{k}={v}").format(k=k, v=v)
                for k, v in ({} if make_vars is None else make_vars).items()
            ),
        )

    def parallel_make(self, make_vars=None):
        return self.run(
            ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
                n=self.option("make_parallelism"),
                vars=self._make_vars(make_vars),
            )
        )

    def make_and_install(self, make_vars=None):
        return [
            self.parallel_make(make_vars),
            self.run(
                ShellQuoted("make install VERBOSE=1 {vars}").format(
                    vars=self._make_vars(make_vars),
                )
            ),
        ]

    def configure(self, name=None):
        autoconf_options = {}
        if name is not None:
            autoconf_options.update(
                self.option("{0}:autoconf_options".format(name), {})
            )
        return [
            self.run(
                ShellQuoted(
                    'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
                    'CFLAGS="$CFLAGS -I"{p}"/include" '
                    'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
                    "PY_PREFIX={p} "
                    "./configure --prefix={p} {args}"
                ).format(
                    p=self.option("prefix"),
                    args=shell_join(
                        " ",
                        (
                            ShellQuoted("{k}={v}").format(k=k, v=v)
                            for k, v in autoconf_options.items()
                        ),
                    ),
                )
            ),
        ]

    def autoconf_install(self, name):
        return self.step(
            "Build and install {0}".format(name),
            [
                self.run(ShellQuoted("autoreconf -ivf")),
            ]
            + self.configure()
            + self.make_and_install(),
        )

    def cmake_configure(self, name, cmake_path=".."):
        cmake_defines = {
            "BUILD_SHARED_LIBS": "ON",
            "CMAKE_INSTALL_PREFIX": self.option("prefix"),
        }

        # Hacks to add thriftpy3 support
        if "BUILD_THRIFT_PY3" in os.environ and "folly" in name:
            cmake_defines["PYTHON_EXTENSIONS"] = "True"

        if "BUILD_THRIFT_PY3" in os.environ and "fbthrift" in name:
            cmake_defines["thriftpy3"] = "ON"

        cmake_defines.update(self.option("{0}:cmake_defines".format(name), {}))
        return [
            self.run(
                ShellQuoted(
                    'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
                    'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
                    "cmake {args} {cmake_path}"
                ).format(
                    p=self.option("prefix"),
                    args=shell_join(
                        " ",
                        (
                            ShellQuoted("-D{k}={v}").format(k=k, v=v)
                            for k, v in cmake_defines.items()
                        ),
                    ),
                    cmake_path=cmake_path,
                )
            ),
        ]

    def cmake_install(self, name, cmake_path=".."):
        return self.step(
            "Build and install {0}".format(name),
            self.cmake_configure(name, cmake_path) + self.make_and_install(),
        )

    def cargo_build(self, name):
        return self.step(
            "Build {0}".format(name),
            [
                self.run(
                    ShellQuoted("cargo build -j {n}").format(
                        n=self.option("make_parallelism")
                    )
                )
            ],
        )

    def fb_github_autoconf_install(self, project_and_path, github_org="facebook"):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.autoconf_install(project_and_path),
        ]

    def fb_github_cmake_install(
        self, project_and_path, cmake_path="..", github_org="facebook"
    ):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.cmake_install(project_and_path, cmake_path),
        ]

    def fb_github_cargo_build(self, project_and_path, github_org="facebook"):
        return [
            self.fb_github_project_workdir(project_and_path, github_org),
            self.cargo_build(project_and_path),
        ]