build/fbcode_builder/make_docker_context.py

#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.

"""
Reads `fbcode_builder_config.py` from the current directory, and prepares a
Docker context directory to build this project.  Prints to stdout the path
to the context directory.

Try `.../make_docker_context.py --help` from a project's `build/` directory.

By default, the Docker context directory will be in /tmp. It will always
contain a Dockerfile, and might also contain copies of your local repos, and
other data needed for the build container.
"""

import os
import tempfile
import textwrap

from docker_builder import DockerFBCodeBuilder
from parse_args import parse_args_to_fbcode_builder_opts


def make_docker_context(
    get_steps_fn, github_project, opts=None, default_context_dir=None
):
    """
    Returns a path to the Docker context directory. See parse_args.py.

    Helper for making a command-line utility that writes your project's
    Dockerfile and associated data into a (temporary) directory.  Your main
    program might look something like this:

        print(make_docker_context(
            lambda builder: [builder.step(...), ...],
            'facebook/your_project',
        ))
    """

    if opts is None:
        opts = {}

    valid_versions = (
        ("ubuntu:16.04", "5"),
        ("ubuntu:18.04", "7"),
    )

    def add_args(parser):
        parser.add_argument(
            "--docker-context-dir",
            metavar="DIR",
            default=default_context_dir,
            help="Write the Dockerfile and its context into this directory. "
            "If empty, make a temporary directory. Default: %(default)s.",
        )
        parser.add_argument(
            "--user",
            metavar="NAME",
            default=opts.get("user", "nobody"),
            help="Build and install as this user. Default: %(default)s.",
        )
        parser.add_argument(
            "--prefix",
            metavar="DIR",
            default=opts.get("prefix", "/home/install"),
            help="Install all libraries in this prefix. Default: %(default)s.",
        )
        parser.add_argument(
            "--projects-dir",
            metavar="DIR",
            default=opts.get("projects_dir", "/home"),
            help="Place project code directories here. Default: %(default)s.",
        )
        parser.add_argument(
            "--os-image",
            metavar="IMG",
            choices=zip(*valid_versions)[0],
            default=opts.get("os_image", valid_versions[0][0]),
            help="Docker OS image -- be sure to use only ones you trust (See "
            "README.docker). Choices: %(choices)s. Default: %(default)s.",
        )
        parser.add_argument(
            "--gcc-version",
            metavar="VER",
            choices=set(zip(*valid_versions)[1]),
            default=opts.get("gcc_version", valid_versions[0][1]),
            help="Choices: %(choices)s. Default: %(default)s.",
        )
        parser.add_argument(
            "--make-parallelism",
            metavar="NUM",
            type=int,
            default=opts.get("make_parallelism", 1),
            help="Use `make -j` on multi-CPU systems with lots of RAM. "
            "Default: %(default)s.",
        )
        parser.add_argument(
            "--local-repo-dir",
            metavar="DIR",
            help="If set, build {0} from a local directory instead of Github.".format(
                github_project
            ),
        )
        parser.add_argument(
            "--ccache-tgz",
            metavar="PATH",
            help="If set, enable ccache for the build. To initialize the "
            "cache, first try to hardlink, then to copy --cache-tgz "
            "as ccache.tgz into the --docker-context-dir.",
        )

    opts = parse_args_to_fbcode_builder_opts(
        add_args,
        # These have add_argument() calls, others are set via --option.
        (
            "docker_context_dir",
            "user",
            "prefix",
            "projects_dir",
            "os_image",
            "gcc_version",
            "make_parallelism",
            "local_repo_dir",
            "ccache_tgz",
        ),
        opts,
        help=textwrap.dedent(
            """

        Reads `fbcode_builder_config.py` from the current directory, and
        prepares a Docker context directory to build {github_project} and
        its dependencies.  Prints to stdout the path to the context
        directory.

        Pass --option {github_project}:git_hash SHA1 to build something
        other than the master branch from Github.

        Or, pass --option {github_project}:local_repo_dir LOCAL_PATH to
        build from a local repo instead of cloning from Github.

        Usage:
            (cd $(./make_docker_context.py) && docker build . 2>&1 | tee log)

        """.format(
                github_project=github_project
            )
        ),
    )

    # This allows travis_docker_build.sh not to know the main Github project.
    local_repo_dir = opts.pop("local_repo_dir", None)
    if local_repo_dir is not None:
        opts["{0}:local_repo_dir".format(github_project)] = local_repo_dir

    if (opts.get("os_image"), opts.get("gcc_version")) not in valid_versions:
        raise Exception(
            "Due to 4/5 ABI changes (std::string), we can only use {0}".format(
                " / ".join("GCC {1} on {0}".format(*p) for p in valid_versions)
            )
        )

    if opts.get("docker_context_dir") is None:
        opts["docker_context_dir"] = tempfile.mkdtemp(prefix="docker-context-")
    elif not os.path.exists(opts.get("docker_context_dir")):
        os.makedirs(opts.get("docker_context_dir"))

    builder = DockerFBCodeBuilder(**opts)
    context_dir = builder.option("docker_context_dir")  # Mark option "in-use"
    # The renderer may also populate some files into the context_dir.
    dockerfile = builder.render(get_steps_fn(builder))

    with os.fdopen(
        os.open(
            os.path.join(context_dir, "Dockerfile"),
            os.O_RDWR | os.O_CREAT | os.O_EXCL,  # Do not overwrite existing files
            0o644,
        ),
        "w",
    ) as f:
        f.write(dockerfile)

    return context_dir


if __name__ == "__main__":
    from utils import read_fbcode_builder_config, build_fbcode_builder_config

    # Load a spec from the current directory
    config = read_fbcode_builder_config("fbcode_builder_config.py")
    print(
        make_docker_context(
            build_fbcode_builder_config(config),
            config["github_project"],
        )
    )