""" Installs packages using PIP """ import logging import re import subprocess from email.parser import FeedParser from typing import Tuple from aws_lambda_builders.architecture import ARM64, X86_64 from aws_lambda_builders.utils import extract_tarfile from .compat import pip_import_string, pip_no_compile_c_env_vars, pip_no_compile_c_shim from .utils import OSUtils LOG = logging.getLogger(__name__) # TODO update the wording here MISSING_DEPENDENCIES_TEMPLATE = r""" Could not install dependencies: %s You will have to build these yourself and vendor them in the vendor folder. """ class PackagerError(Exception): pass class InvalidSourceDistributionNameError(PackagerError): pass class RequirementsFileNotFoundError(PackagerError): """ Exceptions is no longer raised. Keeping it here because this exception is 'public' and could still be used by a customer. """ def __init__(self, requirements_path): super(RequirementsFileNotFoundError, self).__init__("Requirements file not found: %s" % requirements_path) class MissingDependencyError(PackagerError): """Raised when some dependencies could not be packaged for any reason.""" def __init__(self, missing): self.missing = missing class NoSuchPackageError(PackagerError): """Raised when a package name or version could not be found.""" def __init__(self, package_name): super(NoSuchPackageError, self).__init__("Could not satisfy the requirement: %s" % package_name) class PackageDownloadError(PackagerError): """Generic networking error during a package download.""" pass class UnsupportedPackageError(Exception): """Unable to parse package metadata.""" def __init__(self, package_name): # type: (str) -> None super(UnsupportedPackageError, self).__init__("Unable to retrieve name/version for package: %s" % package_name) class UnsupportedPythonVersion(PackagerError): """Generic networking error during a package download.""" def __init__(self, version): super(UnsupportedPythonVersion, self).__init__("'%s' version of python is not supported" % version) def get_lambda_abi(runtime): supported = { "python3.8": "cp38", "python3.9": "cp39", "python3.10": "cp310", "python3.11": "cp311", "python3.12": "cp312", "python3.13": "cp313", } if runtime not in supported: raise UnsupportedPythonVersion(runtime) return supported[runtime] class PythonPipDependencyBuilder(object): def __init__(self, runtime, python_exe, osutils=None, dependency_builder=None, architecture=X86_64): """Initialize a PythonPipDependencyBuilder. :type runtime: str :param runtime: Python version to build dependencies for. This can either be python3.8, python3.9, python3.10, python3.11, python3.12 or python3.13. These are currently the only supported values. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. :type dependency_builder: :class:`DependencyBuilder` :param dependency_builder: This class will be used to build the dependencies of the project. :type architecture: str :param description: Architecture used to build dependencies for. This can be either arm64 or x86_64. The default value is x86_64 if it's not provided. """ self.osutils = osutils if osutils is None: self.osutils = OSUtils() if dependency_builder is None: dependency_builder = DependencyBuilder(self.osutils, python_exe, runtime, architecture=architecture) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, requirements_path, ui=None, config=None): """Builds a python project's dependencies into an artifact directory. :type artifacts_dir_path: str :param artifacts_dir_path: Directory to write dependencies into. :type scratch_dir_path: str :param scratch_dir_path: Directory to write temp files into. :type requirements_path: str :param requirements_path: Path to a requirements.txt file to inspect for a list of dependencies. :type ui: :class:`lambda_builders.utils.UI` or None :param ui: A class that traps all progress information such as status and errors. If injected by the caller, it can be used to monitor the status of the build process or forward this information elsewhere. :type config: :class:`lambda_builders.utils.Config` or None :param config: To be determined. This is an optional config object we can extend at a later date to add more options to how pip is called. """ # TODO: The DependencyBuilder makes the assumption that it is running # in a virtual environment that matches the runtime you want to use. # Since otherwise there is no way to force pip to build wheels for the # correct version of python. We need to enforce that assumption here # by finding/creating a virtualenv of the correct version and when # pip is called set the appropriate env vars. self._dependency_builder.build_site_packages(requirements_path, artifacts_dir_path, scratch_dir_path) class DependencyBuilder(object): """Build site-packages by manually downloading and unpacking wheels. Pip is used to download all the dependency sdists. Then wheels that compatible with lambda are downloaded. Any source packages that do not have a matching wheel file are built into a wheel and that file is checked for compatibility with the lambda python runtime environment. All compatible wheels that are downloaded/built this way are unpacked into a site-packages directory, to be included in the bundle by the packager. """ _COMPATIBLE_PLATFORM_ARM64 = { "any", "linux_aarch64", "manylinux2014_aarch64", } _COMPATIBLE_PLATFORM_X86_64 = { "any", "linux_x86_64", "manylinux1_x86_64", "manylinux2010_x86_64", "manylinux2014_x86_64", } _COMPATIBLE_PLATFORMS = { ARM64: _COMPATIBLE_PLATFORM_ARM64, X86_64: _COMPATIBLE_PLATFORM_X86_64, } _MANYLINUX_LEGACY_MAP = { "manylinux1_x86_64": "manylinux_2_5_x86_64", "manylinux2010_x86_64": "manylinux_2_12_x86_64", "manylinux2014_aarch64": "manylinux_2_17_aarch64", "manylinux2014_x86_64": "manylinux_2_17_x86_64", } _COMPATIBLE_PACKAGE_ALLOWLIST = {"sqlalchemy"} # Mapping of abi to glibc version in Lambda runtime. _RUNTIME_GLIBC = { "cp37m": (2, 17), "cp38": (2, 26), "cp39": (2, 26), "cp310": (2, 26), "cp311": (2, 26), "cp312": (2, 26), } # Fallback version if we're on an unknown python version # not in _RUNTIME_GLIBC. # Unlikely to hit this case. _DEFAULT_GLIBC = (2, 17) def __init__(self, osutils, runtime, python_exe, pip_runner=None, architecture=X86_64): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. :type runtime: str :param runtime: AWS Lambda Python runtime to build for :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. :type architecture: str :param architecture: Architecture to build for. """ self._osutils = osutils self.python_exe = python_exe if pip_runner is None: pip_runner = PipRunner(python_exe=python_exe, pip=SubprocessPip(osutils)) self._pip = pip_runner self.runtime = runtime self.architecture = architecture def build_site_packages(self, requirements_filepath, target_directory, scratch_directory): """Build site-packages directory for a set of requiremetns. :type requirements_filepath: str :param requirement_filepath: The path to a requirements file to inspect for a list of top-level requirements to install. This should be equivilent to ``pip install -r requirements_filepath.txt`` in theory. :type target_directory: str :param target_directory: The directory to build all dependencies into. This directory should be on the PYTHON_PATH of whichever process wants to use thse dependencies. :type scratch_directory: str :param scratch_directory: The directory to write temp files into. :raises MissingDependencyError: This exception is raised if one or more packages could not be installed. The complete list of missing packages is included in the error object's ``missing`` property. """ if self._has_at_least_one_package(requirements_filepath): wheels, packages_without_wheels = self._download_dependencies(scratch_directory, requirements_filepath) self._install_wheels(scratch_directory, target_directory, wheels) if packages_without_wheels: raise MissingDependencyError(packages_without_wheels) def _has_at_least_one_package(self, filename): if not self._osutils.file_exists(filename): return False with open(filename, "r") as f: # This is meant to be a best effort attempt. # This can return True and still have no packages # actually being specified, but those aren't common # cases. for line in f: stripped_line = line.strip() if stripped_line and not stripped_line.startswith("#"): return True return False def _download_dependencies(self, directory, requirements_filename): # Download all dependencies we can, letting pip choose what to # download. # deps should represent the best effort we can make to gather all the # dependencies. deps = self._download_all_dependencies(requirements_filename, directory) # Sort the downloaded packages into three categories: # - sdists (Pip could not get a wheel so it gave us an sdist) # - lambda compatible wheel files # - lambda incompatible wheel files # Pip will give us a wheel when it can, but some distributions do not # ship with wheels at all in which case we will have an sdist for it. # In some cases a platform specific wheel file may be availble so pip # will have downloaded that, if our platform does not match the # platform that the function will run on (x86_64 or arm64) then the # downloaded wheel file may not be compatible with Lambda. Pure python # wheels still will be compatible because they have no platform # dependencies. compatible_wheels = set() incompatible_wheels = set() sdists = set() for package in deps: if package.dist_type == "sdist": sdists.add(package) elif self._is_compatible_wheel_filename(package.filename): compatible_wheels.add(package) else: incompatible_wheels.add(package) LOG.debug("initial compatible: %s", compatible_wheels) LOG.debug("initial incompatible: %s", incompatible_wheels | sdists) # Next we need to go through the downloaded packages and pick out any # dependencies that do not have a compatible wheel file downloaded. # For these packages we need to explicitly try to download a # compatible wheel file. missing_wheels = sdists | incompatible_wheels self._download_binary_wheels(missing_wheels, directory) # Re-count the wheel files after the second download pass. Anything # that has an sdist but not a valid wheel file is still not going to # work on lambda and we must now try and build the sdist into a wheel # file ourselves. compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) LOG.debug("compatible wheels after second download pass: %s", compatible_wheels) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=True) # There is still the case where the package had optional C dependencies # for speedups. In this case the wheel file will have built above with # the C dependencies if it managed to find a C compiler. If we are on # an incompatible architecture this means the wheel file generated will # not be compatible. If we categorize our files once more and find that # there are missing dependencies we can try our last ditch effort of # building the package and trying to sever its ability to find a C # compiler. compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) LOG.debug("compatible after building wheels (no C compiling): %s", compatible_wheels) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=False) # Final pass to find the compatible wheel files and see if there are # any unmet dependencies left over. At this point there is nothing we # can do about any missing wheel files. We tried downloading a # compatible version directly and building from source. compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory) LOG.debug("compatible after building wheels (C compiling): %s", compatible_wheels) # Now there is still the case left over where the setup.py has been # made in such a way to be incompatible with python's setup tools, # causing it to lie about its compatibility. To fix this we have a # manually curated allowlist of packages that will work, despite # claiming otherwise. compatible_wheels, incompatible_wheels = self._apply_wheel_allowlist(compatible_wheels, incompatible_wheels) missing_wheels = deps - compatible_wheels LOG.debug("Final compatible: %s", compatible_wheels) LOG.debug("Final incompatible: %s", incompatible_wheels) LOG.debug("Final missing wheels: %s", missing_wheels) return compatible_wheels, missing_wheels def _download_all_dependencies(self, requirements_filename, directory): # Download dependencies prefering wheel files but falling back to # raw source dependences to get the transitive closure over # the dependency graph. Return the set of all package objects # which will serve as the primary list of dependencies needed to deploy # successfully. self._pip.download_all_dependencies(requirements_filename, directory) deps = { Package(directory, filename, self.python_exe) for filename in self._osutils.get_directory_contents(directory) } LOG.debug("Full dependency closure: %s", deps) return deps def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) lambda_abi = get_lambda_abi(self.runtime) platform = "manylinux2014_aarch64" if self.architecture == ARM64 else "manylinux2014_x86_64" self._pip.download_manylinux_wheels([pkg.identifier for pkg in packages], directory, lambda_abi, platform) def _build_sdists(self, sdists, directory, compile_c=True): LOG.debug("Build missing wheels from sdists " "(C compiling %s): %s", compile_c, sdists) for sdist in sdists: path_to_sdist = self._osutils.joinpath(directory, sdist.filename) self._pip.build_wheel(path_to_sdist, directory, compile_c) def _categorize_wheel_files(self, directory): final_wheels = [ Package(directory, filename, self.python_exe) for filename in self._osutils.get_directory_contents(directory) if filename.endswith(".whl") ] compatible_wheels, incompatible_wheels = set(), set() for wheel in final_wheels: if self._is_compatible_wheel_filename(wheel.filename): compatible_wheels.add(wheel) else: incompatible_wheels.add(wheel) return compatible_wheels, incompatible_wheels def _is_compatible_wheel_filename(self, filename): wheel = filename[:-4] lambda_runtime_abi = get_lambda_abi(self.runtime) for implementation, abi, platform in self._iter_all_compatibility_tags(wheel): if not self._is_compatible_platform_tag(lambda_runtime_abi, platform): continue # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for # py36. if abi == "none": return True prefix_version = implementation[:3] if prefix_version == "cp3": # Deploying python 3 function which means we need cp37m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. if abi in (lambda_runtime_abi, "abi3"): return True # Don't know what we have but it didn't pass compatibility tests. return False def _is_compatible_platform_tag(self, expected_abi, platform): """ Verify if a platform tag is compatible based on PEP 600 https://www.python.org/dev/peps/pep-0600/#specification In addition to checking the tag pattern, we also need to verify the glibc version """ if platform in self._COMPATIBLE_PLATFORMS[self.architecture]: return True arch = "aarch64" if self.architecture == ARM64 else "x86_64" # Verify the tag pattern # Try to get the matching value for legacy values or keep the current perennial_tag = self._MANYLINUX_LEGACY_MAP.get(platform, platform) match = re.match("manylinux_([0-9]+)_([0-9]+)_" + arch, perennial_tag) if match is None: return False # Get the glibc major and minor versions and compare them with the expected ABI # platform: manylinux_2_17_aarch64 -> 2 and 17 # expected_abi: cp37m -> compat glibc -> 2 and 17 # -> Compatible tag_major, tag_minor = [int(x) for x in match.groups()[:2]] runtime_major, runtime_minor = self._RUNTIME_GLIBC.get(expected_abi, self._DEFAULT_GLIBC) return (tag_major, tag_minor) <= (runtime_major, runtime_minor) def _iter_all_compatibility_tags(self, wheel): """ Generates all possible combination of tag sets as described in PEP 425 https://www.python.org/dev/peps/pep-0425/#compressed-tag-sets """ # ex: wheel = numpy-1.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64 implementation_tag, abi_tag, platform_tag = wheel.split("-")[-3:] # cp38, cp38, manylinux_2_17_aarch64.manylinux2014_aarch64 for implementation in implementation_tag.split("."): # cp38 for abi in abi_tag.split("."): # cp38 for platform in platform_tag.split("."): # manylinux_2_17_aarch64 # manylinux2014_aarch64 yield (implementation, abi, platform) def _apply_wheel_allowlist(self, compatible_wheels, incompatible_wheels): compatible_wheels = set(compatible_wheels) actual_incompatible_wheels = set() for missing_package in incompatible_wheels: if missing_package.name in self._COMPATIBLE_PACKAGE_ALLOWLIST: compatible_wheels.add(missing_package) else: actual_incompatible_wheels.add(missing_package) return compatible_wheels, actual_incompatible_wheels def _install_purelib_and_platlib(self, wheel, root): # Take a wheel package and the directory it was just unpacked into and # unpackage the purelib/platlib directories if they are present into # the parent directory. On some systems purelib and platlib need to # be installed into separate locations, for lambda this is not the case # and both should be installed in site-packages. data_dir = self._osutils.joinpath(root, wheel.data_dir) if not self._osutils.directory_exists(data_dir): return unpack_dirs = {"purelib", "platlib"} data_contents = self._osutils.get_directory_contents(data_dir) for content_name in data_contents: if content_name in unpack_dirs: source = self._osutils.joinpath(data_dir, content_name) self._osutils.copytree(source, root) # No reason to keep the purelib/platlib source directory around # so we delete it to conserve space in the package. self._osutils.rmtree(source) def _install_wheels(self, src_dir, dst_dir, wheels): if self._osutils.directory_exists(dst_dir): self._osutils.rmtree(dst_dir) self._osutils.makedirs(dst_dir) for wheel in wheels: zipfile_path = self._osutils.joinpath(src_dir, wheel.filename) self._osutils.extract_zipfile(zipfile_path, dst_dir) self._install_purelib_and_platlib(wheel, dst_dir) class Package(object): """A class to represent a package downloaded but not yet installed.""" def __init__(self, directory, filename, python_exe, osutils=None): self.dist_type = "wheel" if filename.endswith(".whl") else "sdist" self._directory = directory self.filename = filename if osutils is None: osutils = OSUtils() self._osutils = osutils self.python_exe = python_exe self._name, self._version = self._calculate_name_and_version() @property def name(self): return self._name @property def data_dir(self): # The directory format is {distribution}-{version}.data return "%s-%s.data" % (self._name, self._version) def _normalize_name(self, name): # Taken directly from PEP 503 return re.sub(r"[-_.]+", "-", name).lower() @property def identifier(self): return "%s==%s" % (self._name, self._version) def __str__(self): return "%s(%s)" % (self.identifier, self.dist_type) def __repr__(self): return str(self) def __eq__(self, other): if not isinstance(other, Package): return False return self.identifier == other.identifier def __hash__(self): return hash(self.identifier) def _calculate_name_and_version(self): if self.dist_type == "wheel": # From the wheel spec (PEP 427) # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}- # {platform tag}.whl name, version = self.filename.split("-")[:2] else: info_fetcher = SDistMetadataFetcher(self.python_exe, osutils=self._osutils) sdist_path = self._osutils.joinpath(self._directory, self.filename) name, version = info_fetcher.get_package_name_and_version(sdist_path) normalized_name = self._normalize_name(name) return normalized_name, version class SDistMetadataFetcher(object): """This is the "correct" way to get name and version from an sdist.""" # https://git.io/vQkwV _SETUPTOOLS_SHIM = ( "import setuptools, tokenize;__file__=%r;" "f=getattr(tokenize, 'open', open)(__file__);" "code=f.read().replace('\\r\\n', '\\n');" "f.close();" "exec(compile(code, __file__, 'exec'))" ) def __init__(self, python_exe, osutils=None): if osutils is None: osutils = OSUtils() self._osutils = osutils self.python_exe = python_exe def _parse_pkg_info_file(self, filepath): # The PKG-INFO generated by the egg-info command is in an email feed # format, so we use an email feedparser here to extract the metadata # from the PKG-INFO file. data = self._osutils.get_file_contents(filepath, binary=False) parser = FeedParser() parser.feed(data) return parser.close() def _get_pkg_info_filepath(self, package_dir): setup_py = self._osutils.joinpath(package_dir, "setup.py") script = self._SETUPTOOLS_SHIM % setup_py cmd = [self.python_exe, "-c", script, "--no-user-cfg", "egg_info", "--egg-base", "egg-info"] egg_info_dir = self._osutils.joinpath(package_dir, "egg-info") self._osutils.makedirs(egg_info_dir) p = subprocess.Popen( cmd, cwd=package_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self._osutils.original_environ() ) _, stderr = p.communicate() info_contents = self._osutils.get_directory_contents(egg_info_dir) if p.returncode != 0: LOG.debug("Non zero rc (%s) from the setup.py egg_info command: %s", p.returncode, stderr) if info_contents: pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO") else: # This might be a pep 517 package in which case this PKG-INFO file # should be available right in the top level directory of the sdist # in the case where the egg_info command fails. pkg_info_path = self._get_fallback_pkg_info_filepath(package_dir) if not self._osutils.file_exists(pkg_info_path): raise UnsupportedPackageError(self._osutils.basename(package_dir)) return pkg_info_path def _get_fallback_pkg_info_filepath(self, package_dir: str) -> str: """ Gets the string path of the fallback PKG-INFO that is generated by pip Parameters ---------- package_dir: str The path of the current working directory that has the package Returns ------- str Path to a potential PKG-INFO """ LOG.debug("Using fallback location for PKG-INFO file in package directory: %s", package_dir) pkg_info_path = self._osutils.joinpath(package_dir, "PKG-INFO") return pkg_info_path def _unpack_sdist_into_dir(self, sdist_path, unpack_dir): if sdist_path.endswith(".zip"): self._osutils.extract_zipfile(sdist_path, unpack_dir) elif sdist_path.endswith((".tar.gz", ".tar.bz2")): extract_tarfile(sdist_path, unpack_dir) else: raise InvalidSourceDistributionNameError(sdist_path) # There should only be one directory unpacked. contents = self._osutils.get_directory_contents(unpack_dir) return self._osutils.joinpath(unpack_dir, contents[0]) def _get_name_version(self, pkg_info_filepath: str) -> Tuple[str, str]: """ Extracts the name and the version from the PKG-INFO metadata file Parameters ---------- pkg_info_filepath: str The path to the PKG-INFO file to get data from Returns ------- Tuple[str, str] A tuple containing the name and version """ metadata = self._parse_pkg_info_file(pkg_info_filepath) return (metadata["Name"], metadata["Version"]) def _is_default_setuptools_values(self, name: str, version: str) -> bool: """ Checks if the name or the version are the default values that are assigned by setuptools Parameters ---------- name: str The name of the package version: str The version of the package as a string Returns ------- bool True if either name or version are the default values """ # default values logic: # https://github.com/pypa/setuptools/blob/6083e18f4afc40316c0112134c205c336afbcdfd/setuptools/_distutils/dist.py#L1185-L1189 return name.lower() == "unknown" or version == "0.0.0" def get_package_name_and_version(self, sdist_path: str) -> Tuple[str, str]: """ Gets the package's name and version from the metadata file Parameters ---------- sdist_path: str The string path of the downloaded source distribution artifact Returns ------- Tuple[str, str] A tuple of the name and version of the package """ with self._osutils.tempdir() as tempdir: package_dir = self._unpack_sdist_into_dir(sdist_path, tempdir) # get the name and version from the result setup.py pkg_info_filepath = self._get_pkg_info_filepath(package_dir) name, version = self._get_name_version(pkg_info_filepath) # return values if it is not the default values if not self._is_default_setuptools_values(name, version): return name, version # see if we can get the fallback PKG_INFO file from the sdist fallback_pkg_info_fp = self._get_fallback_pkg_info_filepath(package_dir) if self._osutils.file_exists(fallback_pkg_info_fp): # use the fallback values instead of the ones we generated fallback_name, fallback_version = self._get_name_version(fallback_pkg_info_fp) if not self._is_default_setuptools_values(fallback_name, fallback_version): name = fallback_name version = fallback_version return name, version class SubprocessPip(object): """Wrapper around calling pip through a subprocess.""" def __init__(self, osutils=None, python_exe=None, import_string=None): if osutils is None: osutils = OSUtils() self._osutils = osutils self.python_exe = python_exe if import_string is None: import_string = pip_import_string(python_exe=self.python_exe) self._import_string = import_string def main(self, args, env_vars=None, shim=None): if env_vars is None: env_vars = self._osutils.original_environ() if shim is None: shim = "" run_pip = ("import sys; %s; sys.exit(main(%s))") % (self._import_string, args) exec_string = "%s%s" % (shim, run_pip) invoke_pip = [self.python_exe, "-c", exec_string] p = self._osutils.popen(invoke_pip, stdout=self._osutils.pipe, stderr=self._osutils.pipe, env=env_vars) out, err = p.communicate() rc = p.returncode return rc, out, err class PipRunner(object): """Wrapper around pip calls used by chalice.""" # Update regex pattern to correspond with the updated output from pip # Specific commit: # https://github.com/pypa/pip/commit/b28e2c4928cc62d90b738a4613886fb1e2ad6a81#diff-5225c8e359020adb25dfc8c7a505950fd649c6c5775789c6f6517f7913f94542L529 # # Commit that adds extra info to the end: # https://github.com/pypa/pip/commit/c546c99480875cfe4cdeaefa6d16bad9998d0f70#diff-5225c8e359020adb25dfc8c7a505950fd649c6c5775789c6f6517f7913f94542R275-R281 # eg. Processing ./package_a (from 123==1.1.1->-r requirements.txt (line 1)) _LINK_IS_DIR_PATTERNS = ["Processing (.+?)[ ,\n]"] def __init__(self, python_exe, pip, osutils=None): if osutils is None: osutils = OSUtils() self.python_exe = python_exe self._wrapped_pip = pip self._osutils = osutils def _execute(self, command, args, env_vars=None, shim=None): """Execute a pip command with the given arguments.""" main_args = [command] + args LOG.debug("calling pip %s", " ".join(main_args)) rc, out, err = self._wrapped_pip.main(main_args, env_vars=env_vars, shim=shim) LOG.debug("pip stdout: %s", out) LOG.debug("pip stderr: %s", err) return rc, out, err def build_wheel(self, wheel, directory, compile_c=True): """Build an sdist into a wheel file.""" arguments = ["--no-deps", "--wheel-dir", directory, wheel] env_vars = self._osutils.original_environ() shim = "" if not compile_c: env_vars.update(pip_no_compile_c_env_vars) shim = pip_no_compile_c_shim # Ignore rc and stderr from this command since building the wheels # may fail and we will find out when we categorize the files that were # generated. self._execute("wheel", arguments, env_vars=env_vars, shim=shim) def download_all_dependencies(self, requirements_filename, directory): """Download all dependencies as sdist or wheel.""" arguments = ["-r", requirements_filename, "--dest", directory, "--exists-action", "i"] rc, out, err = self._execute("download", arguments) # When downloading all dependencies we expect to get an rc of 0 back # since we are casting a wide net here letting pip have options about # what to download. If a package is not found it is likely because it # does not exist and was mispelled. In this case we raise an error with # the package name. Otherwise a nonzero rc results in a generic # download error where we pass along the stderr. if rc != 0: if err is None: err = b"Unknown error" error = err.decode() match = re.search(("Could not find a version that satisfies the " "requirement (.+?) "), error) if match: package_name = match.group(1) raise NoSuchPackageError(str(package_name)) raise PackageDownloadError(error) # Extract local packages from pip output. # Iterate over possible pip outputs depending on pip version. stdout = out.decode() wheel_package_paths = set() for pattern in self._LINK_IS_DIR_PATTERNS: for match in re.finditer(pattern, stdout): wheel_package_paths.add(str(match.group(1))) for wheel_package_path in wheel_package_paths: # Looks odd we do not check on the error status of building the # wheel here. We can assume this is a valid package path since # we already passed the pip download stage. This stage would have # thrown a PackageDownloadError if any of the listed packages were # not valid. # If it fails the actual build step, it will have the same behavior # as any other package we fail to build a valid wheel for, and # complain at deployment time. self.build_wheel(wheel_package_path, directory) def download_manylinux_wheels(self, packages, directory, lambda_abi, platform="manylinux2014_x86_64"): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed # each package to pip individually. The return code of pip doesn't # matter here since we will inspect the working directory to see which # wheels were downloaded. We are only interested in wheel files # compatible with Lambda, which depends on the function architecture, # and cpython implementation. The compatible abi depends on the python # version and is checked later. for package in packages: arguments = [ "--only-binary=:all:", "--no-deps", "--platform", platform, "--implementation", "cp", "--abi", lambda_abi, "--dest", directory, package, ] self._execute("download", arguments)