in aws_lambda_builders/workflows/python_pip/packager.py [0:0]
def _download_dependencies(self, directory, requirements_filename):
# Download all dependencies we can, letting pip choose what to
# download.
# deps should represent the best effort we can make to gather all the
# dependencies.
deps = self._download_all_dependencies(requirements_filename, directory)
# Sort the downloaded packages into three categories:
# - sdists (Pip could not get a wheel so it gave us an sdist)
# - lambda compatible wheel files
# - lambda incompatible wheel files
# Pip will give us a wheel when it can, but some distributions do not
# ship with wheels at all in which case we will have an sdist for it.
# In some cases a platform specific wheel file may be availble so pip
# will have downloaded that, if our platform does not match the
# platform that the function will run on (x86_64 or arm64) then the
# downloaded wheel file may not be compatible with Lambda. Pure python
# wheels still will be compatible because they have no platform
# dependencies.
compatible_wheels = set()
incompatible_wheels = set()
sdists = set()
for package in deps:
if package.dist_type == "sdist":
sdists.add(package)
elif self._is_compatible_wheel_filename(package.filename):
compatible_wheels.add(package)
else:
incompatible_wheels.add(package)
LOG.debug("initial compatible: %s", compatible_wheels)
LOG.debug("initial incompatible: %s", incompatible_wheels | sdists)
# Next we need to go through the downloaded packages and pick out any
# dependencies that do not have a compatible wheel file downloaded.
# For these packages we need to explicitly try to download a
# compatible wheel file.
missing_wheels = sdists | incompatible_wheels
self._download_binary_wheels(missing_wheels, directory)
# Re-count the wheel files after the second download pass. Anything
# that has an sdist but not a valid wheel file is still not going to
# work on lambda and we must now try and build the sdist into a wheel
# file ourselves.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory)
LOG.debug("compatible wheels after second download pass: %s", compatible_wheels)
missing_wheels = sdists - compatible_wheels
self._build_sdists(missing_wheels, directory, compile_c=True)
# There is still the case where the package had optional C dependencies
# for speedups. In this case the wheel file will have built above with
# the C dependencies if it managed to find a C compiler. If we are on
# an incompatible architecture this means the wheel file generated will
# not be compatible. If we categorize our files once more and find that
# there are missing dependencies we can try our last ditch effort of
# building the package and trying to sever its ability to find a C
# compiler.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory)
LOG.debug("compatible after building wheels (no C compiling): %s", compatible_wheels)
missing_wheels = sdists - compatible_wheels
self._build_sdists(missing_wheels, directory, compile_c=False)
# Final pass to find the compatible wheel files and see if there are
# any unmet dependencies left over. At this point there is nothing we
# can do about any missing wheel files. We tried downloading a
# compatible version directly and building from source.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(directory)
LOG.debug("compatible after building wheels (C compiling): %s", compatible_wheels)
# Now there is still the case left over where the setup.py has been
# made in such a way to be incompatible with python's setup tools,
# causing it to lie about its compatibility. To fix this we have a
# manually curated allowlist of packages that will work, despite
# claiming otherwise.
compatible_wheels, incompatible_wheels = self._apply_wheel_allowlist(compatible_wheels, incompatible_wheels)
missing_wheels = deps - compatible_wheels
LOG.debug("Final compatible: %s", compatible_wheels)
LOG.debug("Final incompatible: %s", incompatible_wheels)
LOG.debug("Final missing wheels: %s", missing_wheels)
return compatible_wheels, missing_wheels