in chalice/deploy/packager.py [0:0]
def _download_dependencies(self, abi, directory, requirements_filename):
# type: (str, str, str) -> Tuple[Set[Package], Set[Package]]
# Download all dependencies we can, letting pip choose what to
# download.
# deps should represent the best effort we can make to gather all the
# dependencies.
deps = self._download_all_dependencies(
requirements_filename, directory)
# Sort the downloaded packages into three categories:
# - sdists (Pip could not get a wheel so it gave us an sdist)
# - lambda compatible wheel files
# - lambda incompatible wheel files
# Pip will give us a wheel when it can, but some distributions do not
# ship with wheels at all in which case we will have an sdist for it.
# In some cases a platform specific wheel file may be availble so pip
# will have downloaded that, if our platform does not match the
# platform lambda runs on (linux_x86_64/manylinux) then the downloaded
# wheel file may not be compatible with lambda. Pure python wheels
# still will be compatible because they have no platform dependencies.
sdists, compatible_wheels, incompatible_wheels = self._categorize_deps(
abi, deps)
logger.debug("Compatible wheels for Lambda: %s", compatible_wheels)
logger.debug("Initial incompatible wheels for Lambda: %s",
incompatible_wheels | sdists)
# Next we need to go through the downloaded packages and pick out any
# dependencies that do not have a compatible wheel file downloaded.
# For these packages we need to explicitly try to download a
# compatible wheel file.
missing_wheels = sdists.union(incompatible_wheels)
self._download_binary_wheels(abi, missing_wheels, directory)
# Re-count the wheel files after the second download pass. Anything
# that has an sdist but not a valid wheel file is still not going to
# work on lambda and we must now try and build the sdist into a wheel
# file ourselves.
# There also may be the case where no sdist was ever downloaded. For
# example if we are on MacOS, and the package in question has a mac
# compatible wheel file but no linux ones, we will only have an
# incompatible wheel file and no sdist. So we need to get any missing
# sdists before we can build them.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
abi, directory)
# The self._download_binary_wheels() can now introduce duplicate
# entries. For example, if we download a macOS whl at first but
# then we're able to download a manylinux1 wheel, we'll now have
# two wheels for the package, so we have to remove any compatible
# wheels from our set of incompatible wheels.
incompatible_wheels -= compatible_wheels
missing_sdists = incompatible_wheels - sdists
self._download_sdists(missing_sdists, directory)
sdists = self._find_sdists(directory)
logger.debug(
"compatible wheels after second download pass: %s",
compatible_wheels
)
missing_wheels = sdists - compatible_wheels
self._build_sdists(missing_wheels, directory, compile_c=True)
# There is still the case where the package had optional C dependencies
# for speedups. In this case the wheel file will have built above with
# the C dependencies if it managed to find a C compiler. If we are on
# an incompatible architecture this means the wheel file generated will
# not be compatible. If we categorize our files once more and find that
# there are missing dependencies we can try our last ditch effort of
# building the package and trying to sever its ability to find a C
# compiler.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
abi, directory)
logger.debug(
"compatible after building wheels (C compiling): %s",
compatible_wheels
)
missing_wheels = sdists - compatible_wheels
self._build_sdists(missing_wheels, directory, compile_c=False)
# Final pass to find the compatible wheel files and see if there are
# any unmet dependencies left over. At this point there is nothing we
# can do about any missing wheel files. We tried downloading a
# compatible version directly and building from source.
compatible_wheels, incompatible_wheels = self._categorize_wheel_files(
abi, directory)
logger.debug(
"compatible after building wheels (no C compiling): %s",
compatible_wheels
)
# Now there is still the case left over where the setup.py has been
# made in such a way to be incompatible with python's setup tools,
# causing it to lie about its compatibility. To fix this we have a
# manually curated whitelist of packages that will work, despite
# claiming otherwise.
compatible_wheels, incompatible_wheels = self._apply_wheel_whitelist(
compatible_wheels, incompatible_wheels)
missing_wheels = deps - compatible_wheels
logger.debug("Final compatible: %s", compatible_wheels)
logger.debug("Final incompatible: %s", incompatible_wheels)
logger.debug("Final missing wheels: %s", missing_wheels)
return compatible_wheels, missing_wheels