in metaflow/plugins/pypi/pip.py [0:0]
def download(self, id_, packages, python, platform):
prefix = self.micromamba.path_to_environment(id_)
metadata_file = METADATA_FILE.format(prefix=prefix)
# download packages only if they haven't ever been downloaded before
if os.path.isfile(metadata_file):
with open(metadata_file, "r") as file:
metadata = json.load(file)
if all(package["url"] in metadata for package in packages):
return
metadata = {}
custom_index_url, extra_index_urls = self.indices(prefix)
# build wheels if needed
with ThreadPoolExecutor() as executor:
def _build(key, package):
dest = "{prefix}/.pip/built_wheels/{key}".format(prefix=prefix, key=key)
cmd = [
"wheel",
"--no-deps",
"--progress-bar=off",
"--wheel-dir=%s" % dest,
"--quiet",
*(["--index-url", custom_index_url] if custom_index_url else []),
*(
chain.from_iterable(
product(["--extra-index-url"], set(extra_index_urls))
)
),
package["url"],
]
self._call(prefix, cmd)
return package, dest
results = list(
executor.map(
lambda x: _build(*x),
enumerate(
package for package in packages if package["require_build"]
),
)
)
for package, path in results:
(wheel,) = [
f
for f in os.listdir(path)
if os.path.isfile(os.path.join(path, f)) and f.endswith(".whl")
]
if (
len(set(pip_tags(python, platform)).intersection(wheel_tags(wheel)))
== 0
):
raise PipException(
"The built wheel %s is not supported for %s with Python %s"
% (wheel, platform, python)
)
target = "{prefix}/.pip/wheels/{hash}/{wheel}".format(
prefix=prefix,
wheel=wheel,
hash=package["hash"],
)
os.makedirs(os.path.dirname(target), exist_ok=True)
shutil.move(os.path.join(path, wheel), target)
metadata["{url}".format(**package)] = target
implementations, platforms, abis = zip(
*[
(tag.interpreter, tag.platform, tag.abi)
for tag in pip_tags(python, platform)
]
)
cmd = [
"download",
"--no-deps",
"--no-index",
"--progress-bar=off",
# if packages are present in Pip cache, this will be a local copy
"--dest={prefix}/.pip/wheels".format(prefix=prefix),
"--quiet",
*(["--index-url", custom_index_url] if custom_index_url else []),
*(
chain.from_iterable(
product(["--extra-index-url"], set(extra_index_urls))
)
),
*(chain.from_iterable(product(["--abi"], set(abis)))),
*(chain.from_iterable(product(["--platform"], set(platforms)))),
# *(chain.from_iterable(product(["--implementations"], set(implementations)))),
]
packages = [package for package in packages if not package["require_build"]]
for package in packages:
cmd.append("{url}".format(**package))
metadata["{url}".format(**package)] = "{prefix}/.pip/wheels/{wheel}".format(
prefix=prefix, wheel=unquote(package["url"].split("/")[-1])
)
self._call(prefix, cmd)
# write the url to wheel mappings in a magic location
with open(metadata_file, "w") as file:
file.write(json.dumps(metadata))