in metaflow/plugins/pypi/conda_environment.py [0:0]
def get_environment(self, step):
environment = {}
for decorator in step.decorators:
# @conda decorator is guaranteed to exist thanks to self.decospecs
if decorator.name in ["conda", "pypi"]:
# handle @conda/@pypi(disabled=True)
disabled = decorator.attributes["disabled"]
if not disabled or str(disabled).lower() == "false":
environment[decorator.name] = {
k: copy.deepcopy(decorator.attributes[k])
for k in decorator.attributes
if k != "disabled"
}
else:
return {}
# Resolve conda environment for @pypi's Python, falling back on @conda's
# Python
env_python = (
environment.get("pypi", environment["conda"]).get("python")
or environment["conda"]["python"]
)
# TODO: Support dependencies for `--metadata`.
# TODO: Introduce support for `--telemetry` as a follow up.
# Certain packages are required for metaflow runtime to function correctly.
# Ensure these packages are available both in Conda channels and PyPI
# repostories.
pinned_packages = get_pinned_conda_libs(env_python, self.datastore_type)
# PyPI dependencies are prioritized over Conda dependencies.
environment.get("pypi", environment["conda"])["packages"] = {
**pinned_packages,
**environment.get("pypi", environment["conda"])["packages"],
}
# Disallow specifying both @conda and @pypi together for now. Mixing Conda
# and PyPI packages comes with a lot of operational pain that we can handle
# as follow-up work in the future.
if all(
map(lambda key: environment.get(key, {}).get("packages"), ["pypi", "conda"])
):
msg = "Mixing and matching PyPI packages and Conda packages within a\n"
msg += "step is not yet supported. Use one of @pypi or @conda only."
raise CondaEnvironmentException(msg)
# To support cross-platform environments, these invariants are maintained
# 1. Conda packages are resolved for target platforms
# 2. Conda packages are resolved for local platform only for PyPI packages
# 3. Conda environments are created only for local platform
# 4. PyPI packages are resolved for target platform within Conda environments
# created for local platform
# 5. All resolved packages (Conda or PyPI) are cached
# 6. PyPI packages are only installed for local platform
target_platform = conda_platform()
for decorator in step.decorators:
# NOTE: Keep the list of supported decorator names for backward compatibility purposes.
# Older versions did not implement the 'support_conda_environment' attribute.
if getattr(
decorator, "supports_conda_environment", False
) or decorator.name in [
"batch",
"kubernetes",
"nvidia",
"snowpark",
"slurm",
"nvct",
]:
target_platform = getattr(decorator, "target_platform", "linux-64")
break
environment["conda"]["platforms"] = [target_platform]
if "pypi" in environment:
# For PyPI packages, resolve conda environment for local platform in
# addition to target platform
environment["conda"]["platforms"] = list(
{target_platform, conda_platform()}
)
environment["pypi"]["platforms"] = [target_platform]
# Match PyPI and Conda python versions with the resolved environment Python.
environment["pypi"]["python"] = environment["conda"]["python"] = env_python
# When using `Application Default Credentials` for private GCP
# PyPI registries, the usage of environment variable `GOOGLE_APPLICATION_CREDENTIALS`
# demands that `keyrings.google-artifactregistry-auth` has to be installed
# and available in the underlying python environment.
if os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
environment["conda"]["packages"][
"keyrings.google-artifactregistry-auth"
] = ">=1.1.1"
# Z combinator for a recursive lambda
deep_sort = (lambda f: f(f))(
lambda f: lambda obj: (
{k: f(f)(v) for k, v in sorted(obj.items())}
if isinstance(obj, dict)
else sorted([f(f)(e) for e in obj]) if isinstance(obj, list) else obj
)
)
return {
**environment,
# Create a stable unique id for the environment.
# Add packageroot to the id so that packageroot modifications can
# invalidate existing environments.
"id_": sha256(
json.dumps(
deep_sort(
{
**environment,
**{
"package_root": _datastore_packageroot(
self.datastore, self.logger
)
},
}
)
).encode()
).hexdigest()[:15],
}