in gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy [376:507]
File setupPythonEnvironmentTasks(Project project, String parserVersion) {
// For offline mode:
// 1. We use the system Python on the PATH, for one set by GLEAN_PYTHON
// 2. We create a virtual environment in ~/.gradle/glean/pythonenv based on
// that Python.
// 3. We expect the wheels for glean_parser and all its depenencies in
// $rootDir/glean-wheels, or GLEAN_PYTHON_WHEELS_DIR. These can be
// downloaded in advance easily with `pip download glean_parser`.
// For online mode:
// 1. We install miniconda into ~/.gradle/glean/
// 2. glean_parser is installed using pip from pypi.org
if (isOffline) {
// This installs a virtual environment in `~/.gradle/glean/pythonenv`, so it is shared
// between multiple projects using Glean.
File envDir = new File(
project.getGradle().gradleUserHomeDir,
"glean/pythonenv"
)
if (!envDir.exists()) {
Task createGleanPythonVirtualEnv = project.task("createGleanPythonVirtualEnv", type: Exec) {
String pythonBinary = System.getenv("GLEAN_PYTHON")
if (!pythonBinary) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
pythonBinary = "python"
} else {
pythonBinary = "python3"
}
}
project.logger.warn("Building in offline mode, therefore, Glean is using a supplied Python at ${pythonBinary}")
project.logger.warn("The Python binary can be overridden with the GLEAN_PYTHON env var.")
commandLine pythonBinary
args "-m"
args "venv"
args envDir.toString()
}
Task installGleanParser = project.task("installGleanParser", type: Exec) {
String pythonPackagesDir = System.getenv("GLEAN_PYTHON_WHEELS_DIR")
if (!pythonPackagesDir) {
pythonPackagesDir = "${project.rootDir}/glean-wheels"
}
project.logger.warn("Installing glean_parser from cached Python packages in ${pythonPackagesDir}")
project.logger.warn("This can be overridden with the GLEAN_PYTHON_WHEELS_DIR env var.")
commandLine getPythonCommand(envDir, isOffline)
args "-m"
args "pip"
args "install"
args "glean_parser"
args "--no-index"
args "-f"
args pythonPackagesDir
}
installGleanParser.dependsOn(createGleanPythonVirtualEnv)
project.preBuild.finalizedBy(installGleanParser)
}
return envDir
} else {
// This sets up tasks to install a Miniconda3 environment. It installs
// into the gradle user home directory so that it will be shared between
// all libraries that use Glean. This is important because it is
// approximately 300MB in installed size.
File condaBootstrapDir = new File(
project.getGradle().gradleUserHomeDir,
"glean/bootstrap-${MINICONDA_VERSION}"
)
// Even though we are installing the Miniconda environment to the gradle user
// home directory, the gradle-python-envs plugin is hardcoded to download the
// installer to the project's build directory. Doing so will fail if the
// project's build directory doesn't already exist. This task ensures that
// the project's build directory exists before downloading and installing the
// Miniconda environment.
// See https://github.com/JetBrains/gradle-python-envs/issues/26
// The fix in the above is not actually sufficient -- we need to add createBuildDir
// as a dependency of Bootstrap_CONDA (where conda is installed), as the preBuild
// task alone isn't early enough.
Task createBuildDir = project.task("createBuildDir") {
description = "Make sure the build dir exists before creating the Python Environments"
onlyIf {
!project.file(project.buildDir).exists()
}
doLast {
project.logger.lifecycle("Creating build directory:" + project.buildDir.getPath())
project.buildDir.mkdir()
}
}
project.envs {
bootstrapDirectory = condaBootstrapDir
pipInstallOptions = "--trusted-host pypi.python.org --no-cache-dir"
// Setup a miniconda environment. conda is used because it works
// non-interactively on Windows, unlike the standard Python installers
// If we have a git package (a la `git+https://github.com`) we install that.
if (parserVersion.matches("git.+")) {
conda "Miniconda3", "Miniconda3-py311_${MINICONDA_VERSION}", "64", [parserVersion]
} else {
conda "Miniconda3", "Miniconda3-py311_${MINICONDA_VERSION}", "64", ["glean_parser~=${parserVersion}"]
}
}
File envDir = new File(
condaBootstrapDir,
"Miniconda3"
)
project.tasks.configureEach { task ->
if (task.name.startsWith('Bootstrap_CONDA')) {
task.dependsOn(createBuildDir)
// The Bootstrap_CONDA* tasks all install miniconda to the
// same place, so they can't run at the same time. This
// holds a semaphore while running the task to make sure
// only one of these classes of tasks runs at the same time.
// Solution proposed in this Gradle bug:
// https://github.com/gradle/gradle/issues/7047#issuecomment-430139316
task.doFirst { bootstrapMinicondaSemaphore.acquire() }
task.doLast { bootstrapMinicondaSemaphore.release() }
}
}
project.preBuild.dependsOn(createBuildDir)
project.preBuild.finalizedBy("build_envs")
return envDir
}
}