in plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java [590:783]
private void stageFlexJavaTemplate(
TemplateDefinitions definition,
BuildPluginManager pluginManager,
String currentTemplateName,
String buildProjectId,
String imagePath,
String metadataFile,
String appRoot,
String commandSpec,
String commandSpecFileName,
String templatePath)
throws MojoExecutionException, IOException, InterruptedException, TemplateException {
String containerName = definition.getTemplateAnnotation().flexContainerName();
String tarFileName =
String.format("%s/%s/%s.tar", outputDirectory.getPath(), containerName, containerName);
Plugin plugin =
plugin(
"com.google.cloud.tools",
"jib-maven-plugin",
null,
List.of(
dependency("com.google.cloud.tools", "jib-layer-filter-extension-maven", "0.3.0")));
List<Element> elements = new ArrayList<>();
// Base image to use
elements.add(element("from", element("image", baseContainerImage)));
// Target image to stage
elements.add(element("to", element("image", imagePath)));
elements.add(
element(
"container",
element("appRoot", appRoot),
// Keep the original entrypoint
element("entrypoint", "INHERIT"),
// Point to the command spec
element("environment", element("DATAFLOW_JAVA_COMMAND_SPEC", commandSpec))));
elements.add(element("outputPaths", element("tar", tarFileName)));
// Only use shaded JAR and exclude libraries if shade was not disabled
if (System.getProperty("skipShade") == null
|| System.getProperty("skipShade").equalsIgnoreCase("false")) {
elements.add(
element(
"extraDirectories",
element(
"paths",
element(
"path",
element("from", targetDirectory + "/classes"),
element("includes", commandSpecFileName),
element("into", "/template/" + containerName + "/resources")))));
elements.add(element("containerizingMode", "packaged"));
elements.add(
element(
"pluginExtensions",
element(
"pluginExtension",
element(
"implementation",
"com.google.cloud.tools.jib.maven.extension.layerfilter.JibLayerFilterExtension"),
element(
"configuration",
attribute(
"implementation",
"com.google.cloud.tools.jib.maven.extension.layerfilter.Configuration"),
element(
"filters",
element("filter", element("glob", "**/libs/*.jar")),
element(
"filter",
element("glob", "**/libs/conscrypt-openjdk-uber-*.jar"),
element("toLayer", "conscrypt")))))));
}
// X-lang templates need to have a custom image which builds both python and java.
String[] flexTemplateBuildCmd;
if (definition.getTemplateAnnotation().type() == TemplateType.XLANG) {
String dockerfileContainer = outputClassesDirectory.getPath() + "/" + containerName;
String dockerfilePath = dockerfileContainer + "/Dockerfile";
File dockerfile = new File(dockerfilePath);
if (!dockerfile.exists()) {
List<String> filesToCopy = List.of(definition.getTemplateAnnotation().filesToCopy());
if (filesToCopy.isEmpty()) {
filesToCopy =
List.of(
String.format("%s-generated-metadata.json", containerName), "requirements.txt");
}
List<String> entryPoint = List.of(definition.getTemplateAnnotation().entryPoint());
if (entryPoint.isEmpty()) {
entryPoint = List.of(javaTemplateLauncherEntryPoint);
}
String xlangCommandSpec =
"/template/" + containerName + "/resources/" + commandSpecFileName;
// Copy in requirements.txt if present
File sourceRequirements = new File(outputClassesDirectory.getPath() + "/requirements.txt");
File destRequirements = new File(dockerfileContainer + "/requirements.txt");
if (sourceRequirements.exists()) {
Files.copy(
sourceRequirements.toPath(),
destRequirements.toPath(),
StandardCopyOption.REPLACE_EXISTING);
}
// Generate Dockerfile
LOG.info("Generating dockerfile " + dockerfilePath);
Set<String> directoriesToCopy = Set.of(containerName);
DockerfileGenerator.Builder dockerfileBuilder =
DockerfileGenerator.builder(
definition.getTemplateAnnotation().type(),
beamVersion,
containerName,
outputClassesDirectory)
.setBasePythonContainerImage(basePythonContainerImage)
.setBaseJavaContainerImage(baseContainerImage)
.setPythonVersion(pythonVersion)
.setEntryPoint(entryPoint)
.setCommandSpec(xlangCommandSpec)
.setFilesToCopy(filesToCopy)
.setDirectoriesToCopy(directoriesToCopy);
// Set Airlock parameters
if (internalMaven) {
dockerfileBuilder
.setServiceAccountSecretName(saSecretName)
.setAirlockPythonRepo(airlockPythonRepo);
}
dockerfileBuilder.build().generate();
}
// Copy java classes and libs to build directory
copyJavaArtifacts(containerName, targetDirectory, project.getArtifact().getFile());
LOG.info("Staging XLANG image using Dockerfile");
stageXlangUsingDockerfile(imagePath, containerName, buildProjectId);
} else {
// Jib's LayerFilter extension is not thread-safe, do only one at a time
synchronized (TemplatesStageMojo.class) {
executeMojo(
plugin,
goal(generateSBOM ? "buildTar" : "build"),
configuration(elements.toArray(new Element[elements.size()])),
executionEnvironment(project, session, pluginManager));
}
if (generateSBOM) {
// Send image tar to Cloud Build for vulnerability scanning before pushing
LOG.info("Using Cloud Build to push image {}", imagePath);
stageFlexTemplateUsingCloudBuild(new File(tarFileName), imagePath, buildProjectId);
}
}
// Skip GCS spec file creation
if (definition.getTemplateAnnotation().stageImageOnly()) {
return;
}
flexTemplateBuildCmd =
new String[] {
"gcloud",
"dataflow",
"flex-template",
"build",
templatePath,
"--image",
imagePath,
"--project",
projectId,
"--sdk-language",
"JAVA",
"--metadata-file",
outputClassesDirectory.getAbsolutePath() + "/" + metadataFile,
"--additional-user-labels",
"goog-dataflow-provided-template-name="
+ currentTemplateName.toLowerCase()
+ ",goog-dataflow-provided-template-version="
+ TemplateDefinitionsParser.parseVersion(stagePrefix)
+ ",goog-dataflow-provided-template-type=flex"
};
LOG.info("Running: {}", String.join(" ", flexTemplateBuildCmd));
Process process = Runtime.getRuntime().exec(flexTemplateBuildCmd);
if (process.waitFor() != 0) {
throw new RuntimeException(
"Error building template using gcloud. Please make sure it is up to date. "
+ new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8)
+ "\n"
+ new String(process.getErrorStream().readAllBytes(), StandardCharsets.UTF_8));
}
}