in plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesStageMojo.java [412:588]
protected String stageFlexTemplate(
TemplateDefinitions definition, ImageSpec imageSpec, BuildPluginManager pluginManager)
throws MojoExecutionException, IOException, InterruptedException, TemplateException {
// These are set by the .mvn/settings.xml file. This tells the plugin to use Airlock repos
// for building artifacts in Dockerfile-based images (XLANG, PYTHON, YAML). Airlock deps are
// only available when running PRs on DataflowTemplates GitHub repo and when releasing
// internally, so avoid specifying these 3 parameters when building custom templates externally.
if (!Strings.isNullOrEmpty(saSecretName)
&& !Strings.isNullOrEmpty(airlockPythonRepo)
&& !Strings.isNullOrEmpty(airlockJavaRepo)) {
internalMaven = true;
}
// Override some image spec attributes available only during staging/release:
String version = TemplateDefinitionsParser.parseVersion(stagePrefix);
String containerName = definition.getTemplateAnnotation().flexContainerName();
boolean stageImageOnly = definition.getTemplateAnnotation().stageImageOnly();
imageSpec.setAdditionalUserLabel("goog-dataflow-provided-template-version", version);
imageSpec.setImage(
generateFlexTemplateImagePath(
containerName,
projectId,
artifactRegion,
artifactRegistry,
stagePrefix,
stageImageOnly));
if (beamVersion == null || beamVersion.isEmpty()) {
beamVersion = project.getProperties().getProperty("beam-python.version");
}
String currentTemplateName = definition.getTemplateAnnotation().name();
TemplateSpecsGenerator generator = new TemplateSpecsGenerator();
boolean stageImageBeforePromote =
generateSBOM && !Strings.isNullOrEmpty(stagingArtifactRegistry);
String imagePath =
stageImageBeforePromote
? generateFlexTemplateImagePath(
containerName,
projectId,
null,
stagingArtifactRegistry,
stagePrefix,
stageImageOnly)
: imageSpec.getImage();
String buildProjectId =
stageImageBeforePromote
? new PromoteHelper.ArtifactRegImageSpec(imagePath).project
: projectId;
LOG.info("Stage image to GCR: {}", imagePath);
String metadataFile = "";
if (!stageImageOnly) {
metadataFile =
generator
.saveMetadata(definition, imageSpec.getMetadata(), outputClassesDirectory)
.getName();
}
File xlangOutputDir;
File commandSpecFile;
if (definition.getTemplateAnnotation().type() == TemplateType.XLANG) {
xlangOutputDir =
new File(outputClassesDirectory.getPath() + "/" + containerName + "/resources");
commandSpecFile = generator.saveCommandSpec(definition, xlangOutputDir);
} else {
commandSpecFile = generator.saveCommandSpec(definition, outputClassesDirectory);
}
String appRoot = "/template/" + containerName;
String commandSpec = appRoot + "/resources/" + commandSpecFile.getName();
String templatePath =
"gs://" + bucketNameOnly(bucketName) + "/" + stagePrefix + "/flex/" + currentTemplateName;
File imageSpecFile = null;
if (definition.getTemplateAnnotation().type() == TemplateType.JAVA
|| definition.getTemplateAnnotation().type() == TemplateType.XLANG) {
stageFlexJavaTemplate(
definition,
pluginManager,
currentTemplateName,
buildProjectId,
imagePath,
metadataFile,
appRoot,
commandSpec,
commandSpecFile.getName(),
templatePath);
// stageFlexJavaTemplate calls `gcloud dataflow flex-template build` command, which takes
// metadataFile as input and generates its own imageSpecFile at templatePath location, but it
// doesn't use metadataFile as-is and only picks a few attributes from it.
// Below, we are going to override this file with the one generated by the plugin, to avoid
// having a dependency on gcloud CLI. Otherwise every time a new attribute is added to the
// metadata we'll have to update gcloud CLI logic accordingly.
// TODO: Check if the same should be applied to Python templates:
if (!stageImageOnly) {
imageSpecFile = generator.saveImageSpec(definition, imageSpec, outputClassesDirectory);
LOG.info(
"Overriding Flex template spec file generated by gcloud command at [{}] with local file"
+ " [{}]",
templatePath,
imageSpecFile.getName());
}
} else if (definition.getTemplateAnnotation().type() == TemplateType.PYTHON) {
stageFlexPythonTemplate(
definition,
currentTemplateName,
buildProjectId,
imagePath,
metadataFile,
containerName,
templatePath);
} else if (definition.getTemplateAnnotation().type() == TemplateType.YAML) {
stageFlexYamlTemplate(
definition,
currentTemplateName,
buildProjectId,
imagePath,
metadataFile,
containerName,
templatePath);
} else {
throw new IllegalArgumentException(
"Type not known: " + definition.getTemplateAnnotation().type());
}
if (generateSBOM) {
// generate SBOM
File buildDir = new File(outputClassesDirectory.getAbsolutePath());
performVulnerabilityScanAndGenerateUserSBOM(imagePath, buildProjectId, buildDir);
GenerateSBOMRunnable runnable = new GenerateSBOMRunnable(imagePath);
Failsafe.with(GenerateSBOMRunnable.sbomRetryPolicy()).run(runnable);
String digest = runnable.getDigest();
if (stageImageBeforePromote) {
// promote image
PromoteHelper promoteHelper = new PromoteHelper(imagePath, imageSpec.getImage(), digest);
promoteHelper.promote();
if (!stageImageOnly) {
// overwrite image spec file
if (imageSpecFile == null) {
File folder = new File(outputClassesDirectory.getAbsolutePath() + containerName);
if (!folder.exists()) {
folder.mkdir();
}
imageSpecFile = new File(folder, currentTemplateName + "-spec-generated-metadata.json");
gcsCopy(templatePath, imageSpecFile.getAbsolutePath());
}
String content =
new String(Files.readAllBytes(imageSpecFile.toPath()), StandardCharsets.UTF_8);
String replaced = content.replace(imagePath, imageSpec.getImage());
// verify we have replaced the image path. Note: the file content may already have the
// final target image path if it was overwritten before (see "Overriding Flex template
// spec file ...") above
if (replaced.equals(content) && !content.contains(imageSpec.getImage())) {
throw new RuntimeException(
String.format(
"Unable overwrite %s to %s. Content: %s",
imagePath, imageSpec.getImage(), content.substring(0, 1000)));
}
Files.writeString(imageSpecFile.toPath(), replaced);
}
}
}
if (imageSpecFile != null) {
gcsCopy(imageSpecFile.getAbsolutePath(), templatePath);
}
LOG.info("Flex Template was staged! {}", stageImageOnly ? imageSpec.getImage() : templatePath);
return templatePath;
}