concourse/pipelines/artifact-releaser-test.jsonnet (734 lines of code) (raw):

// Imports. local arle = import '../templates/arle.libsonnet'; local common = import '../templates/common.libsonnet'; // Get the repository given the build. local get_repo(build) = if std.startsWith(build, 'deb') then 'guest-arle-autopush-trusty' else if std.startsWith(build, 'el') then 'guest-arle-autopush-el7-x86_64' else 'guest-arle-autopush'; // Get the universe given the build. local get_universe(build) = if std.startsWith(build, 'deb') then 'cloud-apt' else if std.startsWith(build, 'el') then 'cloud-yum' else 'cloud-yuck'; // Package versions are preceded by '_' in debian builds, '-' in EL builds, and '.' in Windows builds. local get_filename(filename, build) = if build == 'goo' then filename + '.' else if std.startsWith(build, 'deb') then filename + '_' else filename + '-'; // Get builds given the index, or index 0 if there is only one build. local get_build(builds, index) = if std.length(builds) > 1 then builds[index] else builds[0]; // Change '-' to '_', mainly used for images. local underscore(input) = std.strReplace(input, '-', '_'); // Publish task results local publishresulttask = { local tl = self, result:: error 'must set result in publishresulttask', job:: error 'must set job in publishresulttask', task: tl.result, config: { platform: 'linux', image_resource: { type: 'registry-image-private', source: { repository: 'gcr.io/gcp-guest/concourse-metrics', google_auth: true, }, }, run: { path: '/publish-job-result', args: [ '--project-id=gcp-guest', '--zone=us-west1-a', '--pipeline=artifact-releaser-test', '--job=' + tl.job, '--task=publish-job-result', '--result-state=' + tl.result, '--start-timestamp=((.:start-timestamp-ms))', '--metric-path=concourse/job/duration', ], }, }, }; local upload_arle_autopush_staging = { local tl = self, package:: error 'must set package in upload_arle_autopush_staging', builds:: error 'must set builds in upload_arle_autopush_staging', file_endings:: error 'must set file_endings in upload_arle_autopush_staging', gcs_dir:: tl.package, gcs_pkg_names:: error 'must set gcs_pkg_names in upload_arle_autopush_staging', name: 'upload-arle-autopush-staging-%s' % tl.package, plan: [ { in_parallel: { steps: [ { get: 'guest-test-infra' }, { get: 'compute-image-tools' }, { get: 'time-interval', trigger: true }, { get: '%s-tag' % tl.package, trigger: true }, ], }, }, { task: 'generate-timestamp', file: 'guest-test-infra/concourse/tasks/generate-timestamp.yaml' }, { load_var: 'start-timestamp-ms', file: 'timestamp/timestamp-ms' }, { load_var: 'package-version', file: '%s-tag/version' % tl.package }, { // Copy all the relevant packages from prod to autopush. in_parallel: { steps: [ { task: 'upload-to-autopush-%s-%s' % [tl.package, get_build(tl.builds, i)], config: { platform: 'linux', image_resource: { type: 'registry-image', source: { repository: 'google/cloud-sdk', tag: 'alpine' }, }, run: { path: 'gcloud', args: [ 'storage', 'cp', '-n', 'gs://%s/%s/%s((.:package-version))%s' % [ common.prod_package_bucket, tl.gcs_dir, get_filename( if std.length(tl.gcs_pkg_names) > 1 then tl.gcs_pkg_names[i] else tl.gcs_pkg_names[0], get_build(tl.builds, i) ), tl.file_endings[i], ], 'gs://%s/%s/%s((.:package-version))%s' % [ common.autopush_package_bucket, tl.gcs_dir, get_filename( if std.length(tl.gcs_pkg_names) > 1 then tl.gcs_pkg_names[i] else tl.gcs_pkg_names[0], get_build(tl.builds, i) ), tl.file_endings[i], ], ], }, }, } for i in std.range(0, std.length(tl.file_endings) - 1) ], }, }, { in_parallel: { steps: [ arle.packagepublishtask { task: 'upload-arle-autopush-staging-%s-%s' % [tl.package, get_build(tl.builds, i)], topic: 'projects/artifact-releaser-autopush/topics/gcp-guest-package-upload-autopush', package_paths: '{"bucket":"%s","object":"%s/%s((.:package-version))%s"}' % [ common.autopush_package_bucket, tl.gcs_dir, get_filename( if std.length(tl.gcs_pkg_names) > 1 then tl.gcs_pkg_names[i] else tl.gcs_pkg_names[0], get_build(tl.builds, i) ), tl.file_endings[i], ], repo: get_repo(get_build(tl.builds, i)), universe: get_universe(get_build(tl.builds, i)), } // file_endings is the common denominator, so we loop through that instead. for i in std.range(0, std.length(tl.file_endings) - 1) ], }, }, ], on_success: publishresulttask { result: 'success', job: tl.name, }, on_failure: publishresulttask { result: 'failure', job: tl.name, }, }; local promote_arle_autopush_stable = { local tl = self, repos:: ['guest-arle-autopush-trusty', 'guest-arle-autopush-el7-x86_64', 'guest-arle-autopush'], universes:: ['cloud-apt', 'cloud-yum', 'cloud-yuck'], env:: 'stable', topic:: 'projects/artifact-releaser-autopush/topics/gcp-guest-package-upload-autopush', artopic:: 'projects/artifact-releaser-autopush/subscriptions/artifact-registry-package-upload-autopush', ostypes:: ['EL8_YUM', 'BULLSEYE_APT'], name: 'promote-arle-autopush-stable', plan: [ { get: 'guest-test-infra' }, { get: 'compute-image-tools' }, { get: 'time-interval', trigger: true }, { task: 'generate-timestamp', file: 'guest-test-infra/concourse/tasks/generate-timestamp.yaml', }, { load_var: 'start-timestamp-ms', file: 'timestamp/timestamp-ms' }, ] + [ { task: 'promote-autopush-stable', config: { platform: 'linux', image_resource: { type: 'registry-image', source: { repository: 'google/cloud-sdk', tag: 'alpine' }, }, run: { path: 'gcloud', args: [ 'pubsub', 'topics', 'publish', tl.topic, '--message', '{"type": "insertPackage", "request": {"universe": "%s", "repo": "%s", "environment": "%s"}}' % [ tl.universes[i], tl.repos[i], tl.env, ], ], }, }, } for i in std.range(0, std.length(tl.repos) - 1) ] + [ { task: 'promote-registry-autopush-stable', config: { platform: 'linux', image_resource: { type: 'registry-image', source: { repository: 'google/cloud-sdk', tag: 'alpine' }, }, run: { path: 'gcloud', args: [ 'pubsub', 'topics', 'publish', tl.artopic, '--message', '{"type": "uploadToArtifactReleaser", "request": {"ostype": "%s", "pkgname": "some-pkg", "pkgversion": "some-version", "reponame": "some-repo"}}' % [tl.ostypes[i]], ], }, }, } for i in std.range(0, std.length(tl.ostypes) - 1) ], on_success: publishresulttask { result: 'success', job: tl.name, }, on_failure: publishresulttask { result: 'failure', job: tl.name, }, }; local arle_publish_images_autopush = { local tl = self, image:: error 'must set image in arle_publish_images_autopush', is_windows:: false, workflow_dir:: error 'must set workflow_dir in arle_publish_images_autopush', workflow:: '%s/%s.publish.json' % [self.workflow_dir, underscore(tl.image) + if tl.is_windows then '-uefi' else ''], gcs_dir:: error 'must set gcs_dir in arle-publish-images-autopush', gcs_bucket:: common.prod_bucket, gcs:: 'gs://%s/%s' % [self.gcs_bucket, self.gcs_dir], name: 'arle-publish-images-autopush-%s' % tl.image, plan: [ { in_parallel: { steps: [ { get: 'guest-test-infra' }, { get: 'compute-image-tools' }, { get: 'time-interval', trigger: true }, { get: '%s-gcs' % tl.image, trigger: true, params: { skip_download: 'true' }, }, ], }, }, { task: 'generate-timestamp', file: 'guest-test-infra/concourse/tasks/generate-timestamp.yaml' }, { load_var: 'start-timestamp-ms', file: 'timestamp/timestamp-ms' }, { load_var: 'source-version', file: '%s-gcs/version' % tl.image }, { task: 'generate-version', file: 'guest-test-infra/concourse/tasks/generate-version.yaml' }, { load_var: 'publish-version', file: 'publish-version/version' }, { task: 'publish-autopush-%s' % tl.image, config: arle.arlepublishtask { topic: 'projects/artifact-releaser-autopush/topics/gcp-guest-image-release-autopush', image_name: tl.image, gcs_image_path: tl.gcs, source_version: 'v((.:source-version))', publish_version: '((.:publish-version))', wf: tl.workflow, }, }, ], on_success: publishresulttask { result: 'success', job: tl.name, }, on_failure: publishresulttask { result: 'failure', job: tl.name, }, }; // Image group definition. local imggroup = { local tl = self, images:: error 'must set images in imggroup', name: error 'must set name in imggroup', jobs: [ 'arle-publish-images-autopush-' + image for image in tl.images ], }; // Package group definition. local pkggroup = { local tl = self, packages:: [tl.name], name: error 'must set name in pkggroup', jobs: [ 'upload-arle-autopush-staging-%s' % package for package in tl.packages ], }; { // Linux images. local almalinux = ['almalinux-8', 'almalinux-9'], local debian = ['debian-10', 'debian-11', 'debian-11-arm64', 'debian-12', 'debian-12-arm64'], local centos = ['centos-7', 'centos-stream-9'], local rhel = [ 'rhel-7', 'rhel-7-byos', 'rhel-8', 'rhel-8-byos', 'rhel-9', 'rhel-9-arm64', 'rhel-9-byos', ], local rocky_linux = [ 'rocky-linux-8', 'rocky-linux-8-optimized-gcp', 'rocky-linux-8-optimized-gcp-arm64', 'rocky-linux-9', 'rocky-linux-9-arm64', 'rocky-linux-9-optimized-gcp', 'rocky-linux-9-optimized-gcp-arm64', ], // Windows images local windows_10_images = [ 'windows-10-21h2-ent-x64', ], local windows_11_images = [ 'windows-11-21h2-ent-x64', ], local windows_2012_images = [ 'windows-server-2012-r2-dc', 'windows-server-2012-r2-dc-core', ], local windows_2016_images = [ 'windows-server-2016-dc', 'windows-server-2016-dc-core', ], local windows_2019_images = [ 'windows-server-2019-dc', 'windows-server-2019-dc-core', ], local windows_2022_images = [ 'windows-server-2022-dc', 'windows-server-2022-dc-core', ], local sql_2014_images = [ 'sql-2014-enterprise-windows-2012-r2-dc', 'sql-2014-enterprise-windows-2016-dc', 'sql-2014-standard-windows-2012-r2-dc', 'sql-2014-web-windows-2012-r2-dc', ], local sql_2016_images = [ 'sql-2016-enterprise-windows-2012-r2-dc', 'sql-2016-enterprise-windows-2016-dc', 'sql-2016-enterprise-windows-2019-dc', 'sql-2016-standard-windows-2012-r2-dc', 'sql-2016-standard-windows-2016-dc', 'sql-2016-standard-windows-2019-dc', 'sql-2016-web-windows-2012-r2-dc', 'sql-2016-web-windows-2016-dc', 'sql-2016-web-windows-2019-dc', ], local sql_2017_images = [ 'sql-2017-enterprise-windows-2016-dc', 'sql-2017-enterprise-windows-2019-dc', 'sql-2017-enterprise-windows-2022-dc', 'sql-2017-express-windows-2012-r2-dc', 'sql-2017-express-windows-2016-dc', 'sql-2017-express-windows-2019-dc', 'sql-2017-standard-windows-2016-dc', 'sql-2017-standard-windows-2019-dc', 'sql-2017-standard-windows-2022-dc', 'sql-2017-web-windows-2016-dc', 'sql-2017-web-windows-2019-dc', 'sql-2017-web-windows-2022-dc', ], local sql_2019_images = [ 'sql-2019-enterprise-windows-2019-dc', 'sql-2019-enterprise-windows-2022-dc', 'sql-2019-standard-windows-2019-dc', 'sql-2019-standard-windows-2022-dc', 'sql-2019-web-windows-2019-dc', 'sql-2019-web-windows-2022-dc', ], local sql_2022_images = [ 'sql-2022-enterprise-windows-2019-dc', 'sql-2022-enterprise-windows-2022-dc', 'sql-2022-standard-windows-2019-dc', 'sql-2022-standard-windows-2022-dc', 'sql-2022-web-windows-2019-dc', 'sql-2022-web-windows-2022-dc', ], local mirantis_images = [ 'windows-server-2019-dc-for-containers', 'windows-server-2019-dc-core-for-containers', ], local docker_ce_images = [ 'windows-server-2019-dc-for-containers-ce', 'windows-server-2019-dc-core-for-containers-ce', ], local windows_install_media_images = [ 'windows-install-media', ], local prerelease_images = [ 'sql-2022-preview-windows-2022-dc', ], local windows_client_images = windows_10_images + windows_11_images, local windows_server_images = windows_2012_images + windows_2016_images + windows_2019_images + windows_2022_images, local sql_images = sql_2014_images + sql_2016_images + sql_2017_images + sql_2019_images + sql_2022_images, // Package builds and file endings. local guest_agent_builds = ['deb10', 'deb11-arm64', 'el7', 'el8', 'el8-arch64', 'el9', 'el9-arch64'], local guest_agent_file_endings = [ '-g1_amd64.deb', '-g1_arm64.deb', '-g1.el7.x86_64.rpm', '-g1.el8.x86_64.rpm', '-g1.el8.aarch64.rpm', '-g1.el9.x86_64.rpm', '-g1.el9.aarch64.rpm', ], local oslogin_builds = ['deb10', 'deb11', 'deb11-arm64', 'deb12', 'deb12-arm64', 'el7', 'el8', 'el8-arch64', 'el9', 'el9-arch64'], local oslogin_file_endings = [ '-g1+deb10_amd64.deb', '-g1+deb11_amd64.deb', '-g1+deb11_arm64.deb', '-g1+deb12_amd64.deb', '-g1+deb12_arm64.deb', '-g1.el7.x86_64.rpm', '-g1.el8.x86_64.rpm', '-g1.el8.aarch64.rpm', '-g1.el9.x86_64.rpm', '-g1.el9.aarch64.rpm', ], local guest_diskexpand_builds = ['deb10', 'el7', 'el8', 'el9'], local guest_diskexpand_file_endings = [ '-g1_all.deb', '-g1.el7.noarch.rpm', '-g1.el8.noarch.rpm', '-g1.el9.noarch.rpm', ], local guest_config_builds = ['deb10', 'el7', 'el8', 'el9'], local guest_config_file_endings = [ '-g1_all.deb', '-g1.el7.noarch.rpm', '-g1.el8.noarch.rpm', '-g1.el9.noarch.rpm', ], local yum_plugin_dnf_builds = ['el8', 'el8-arch64', 'el9', 'el9-arch64'], local yum_plugin_dnf_file_endings = [ '-g1.el8.x86_64.rpm', '-g1.el8.aarch64.rpm', '-g1.el9.x86_64.rpm', '-g1.el9.aarch64.rpm', ], local apt_transport_builds = ['deb10', 'deb11-arm64'], local apt_transport_file_endings = [ '-g1_amd64.deb', '-g1_arm64.deb', ], local compute_image_windows_packages = ['certgen', 'auto-updater', 'powershell', 'sysprep', 'ssh'], local compute_image_windows_gcs_names = [ 'certgen.x86_64', 'google-compute-engine-auto-updater.noarch', 'google-compute-engine-powershell.noarch', 'google-compute-engine-sysprep.noarch', 'google-compute-engine-ssh.x86_64', ], local compute_image_windows_file_endings = [ '.0@1.goo', '@1.goo', '@1.goo', '@1.goo', '.0@1.goo', ], // List of all packages. local packages = [ 'guest-agent', 'guest-oslogin', 'guest-diskexpand', 'guest-configs', 'artifact-registry-yum-plugin', 'artifact-registry-apt-transport', 'compute-image-windows', 'compute-image-tools', ], // Start of output. resource_types: [ { name: 'gcs', type: 'registry-image', source: { repository: 'frodenas/gcs-resource' }, }, { name: 'cron-resource', type: 'docker-image', source: { repository: 'cftoolsmiths/cron-resource' }, }, common.RegistryImagePrivate, ], resources: [ common.GitResource('guest-test-infra'), common.GitResource('compute-image-tools'), // Time resource. { name: 'time-interval', type: 'cron-resource', icon: 'clock-outline', source: { // Every day at midnight. expression: '0 0 * * *', fire_immediately: true, }, }, ] + [ // Package resources. { name: '%s-tag' % package, type: 'github-release', source: { owner: 'GoogleCloudPlatform', repository: package, access_token: '((github-token.token))', tag_filter: '([0-9]+\\.[0-9]+)', }, } for package in packages ] + // Image resources [common.gcsimgresource { image: image, gcs_dir: 'almalinux' } for image in almalinux] + [common.gcsimgresource { image: image, gcs_dir: 'rocky-linux' } for image in rocky_linux] + [common.gcsimgresource { image: image, regexp: 'debian/%s-v([0-9]+).tar.gz' % common.debian_image_prefixes[self.image], } for image in debian] + [common.gcsimgresource { image: image, gcs_dir: 'centos' } for image in centos] + [common.gcsimgresource { image: image, gcs_dir: 'rhel' } for image in rhel] + // Windows resources. [common.gcsimgresource { image: image, gcs_dir: 'windows-uefi', } for image in windows_client_images + windows_server_images + mirantis_images + docker_ce_images] + [common.gcsimgresource { image: image, gcs_dir: 'sqlserver-uefi' } for image in sql_images + prerelease_images] + [common.gcsimgresource { image: image, gcs_dir: 'windows-install-media' } for image in windows_install_media_images], jobs: [ upload_arle_autopush_staging { package: 'guest-agent', builds: guest_agent_builds, gcs_pkg_names: ['google-guest-agent'], file_endings: guest_agent_file_endings, }, upload_arle_autopush_staging { package: 'guest-oslogin', gcs_dir: 'oslogin', builds: oslogin_builds, gcs_pkg_names: ['google-compute-engine-oslogin'], file_endings: oslogin_file_endings, }, upload_arle_autopush_staging { package: 'guest-diskexpand', gcs_dir: 'gce-disk-expand', builds: guest_diskexpand_builds, gcs_pkg_names: ['gce-disk-expand'], file_endings: guest_diskexpand_file_endings, }, upload_arle_autopush_staging { package: 'guest-configs', gcs_dir: 'google-compute-engine', builds: guest_config_builds, gcs_pkg_names: ['google-compute-engine'], file_endings: guest_config_file_endings, }, upload_arle_autopush_staging { package: 'artifact-registry-yum-plugin', gcs_dir: 'yum-plugin-artifact-registry', builds: yum_plugin_dnf_builds, gcs_pkg_names: ['dnf-plugin-artifact-registry'], file_endings: yum_plugin_dnf_file_endings, }, upload_arle_autopush_staging { package: 'artifact-registry-apt-transport', builds: apt_transport_builds, gcs_pkg_names: ['apt-transport-artifact-registry'], file_endings: apt_transport_file_endings, }, upload_arle_autopush_staging { package: 'compute-image-tools', gcs_pkg_names: ['google-compute-engine-diagnostics.x86_64'], builds: ['goo'], file_endings: ['.0@0.goo'], }, upload_arle_autopush_staging { package: 'compute-image-windows', gcs_pkg_names: compute_image_windows_gcs_names, builds: ['goo'], file_endings: compute_image_windows_file_endings, }, ] + [ // Promote all packages in every repo. promote_arle_autopush_stable, ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'debian', workflow_dir: 'debian', } for image in debian ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'rhel', workflow_dir: 'enterprise_linux', } for image in rhel ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'centos', workflow_dir: 'enterprise_linux', } for image in centos ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'almalinux', workflow_dir: 'enterprise_linux', } for image in almalinux ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'rocky-linux', workflow_dir: 'enterprise_linux', } for image in rocky_linux ] + [ arle_publish_images_autopush { image: image, gcs_dir: 'windows-uefi', workflow_dir: 'windows', } for image in windows_client_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'windows-uefi', workflow_dir: 'windows', } for image in windows_server_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'sqlserver-uefi', workflow_dir: 'sqlserver', } for image in sql_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'sqlserver-uefi', workflow_dir: 'sqlserver', } for image in prerelease_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'windows-uefi', workflow_dir: 'windows_container', } for image in docker_ce_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'windows-uefi', workflow_dir: 'windows_container', } for image in mirantis_images ] + [ arle_publish_images_autopush { image: image, is_windows: true, gcs_dir: 'windows-install-media', workflow_dir: 'windows', } for image in windows_install_media_images ], groups: [ // Image groups imggroup { name: 'debian', images: debian }, imggroup { name: 'rhel', images: rhel }, imggroup { name: 'centos', images: centos }, imggroup { name: 'almalinux', images: almalinux }, imggroup { name: 'rocky-linux', images: rocky_linux }, imggroup { name: 'windows-client', images: windows_client_images }, imggroup { name: 'windows-server', images: windows_server_images }, imggroup { name: 'windows-sql', images: sql_images }, imggroup { name: 'windows-container', images: docker_ce_images + mirantis_images }, imggroup { name: 'windows-prerelease', images: prerelease_images }, imggroup { name: 'windows-install-media', images: windows_install_media_images }, // Package groups pkggroup { name: 'guest-agent' }, pkggroup { name: 'guest-oslogin' }, pkggroup { name: 'guest-diskexpand' }, pkggroup { name: 'guest-configs' }, pkggroup { name: 'artifact-registry-yum-plugin' }, pkggroup { name: 'artifact-registry-apt-transport' }, pkggroup { name: 'compute-image-tools' }, pkggroup { name: 'compute-image-windows' }, // Other groups. { name: 'promote-autopush-stable', jobs: ['promote-arle-autopush-stable'], }, ], }