cli_tools/common/utils/daisyutils/daisy_utils.go (454 lines of code) (raw):
// Copyright 2018 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package daisyutils
import (
"context"
"errors"
"fmt"
"os"
"os/signal"
"regexp"
"sort"
"strings"
daisy "github.com/GoogleCloudPlatform/compute-daisy"
stringutils "github.com/GoogleCloudPlatform/compute-image-import/cli_tools/common/utils/string"
)
const (
// BuildIDOSEnvVarName is the os env var name to get build id
BuildIDOSEnvVarName = "BUILD_ID"
translateFailedPrefix = "TranslateFailed"
)
// TranslationSettings includes information that needs to be added to a disk or image after it is imported,
// for a particular OS and version.
type TranslationSettings struct {
// GcloudOsFlag is the user-facing string corresponding to this OS, version, and licensing mode.
// It is passed as a value of the `--os` flag.
GcloudOsFlag string
// LicenseURI is the GCP Compute license corresponding to this OS, version, and licensing mode:
// https://cloud.google.com/compute/docs/reference/rest/v1/licenses
LicenseURI string
// WorkflowPath is the path to a Daisy json workflow, relative to the
// `daisy_workflows/image_import` directory.
WorkflowPath string
}
var (
supportedOS = []TranslationSettings{
// Enterprise Linux
{
GcloudOsFlag: "centos-7",
WorkflowPath: "enterprise_linux/translate_centos_7.wf.json",
LicenseURI: "projects/centos-cloud/global/licenses/centos-7",
}, {
GcloudOsFlag: "centos-stream-8",
WorkflowPath: "enterprise_linux/translate_centos_stream_8.wf.json",
LicenseURI: "projects/centos-cloud/global/licenses/centos-stream",
}, {
GcloudOsFlag: "centos-stream-9",
WorkflowPath: "enterprise_linux/translate_centos_stream_9.wf.json",
LicenseURI: "projects/centos-cloud/global/licenses/centos-stream-9",
}, {
GcloudOsFlag: "rhel-6",
WorkflowPath: "enterprise_linux/translate_rhel_6_licensed.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-6-server",
}, {
GcloudOsFlag: "rhel-6-byol",
WorkflowPath: "enterprise_linux/translate_rhel_6_byol.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-6-byol",
}, {
GcloudOsFlag: "rhel-7",
WorkflowPath: "enterprise_linux/translate_rhel_7_licensed.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-7-server",
}, {
GcloudOsFlag: "rhel-7-byol",
WorkflowPath: "enterprise_linux/translate_rhel_7_byol.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-7-byol",
}, {
GcloudOsFlag: "rhel-8",
WorkflowPath: "enterprise_linux/translate_rhel_8_licensed.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-8-server",
}, {
GcloudOsFlag: "rhel-8-byol",
WorkflowPath: "enterprise_linux/translate_rhel_8_byol.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-8-byos",
}, {
GcloudOsFlag: "rhel-9",
WorkflowPath: "enterprise_linux/translate_rhel_9_licensed.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-9-server",
}, {
GcloudOsFlag: "rhel-9-byol",
WorkflowPath: "enterprise_linux/translate_rhel_9_byol.wf.json",
LicenseURI: "projects/rhel-cloud/global/licenses/rhel-9-byos",
}, {
GcloudOsFlag: "rocky-8",
WorkflowPath: "enterprise_linux/translate_rocky_8.wf.json",
LicenseURI: "projects/rocky-linux-cloud/global/licenses/rocky-linux-8",
}, {
GcloudOsFlag: "rocky-9",
WorkflowPath: "enterprise_linux/translate_rocky_9.wf.json",
LicenseURI: "projects/rocky-linux-cloud/global/licenses/rocky-linux-9",
},
// SUSE
{
GcloudOsFlag: "opensuse-15",
WorkflowPath: "suse/translate_opensuse_15.wf.json",
LicenseURI: "projects/opensuse-cloud/global/licenses/opensuse-leap-42",
}, {
GcloudOsFlag: "sles-12",
WorkflowPath: "suse/translate_sles_12.wf.json",
LicenseURI: "projects/suse-cloud/global/licenses/sles-12",
}, {
GcloudOsFlag: "sles-12-byol",
WorkflowPath: "suse/translate_sles_12_byol.wf.json",
LicenseURI: "projects/suse-byos-cloud/global/licenses/sles-12-byos",
}, {
GcloudOsFlag: "sles-sap-12",
WorkflowPath: "suse/translate_sles_sap_12.wf.json",
LicenseURI: "projects/suse-sap-cloud/global/licenses/sles-sap-12",
}, {
GcloudOsFlag: "sles-sap-12-byol",
WorkflowPath: "suse/translate_sles_sap_12_byol.wf.json",
LicenseURI: "projects/suse-byos-cloud/global/licenses/sles-sap-12-byos",
}, {
GcloudOsFlag: "sles-15",
WorkflowPath: "suse/translate_sles_15.wf.json",
LicenseURI: "projects/suse-cloud/global/licenses/sles-15",
}, {
GcloudOsFlag: "sles-15-byol",
WorkflowPath: "suse/translate_sles_15_byol.wf.json",
LicenseURI: "projects/suse-byos-cloud/global/licenses/sles-15-byos",
}, {
GcloudOsFlag: "sles-sap-15",
WorkflowPath: "suse/translate_sles_sap_15.wf.json",
LicenseURI: "projects/suse-sap-cloud/global/licenses/sles-sap-15",
}, {
GcloudOsFlag: "sles-sap-15-byol",
WorkflowPath: "suse/translate_sles_sap_15_byol.wf.json",
LicenseURI: "projects/suse-byos-cloud/global/licenses/sles-sap-15-byos",
},
// Debian
{
GcloudOsFlag: "debian-8",
WorkflowPath: "debian/translate_debian_8.wf.json",
LicenseURI: "projects/debian-cloud/global/licenses/debian-8-jessie",
}, {
GcloudOsFlag: "debian-9",
WorkflowPath: "debian/translate_debian_9.wf.json",
LicenseURI: "projects/debian-cloud/global/licenses/debian-9-stretch",
}, {
GcloudOsFlag: "debian-10",
WorkflowPath: "debian/translate_debian_10.wf.json",
LicenseURI: "projects/debian-cloud/global/licenses/debian-10-buster",
}, {
GcloudOsFlag: "debian-11",
WorkflowPath: "debian/translate_debian_11.wf.json",
LicenseURI: "projects/debian-cloud/global/licenses/debian-11-bullseye",
},
// Ubuntu
{
GcloudOsFlag: "ubuntu-1404",
WorkflowPath: "ubuntu/translate_ubuntu_1404.wf.json",
LicenseURI: "projects/ubuntu-os-cloud/global/licenses/ubuntu-1404-trusty",
}, {
GcloudOsFlag: "ubuntu-1604",
WorkflowPath: "ubuntu/translate_ubuntu_1604.wf.json",
LicenseURI: "projects/ubuntu-os-cloud/global/licenses/ubuntu-1604-xenial",
}, {
GcloudOsFlag: "ubuntu-1804",
WorkflowPath: "ubuntu/translate_ubuntu_1804.wf.json",
LicenseURI: "projects/ubuntu-os-cloud/global/licenses/ubuntu-1804-lts",
}, {
GcloudOsFlag: "ubuntu-2004",
WorkflowPath: "ubuntu/translate_ubuntu_2004.wf.json",
LicenseURI: "projects/ubuntu-os-cloud/global/licenses/ubuntu-2004-lts",
}, {
GcloudOsFlag: "ubuntu-2204",
WorkflowPath: "ubuntu/translate_ubuntu_2204.wf.json",
LicenseURI: "projects/ubuntu-os-cloud/global/licenses/ubuntu-2204-lts",
},
// Windows
{
GcloudOsFlag: "windows-7-x64-byol",
WorkflowPath: "windows/translate_windows_7_x64_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-7-x64-byol",
}, {
GcloudOsFlag: "windows-7-x86-byol",
WorkflowPath: "windows/translate_windows_7_x86_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-7-x86-byol",
}, {
GcloudOsFlag: "windows-8-x64-byol",
WorkflowPath: "windows/translate_windows_8_x64_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-8-x64-byol",
}, {
GcloudOsFlag: "windows-8-x86-byol",
WorkflowPath: "windows/translate_windows_8_x86_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-8-x86-byol",
}, {
GcloudOsFlag: "windows-10-x64-byol",
WorkflowPath: "windows/translate_windows_10_x64_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-10-x64-byol",
}, {
GcloudOsFlag: "windows-10-x86-byol",
WorkflowPath: "windows/translate_windows_10_x86_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-10-x86-byol",
}, {
GcloudOsFlag: "windows-2008r2",
WorkflowPath: "windows/translate_windows_2008_r2.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2008-r2-dc",
}, {
GcloudOsFlag: "windows-2008r2-byol",
WorkflowPath: "windows/translate_windows_2008_r2_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2008-r2-byol",
}, {
GcloudOsFlag: "windows-2012",
WorkflowPath: "windows/translate_windows_2012.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2012-dc",
}, {
GcloudOsFlag: "windows-2012-byol",
WorkflowPath: "windows/translate_windows_2012_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2012-byol",
}, {
GcloudOsFlag: "windows-2012r2",
WorkflowPath: "windows/translate_windows_2012_r2.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2012-r2-dc",
}, {
GcloudOsFlag: "windows-2012r2-byol",
WorkflowPath: "windows/translate_windows_2012_r2_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2012-r2-byol",
}, {
GcloudOsFlag: "windows-2016",
WorkflowPath: "windows/translate_windows_2016.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2016-dc",
}, {
GcloudOsFlag: "windows-2016-byol",
WorkflowPath: "windows/translate_windows_2016_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2016-byol",
}, {
GcloudOsFlag: "windows-2019",
WorkflowPath: "windows/translate_windows_2019.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2019-dc",
}, {
GcloudOsFlag: "windows-2019-byol",
WorkflowPath: "windows/translate_windows_2019_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2019-byol",
}, {
GcloudOsFlag: "windows-2022",
WorkflowPath: "windows/translate_windows_2022.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2022-dc",
}, {
GcloudOsFlag: "windows-2022-byol",
WorkflowPath: "windows/translate_windows_2022_byol.wf.json",
LicenseURI: "projects/windows-cloud/global/licenses/windows-server-2022-byol",
},
}
// osIDsReplacements maps operating systems versions identifier to an internal used namas.
osIDsReplacements = map[string]string{
"windows-7-byol": "windows-7-x64-byol",
"windows-8-1-x64-byol": "windows-8-x64-byol",
"windows-10-byol": "windows-10-x64-byol",
// Windows 11 is genuinely Windows 10 with a new explorer.exe,
// So, we're triggering the same process for windows 11 as windows 10.
"windows-11-byol": "windows-10-x64-byol",
"windows-11-x64-byol": "windows-10-x64-byol",
}
privacyRegex = regexp.MustCompile(`\[Privacy\->.*?<\-Privacy\]`)
privacyTagRegex = regexp.MustCompile(`(\[Privacy\->)|(<\-Privacy\])`)
debianWorkerRegex = regexp.MustCompile("projects/compute-image-import/global/images/debian-\\d+-worker-v")
)
// GetSortedOSIDs returns the supported OS identifiers, sorted.
func GetSortedOSIDs() []string {
choices := make([]string, 0, len(supportedOS))
for _, k := range supportedOS {
choices = append(choices, k.GcloudOsFlag)
}
sort.Strings(choices)
return choices
}
// ValidateOS validates that osID is supported by Daisy image import
func ValidateOS(osID string) error {
_, err := GetTranslationSettings(osID)
return err
}
// GetTranslationSettings returns parameters required for translating a particular OS, version,
// and licensing mode to run on GCE.
//
// An error is returned if the OS, version, and licensing mode is not supported for import.
func GetTranslationSettings(osID string) (spec TranslationSettings, err error) {
if osID == "" {
return spec, errors.New("osID is empty")
}
if replacement := osIDsReplacements[osID]; replacement != "" {
osID = replacement
}
for _, choice := range supportedOS {
if choice.GcloudOsFlag == osID {
return choice, nil
}
}
allowedValuesMsg := fmt.Sprintf("Allowed values: %v", GetSortedOSIDs())
return spec, daisy.Errf("os `%v` is invalid. "+allowedValuesMsg, osID)
}
// UpdateToUEFICompatible marks workflow resources (disks and images) to be UEFI
// compatible by adding "UEFI_COMPATIBLE" to GuestOSFeatures. Debian workers
// are excluded until UEFI becomes the default boot method.
func UpdateToUEFICompatible(workflow *daisy.Workflow) {
workflow.IterateWorkflowSteps(func(step *daisy.Step) {
if step.CreateDisks != nil {
for _, disk := range *step.CreateDisks {
// for the time being, don't run Debian worker in UEFI mode
if debianWorkerRegex.MatchString(disk.SourceImage) {
continue
}
// also, don't run Windows bootstrap worker in UEFI mode
if strings.Contains(disk.SourceImage, "projects/windows-cloud/global/images/family/windows-2019-core") && strings.Contains(disk.Name, "disk-bootstrap") {
continue
}
disk.Disk.GuestOsFeatures = daisy.CombineGuestOSFeatures(disk.Disk.GuestOsFeatures, "UEFI_COMPATIBLE")
}
}
if step.CreateImages != nil {
for _, image := range step.CreateImages.Images {
image.GuestOsFeatures = stringutils.CombineStringSlices(image.GuestOsFeatures, "UEFI_COMPATIBLE")
image.Image.GuestOsFeatures = daisy.CombineGuestOSFeatures(image.Image.GuestOsFeatures, "UEFI_COMPATIBLE")
}
for _, image := range step.CreateImages.ImagesBeta {
image.GuestOsFeatures = stringutils.CombineStringSlices(image.GuestOsFeatures, "UEFI_COMPATIBLE")
image.Image.GuestOsFeatures = daisy.CombineGuestOSFeaturesBeta(image.Image.GuestOsFeatures, "UEFI_COMPATIBLE")
}
}
})
}
// RemovePrivacyLogInfo removes privacy log information.
func RemovePrivacyLogInfo(message string) string {
// Since translation scripts vary and is hard to predict the output, we have to hide the
// details and only remain "TranslateFailed"
if strings.Contains(message, translateFailedPrefix) {
return translateFailedPrefix
}
// All import/export bash scripts enclose privacy info inside "[Privacy-> XXX <-Privacy]". Let's
// remove it for privacy.
message = privacyRegex.ReplaceAllString(message, "")
return message
}
// RemovePrivacyLogTag removes privacy log tag.
func RemovePrivacyLogTag(message string) string {
// All import/export bash scripts enclose privacy info inside a pair of tag "[Privacy->XXX<-Privacy]".
// Let's remove the tag to improve the readability.
message = privacyTagRegex.ReplaceAllString(message, "")
return message
}
// PostProcessDErrorForNetworkFlag determines whether to show more hints for network flag
func PostProcessDErrorForNetworkFlag(action string, err error, network string, w *daisy.Workflow) {
if derr, ok := err.(daisy.DError); ok {
if derr.CausedByErrType("networkResourceDoesNotExist") && network == "" {
w.LogWorkflowInfo("A VPC network is required for running %v,"+
" and the default VPC network does not exist in your project. You will need to"+
" specify a VPC network with the --network flag. For more information about"+
" VPC networks, see https://cloud.google.com/vpc.", action)
}
}
}
// RunWorkflowWithCancelSignal runs a Daisy workflow, and allows for cancellation from two sources:
// 1. The user types Ctrl-C on their keyboard.
// 2. The caller sends a cancellation reason on the cancel channel (or closes it).
func RunWorkflowWithCancelSignal(w *daisy.Workflow, cancel <-chan string) error {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func(w *daisy.Workflow) {
select {
case reason := <-cancel:
if reason != "" {
w.CancelWithReason(reason)
} else {
w.CancelWorkflow()
}
break
case <-c:
w.LogWorkflowInfo("\nCtrl-C caught, sending cancel signal to %q...\n", w.Name)
w.CancelWorkflow()
break
case <-w.Cancel:
}
}(w)
// Daisy doesn't support cancellation through context; if the context that's passed in
// is cancelled, then all of its clients die, causing confusing errors and resources
// being left that should have been cleaned up.
return w.Run(context.Background())
}
// NewStep creates a new step for the workflow along with dependencies.
func NewStep(w *daisy.Workflow, name string, dependencies ...*daisy.Step) (*daisy.Step, error) {
s, err := w.NewStep(name)
if err != nil {
return nil, err
}
err = w.AddDependency(s, dependencies...)
return s, err
}
// GetResourceID gets resource id from its URI. Definition of resource ID:
// https://cloud.google.com/apis/design/resource_names#resource_id
func GetResourceID(resourceURI string) string {
dm := strings.Split(resourceURI, "/")
return dm[len(dm)-1]
}
// GetDeviceURI gets a URI for a device based on its attributes. A device is a disk
// attached to a instance.
func GetDeviceURI(project, zone, name string) string {
return fmt.Sprintf("projects/%v/zones/%v/devices/%v", project, zone, name)
}
// GetDiskURI gets a URI for a disk based on its attributes. Introduction
// to a disk resource: https://cloud.google.com/compute/docs/reference/rest/v1/disks
func GetDiskURI(project, zone, name string) string {
return fmt.Sprintf("projects/%v/zones/%v/disks/%v", project, zone, name)
}
// GetInstanceURI gets a URI for a instance based on its attributes. Introduction
// to a instance resource: https://cloud.google.com/compute/docs/reference/rest/v1/instances
func GetInstanceURI(project, zone, name string) string {
return fmt.Sprintf("projects/%v/zones/%v/instances/%v", project, zone, name)
}
// ParseWorkflow parses Daisy workflow file and returns Daisy workflow object or error in case of failure
func ParseWorkflow(path string, varMap map[string]string, project, zone, gcsPath, oauth, dTimeout string, disableGCSLogs, disableCloudLogs, disableStdoutLogs bool) (*daisy.Workflow, error) {
w, err := daisy.NewFromFile(path)
if err != nil {
return nil, err
}
Loop:
for k, v := range varMap {
for wv := range w.Vars {
if k == wv {
w.AddVar(k, v)
continue Loop
}
}
return nil, daisy.Errf("unknown workflow Var %q passed to Workflow %q", k, w.Name)
}
EnvironmentSettings{
Project: project,
Zone: zone,
GCSPath: gcsPath,
OAuth: oauth,
Timeout: dTimeout,
DisableGCSLogs: disableGCSLogs,
DisableCloudLogs: disableCloudLogs,
DisableStdoutLogs: disableStdoutLogs,
}.ApplyToWorkflow(w)
return w, nil
}
// Tool is used to communicate the tool's name ot the user.
type Tool struct {
// HumanReadableName is used for error messages, for example: "image import".
HumanReadableName string
// ResourceLabelName is used when labeling temporary resources, for example: "image-import"
ResourceLabelName string
}
// EndpointsOverride is a type for google cloud APIs that may be overridden
type EndpointsOverride struct {
Compute string `json:",omitempty"`
Storage string `json:",omitempty"`
CloudLogging string `json:",omitempty"`
}
// EnvironmentSettings controls the resources that are used during tool execution.
type EnvironmentSettings struct {
// Location of workflows
WorkflowDirectory string
// Fields from daisy.Workflow
Project, Zone, GCSPath, OAuth, Timeout string
DisableGCSLogs, DisableCloudLogs, DisableStdoutLogs bool
EndpointsOverride EndpointsOverride
// An optional prefix to include in the bracketed portion of daisy's stdout logs.
// Gcloud does a prefix match to determine whether to show a log line to a user.
//
// With a prefix of `disk-1`, for example, the workflow in `importer.NewDaisyInflater`
// emits log messages starting with `[disk-1-inflate]`.
DaisyLogLinePrefix string
// Worker instance customizations
Network, Subnet string
ComputeServiceAccount string
NoExternalIP bool
Labels map[string]string
ExecutionID string
StorageLocation string
Tool Tool
NestedVirtualizationEnabled bool
WorkerMachineSeries []string
}
// ApplyToWorkflow sets fields on daisy.Workflow from the environment settings.
func (env EnvironmentSettings) ApplyToWorkflow(w *daisy.Workflow) {
w.Project = env.Project
w.Zone = env.Zone
if env.GCSPath != "" {
w.GCSPath = env.GCSPath
}
if env.OAuth != "" {
w.OAuthPath = env.OAuth
}
if env.Timeout != "" {
w.DefaultTimeout = env.Timeout
}
if env.DisableGCSLogs {
w.DisableGCSLogging()
}
if env.DisableCloudLogs {
w.DisableCloudLogging()
}
if env.DisableStdoutLogs {
w.DisableStdoutLogging()
}
}
// UpdateAllInstanceNoExternalIP updates all Create Instance steps in the workflow to operate
// when no external IP access is allowed by the VPC Daisy workflow is running in.
func UpdateAllInstanceNoExternalIP(workflow *daisy.Workflow, noExternalIP bool) {
if !noExternalIP {
return
}
(&RemoveExternalIPHook{}).PreRunHook(workflow)
}
// GenerateValidDisksImagesName generates a valid name for disks/images , It ensures that the generated name
// length is not greater than 63 by removing characters from the end of the name
// if the name has ID number at the end (e.g "diskName-id") remove chars before the id
func GenerateValidDisksImagesName(name string) string {
totalLen := len(name)
if totalLen > 63 {
hasID, idx := hasIDAtTheEnd(name)
charsToBeRemoved := totalLen - 63
if !hasID {
name = name[0 : len(name)-charsToBeRemoved]
} else {
nameWithoutID := name[0 : idx-charsToBeRemoved]
name = fmt.Sprintf("%s%s", nameWithoutID, name[idx:])
}
}
return name
}
// hasIDAtTheEnd uses regexp to check if name has id at the end and return its indx,
// e.g. for name = "XYZ-123", it should return (true, 3)
func hasIDAtTheEnd(name string) (bool, int) {
re := regexp.MustCompile("-[0-9]+")
allIndices := re.FindAllStringIndex(name, -1)
if len(allIndices) > 0 && allIndices[len(allIndices)-1][1] == len(name) {
return true, allIndices[len(allIndices)-1][0]
}
return false, 0
}