cli_tools/gce_windows_upgrade/upgrader/workflows.go (546 lines of code) (raw):
// Copyright 2020 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package upgrader
import (
"fmt"
"os"
daisy "github.com/GoogleCloudPlatform/compute-daisy"
"google.golang.org/api/compute/v1"
"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/daisyutils"
"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/path"
)
var (
upgradeSteps = map[string]func(*upgrader, *daisy.Workflow) error{
versionWindows2012r2: populateUpgradeStepsFrom2008r2To2012r2,
versionWindows2016: populateUpgradeStepsTo2016,
versionWindows2019: populateUpgradeStepsTo2019,
versionWindows2022: populateUpgradeStepsTo2022,
}
retryUpgradeSteps = map[string]func(*upgrader, *daisy.Workflow) error{versionWindows2012r2: populateRetryUpgradeStepsFrom2008r2To2012r2}
)
func (u *upgrader) prepare() (daisyutils.DaisyWorker, error) {
if u.prepareFn != nil {
return u.prepareFn()
}
return u.runWorkflowWithSteps("windows-upgrade-preparation", u.Timeout, populatePrepareSteps)
}
func populatePrepareSteps(u *upgrader, w *daisy.Workflow) error {
currentExecutablePath := os.Args[0]
w.Sources = map[string]string{"upgrade_script.ps1": path.ToWorkingDir("upgrade_script.ps1", currentExecutablePath)}
stepStopInstance, err := daisyutils.NewStep(w, "stop-instance")
if err != nil {
return err
}
stepStopInstance.StopInstances = &daisy.StopInstances{
Instances: []string{u.instanceURI},
}
prevStep := stepStopInstance
if u.CreateMachineBackup {
stepBackupMachineImage, err := daisyutils.NewStep(w, "backup-machine-image", stepStopInstance)
if err != nil {
return err
}
stepBackupMachineImage.CreateMachineImages = &daisy.CreateMachineImages{
&daisy.MachineImage{
MachineImage: compute.MachineImage{
Name: u.machineImageBackupName,
SourceInstance: u.instanceURI,
},
Resource: daisy.Resource{
ExactName: true,
NoCleanup: true,
},
},
}
prevStep = stepBackupMachineImage
}
stepBackupOSDiskSnapshot, err := daisyutils.NewStep(w, "backup-os-disk-snapshot", prevStep)
if err != nil {
return err
}
stepBackupOSDiskSnapshot.CreateSnapshots = &daisy.CreateSnapshots{
&daisy.Snapshot{
Snapshot: compute.Snapshot{
Name: u.osDiskSnapshotName,
SourceDisk: u.osDiskURI,
},
Resource: daisy.Resource{
ExactName: true,
NoCleanup: true,
},
},
}
stepCreateNewOSDisk, err := daisyutils.NewStep(w, "create-new-os-disk", stepBackupOSDiskSnapshot)
if err != nil {
return err
}
stepCreateNewOSDisk.CreateDisks = &daisy.CreateDisks{
&daisy.Disk{
Disk: compute.Disk{
Name: u.newOSDiskName,
Zone: u.instanceZone,
Type: u.osDiskType,
SourceSnapshot: u.osDiskSnapshotName,
Licenses: []string{upgradePaths[u.SourceOS][u.TargetOS].licenseToAdd},
},
Resource: daisy.Resource{
ExactName: true,
NoCleanup: true,
},
},
}
stepDetachOldOSDisk, err := daisyutils.NewStep(w, "detach-old-os-disk", stepCreateNewOSDisk)
if err != nil {
return err
}
stepDetachOldOSDisk.DetachDisks = &daisy.DetachDisks{
&daisy.DetachDisk{
Instance: u.instanceURI,
DeviceName: daisyutils.GetDeviceURI(u.instanceProject, u.instanceZone, u.osDiskDeviceName),
},
}
stepAttachNewOSDisk, err := daisyutils.NewStep(w, "attach-new-os-disk", stepDetachOldOSDisk)
if err != nil {
return err
}
stepAttachNewOSDisk.AttachDisks = &daisy.AttachDisks{
&daisy.AttachDisk{
Instance: u.instanceURI,
AttachedDisk: compute.AttachedDisk{
Source: u.newOSDiskName,
DeviceName: u.osDiskDeviceName,
AutoDelete: u.osDiskAutoDelete,
Boot: true,
},
},
}
stepCreateInstallDisk, err := daisyutils.NewStep(w, "create-install-disk", stepAttachNewOSDisk)
if err != nil {
return err
}
stepCreateInstallDisk.CreateDisks = &daisy.CreateDisks{
&daisy.Disk{
Disk: compute.Disk{
Name: u.installMediaDiskName,
Zone: u.instanceZone,
Type: "pd-ssd",
SourceImage: "projects/compute-image-tools/global/images/family/windows-install-media",
},
Resource: daisy.Resource{
ExactName: true,
NoCleanup: true,
},
},
}
if u.UseStagingInstallMedia {
(*stepCreateInstallDisk.CreateDisks)[0].SourceImage = "projects/bct-prod-images/global/images/family/windows-install-media"
}
stepAttachInstallDisk, err := daisyutils.NewStep(w, "attach-install-disk", stepCreateInstallDisk)
if err != nil {
return err
}
stepAttachInstallDisk.AttachDisks = &daisy.AttachDisks{
&daisy.AttachDisk{
Instance: u.instanceURI,
AttachedDisk: compute.AttachedDisk{
Source: u.installMediaDiskName,
AutoDelete: true,
},
},
}
prevStep = stepAttachInstallDisk
// If there isn't an original url, just skip the backup step.
if u.originalWindowsStartupScriptURL != nil {
fmt.Printf("\nDetected an existing metadata for key '%v', value='%v'. Will backup to '%v'.\n\n", metadataWindowsStartupScriptURL,
*u.originalWindowsStartupScriptURL, metadataWindowsStartupScriptURLBackup)
stepBackupScript, err := daisyutils.NewStep(w, "backup-script", stepAttachInstallDisk)
if err != nil {
return err
}
stepBackupScript.UpdateInstancesMetadata = &daisy.UpdateInstancesMetadata{
&daisy.UpdateInstanceMetadata{
Instance: u.instanceURI,
Metadata: map[string]string{metadataWindowsStartupScriptURLBackup: *u.originalWindowsStartupScriptURL},
},
}
prevStep = stepBackupScript
}
stepSetScript, err := daisyutils.NewStep(w, "set-script", prevStep)
if err != nil {
return err
}
stepSetScript.UpdateInstancesMetadata = &daisy.UpdateInstancesMetadata{
&daisy.UpdateInstanceMetadata{
Instance: u.instanceURI,
Metadata: map[string]string{
metadataWindowsStartupScriptURL: "${SOURCESPATH}/upgrade_script.ps1",
"expected-current-version": upgradePaths[u.SourceOS][u.TargetOS].expectedCurrentVersion,
"expected-new-version": upgradePaths[u.SourceOS][u.TargetOS].expectedNewVersion,
"install-folder": upgradePaths[u.SourceOS][u.TargetOS].installFolder,
},
},
}
return nil
}
func (u *upgrader) upgrade() (daisyutils.DaisyWorker, error) {
if u.upgradeFn != nil {
return u.upgradeFn()
}
return u.runWorkflowWithSteps("upgrade", u.Timeout, upgradeSteps[u.TargetOS])
}
func populateUpgradeStepsFrom2008r2To2012r2(u *upgrader, w *daisy.Workflow) error {
cleanupWorkflow, err := u.generateWorkflowWithSteps("cleanup", "10m", populateCleanupSteps)
if err != nil {
return nil
}
w.Steps = map[string]*daisy.Step{
"start-instance": {
StartInstances: &daisy.StartInstances{
Instances: []string{u.instanceURI},
},
},
"wait-for-boot": {
Timeout: "15m",
WaitForInstancesSignal: &daisy.WaitForInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: "Beginning upgrade startup script.",
},
},
},
},
"wait-for-upgrade": {
WaitForAnyInstancesSignal: &daisy.WaitForAnyInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: "windows_upgrade_current_version='Windows Server 2012 R2 Datacenter'",
FailureMatch: []string{"UpgradeFailed:"},
StatusMatch: "GCEMetadataScripts:",
},
},
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 3,
// These errors were thrown from setup.exe.
FailureMatch: []string{"Windows needs to be restarted", "CheckDiskSpaceRequirements not satisfied"},
// This is the prefix of error log emitted from install media. Catch it and write to daisy log for debugging.
StatusMatch: "$WINDOWS.~BT setuperr$",
},
},
},
},
"cleanup-temp-resources": {
IncludeWorkflow: &daisy.IncludeWorkflow{
Workflow: cleanupWorkflow,
},
},
}
w.Dependencies = map[string][]string{
"wait-for-boot": {"start-instance"},
"wait-for-upgrade": {"start-instance"},
"cleanup-temp-resources": {"wait-for-upgrade"},
}
return nil
}
func populateUpgradeStepsTo2016(u *upgrader, w *daisy.Workflow) error {
return populateUpgradeStepsTemplate(u, w, versionStringForWindows2016)
}
func populateUpgradeStepsTo2019(u *upgrader, w *daisy.Workflow) error {
return populateUpgradeStepsTemplate(u, w, versionStringForWindows2019)
}
func populateUpgradeStepsTo2022(u *upgrader, w *daisy.Workflow) error {
return populateUpgradeStepsTemplate(u, w, versionStringForWindows2022)
}
func populateUpgradeStepsTemplate(u *upgrader, w *daisy.Workflow, newVersionString string) error {
cleanupWorkflow, err := u.generateWorkflowWithSteps("cleanup", "10m", populateCleanupSteps)
if err != nil {
return nil
}
w.Steps = map[string]*daisy.Step{
"start-instance": {
StartInstances: &daisy.StartInstances{
Instances: []string{u.instanceURI},
},
},
"wait-for-boot": {
Timeout: "15m",
WaitForInstancesSignal: &daisy.WaitForInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: "Beginning upgrade startup script.",
},
},
},
},
"wait-for-upgrade": {
WaitForAnyInstancesSignal: &daisy.WaitForAnyInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: fmt.Sprintf("windows_upgrade_current_version='%v'", newVersionString),
FailureMatch: []string{"UpgradeFailed:"},
StatusMatch: "GCEMetadataScripts:",
},
},
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 3,
// These errors were thrown from setup.exe.
FailureMatch: []string{"CheckDiskSpaceRequirements not satisfied"}, //TODO: disk issue, CPU/mem issue
// This is the prefix of error log emitted from install media. Catch it and write to daisy log for debugging.
StatusMatch: "$WINDOWS.~BT setuperr$",
},
},
},
},
"cleanup-temp-resources": {
IncludeWorkflow: &daisy.IncludeWorkflow{
Workflow: cleanupWorkflow,
},
},
}
w.Dependencies = map[string][]string{
"wait-for-boot": {"start-instance"},
"wait-for-upgrade": {"start-instance"},
"cleanup-temp-resources": {"wait-for-upgrade"},
}
return nil
}
func (u *upgrader) retryUpgrade() (daisyutils.DaisyWorker, error) {
if u.retryUpgradeFn != nil {
return u.retryUpgradeFn()
}
return u.runWorkflowWithSteps("retry-upgrade", u.Timeout, retryUpgradeSteps[u.TargetOS])
}
func populateRetryUpgradeStepsFrom2008r2To2012r2(u *upgrader, w *daisy.Workflow) error {
cleanupWorkflow, err := u.generateWorkflowWithSteps("cleanup", "10m", populateCleanupSteps)
if err != nil {
return nil
}
w.Steps = map[string]*daisy.Step{
"wait-for-boot": {
Timeout: "15m",
WaitForInstancesSignal: &daisy.WaitForInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: "Beginning upgrade startup script.",
},
},
},
},
"wait-for-upgrade": {
WaitForAnyInstancesSignal: &daisy.WaitForAnyInstancesSignal{
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 1,
SuccessMatch: "windows_upgrade_current_version='Windows Server 2012 R2 Datacenter'",
FailureMatch: []string{"UpgradeFailed:"},
StatusMatch: "GCEMetadataScripts:",
},
},
{
Name: u.instanceURI,
SerialOutput: &daisy.SerialOutput{
Port: 3,
// These errors were thrown from setup.exe.
FailureMatch: []string{"Windows needs to be restarted", "CheckDiskSpaceRequirements not satisfied"},
},
},
},
},
"cleanup-temp-resources": {
IncludeWorkflow: &daisy.IncludeWorkflow{
Workflow: cleanupWorkflow,
},
},
}
w.Dependencies = map[string][]string{
"cleanup-temp-resources": {"wait-for-upgrade"},
}
return nil
}
func (u *upgrader) reboot() (daisyutils.DaisyWorker, error) {
if u.rebootFn != nil {
return u.rebootFn()
}
return u.runWorkflowWithSteps("reboot", "15m", populateRebootSteps)
}
func populateRebootSteps(u *upgrader, w *daisy.Workflow) error {
w.Steps = map[string]*daisy.Step{
"stop-instance": {
StopInstances: &daisy.StopInstances{
Instances: []string{u.instanceURI},
},
},
"start-instance": {
StartInstances: &daisy.StartInstances{
Instances: []string{u.instanceURI},
},
},
}
w.Dependencies = map[string][]string{
"start-instance": {"stop-instance"},
}
return nil
}
func (u *upgrader) cleanup() (daisyutils.DaisyWorker, error) {
if u.cleanupFn != nil {
return u.cleanupFn()
}
return u.runWorkflowWithSteps("cleanup", "20m", populateCleanupSteps)
}
func populateCleanupSteps(u *upgrader, w *daisy.Workflow) error {
w.Steps = map[string]*daisy.Step{
"restore-script": {
UpdateInstancesMetadata: &daisy.UpdateInstancesMetadata{
{
Instance: u.instanceURI,
Metadata: map[string]string{
metadataWindowsStartupScriptURL: u.getOriginalStartupScriptURL(),
metadataWindowsStartupScriptURLBackup: "",
},
},
},
},
"detach-install-media-disk": {
DetachDisks: &daisy.DetachDisks{
{
Instance: u.instanceURI,
DeviceName: daisyutils.GetDeviceURI(u.instanceProject, u.instanceZone, u.installMediaDiskName),
},
},
},
"delete-install-media-disk": {
DeleteResources: &daisy.DeleteResources{
Disks: []string{
daisyutils.GetDiskURI(u.instanceProject, u.instanceZone, u.installMediaDiskName),
},
},
},
// TODO: use a flag to determine whether to stop the instance. b/156668741
"stop-instance": {
StopInstances: &daisy.StopInstances{
Instances: []string{u.instanceURI},
},
},
}
w.Dependencies = map[string][]string{
"delete-install-media-disk": {"detach-install-media-disk"},
}
return nil
}
func (u *upgrader) rollback() (daisyutils.DaisyWorker, error) {
if u.rollbackFn != nil {
return u.rollbackFn()
}
return u.runWorkflowWithSteps("rollback", u.Timeout, populateRollbackSteps)
}
func populateRollbackSteps(u *upgrader, w *daisy.Workflow) error {
stepStopInstance, err := daisyutils.NewStep(w, "stop-instance")
if err != nil {
return err
}
stepStopInstance.StopInstances = &daisy.StopInstances{
Instances: []string{u.instanceURI},
}
stepDetachNewOSDisk, err := daisyutils.NewStep(w, "detach-new-os-disk", stepStopInstance)
if err != nil {
return err
}
stepDetachNewOSDisk.DetachDisks = &daisy.DetachDisks{
{
Instance: u.instanceURI,
DeviceName: daisyutils.GetDeviceURI(u.instanceProject, u.instanceZone, u.osDiskDeviceName),
},
}
stepAttachOldOSDisk, err := daisyutils.NewStep(w, "attach-old-os-disk", stepDetachNewOSDisk)
if err != nil {
return err
}
stepAttachOldOSDisk.AttachDisks = &daisy.AttachDisks{
{
Instance: u.instanceURI,
AttachedDisk: compute.AttachedDisk{
Source: u.osDiskURI,
DeviceName: u.osDiskDeviceName,
AutoDelete: u.osDiskAutoDelete,
Boot: true,
},
},
}
stepDetachInstallMediaDisk, err := daisyutils.NewStep(w, "detach-install-media-disk", stepAttachOldOSDisk)
if err != nil {
return err
}
stepDetachInstallMediaDisk.DetachDisks = &daisy.DetachDisks{
{
Instance: u.instanceURI,
DeviceName: daisyutils.GetDeviceURI(u.instanceProject, u.instanceZone, u.installMediaDiskName),
},
}
stepRestoreScript, err := daisyutils.NewStep(w, "restore-script", stepDetachInstallMediaDisk)
if err != nil {
return err
}
stepRestoreScript.UpdateInstancesMetadata = &daisy.UpdateInstancesMetadata{
{
Instance: u.instanceURI,
Metadata: map[string]string{
metadataWindowsStartupScriptURL: u.getOriginalStartupScriptURL(),
metadataWindowsStartupScriptURLBackup: "",
},
},
}
// TODO: use a flag to determine whether to start the instance. b/156668741
stepDeleteNewOSDiskAndInstallMediaDisk, err := daisyutils.NewStep(w, "delete-new-os-disk-and-install-media-disk", stepRestoreScript)
if err != nil {
return err
}
stepDeleteNewOSDiskAndInstallMediaDisk.DeleteResources = &daisy.DeleteResources{
Disks: []string{
daisyutils.GetDiskURI(u.instanceProject, u.instanceZone, u.newOSDiskName),
daisyutils.GetDiskURI(u.instanceProject, u.instanceZone, u.installMediaDiskName),
},
}
return nil
}
func (u *upgrader) getOriginalStartupScriptURL() string {
originalStartupScriptURL := ""
if u.originalWindowsStartupScriptURL != nil {
originalStartupScriptURL = *u.originalWindowsStartupScriptURL
}
return originalStartupScriptURL
}
func (u *upgrader) runWorkflowWithSteps(workflowName string, timeout string, populateStepsFunc func(*upgrader, *daisy.Workflow) error) (daisyutils.DaisyWorker, error) {
workflowProvider := func() (*daisy.Workflow, error) {
return u.generateWorkflowWithSteps(workflowName, timeout, populateStepsFunc)
}
env := daisyutils.EnvironmentSettings{
Project: u.instanceProject,
Zone: u.instanceZone,
GCSPath: u.ScratchBucketGcsPath,
OAuth: u.Oauth,
Timeout: u.Timeout,
ComputeEndpoint: u.Ce,
DisableGCSLogs: u.GcsLogsDisabled,
DisableCloudLogs: u.CloudLogsDisabled,
DisableStdoutLogs: u.StdoutLogsDisabled,
ExecutionID: u.executionID,
Tool: daisyutils.Tool{
HumanReadableName: "windows upgrade",
ResourceLabelName: "windows-upgrade",
},
}
worker := daisyutils.NewDaisyWorker(workflowProvider, env, u.logger)
err := worker.Run(map[string]string{})
return worker, err
}
func (u *upgrader) generateWorkflowWithSteps(workflowName string, timeout string, populateStepsFunc func(*upgrader, *daisy.Workflow) error) (*daisy.Workflow, error) {
w := daisy.New()
w.Name = workflowName
w.DefaultTimeout = timeout
err := populateStepsFunc(u, w)
return w, err
}