agent/update/processor/processor.go

// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may not
// use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package processor contains the methods for update ssm agent.
// It also provides methods for sendReply and updateInstanceInfo
package processor

import (
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"time"

	"github.com/aws/amazon-ssm-agent/common/utility"

	"github.com/aws/amazon-ssm-agent/agent/appconfig"
	"github.com/aws/amazon-ssm-agent/agent/context"
	"github.com/aws/amazon-ssm-agent/agent/contracts"
	"github.com/aws/amazon-ssm-agent/agent/fileutil"
	"github.com/aws/amazon-ssm-agent/agent/fileutil/artifact"
	"github.com/aws/amazon-ssm-agent/agent/log"
	"github.com/aws/amazon-ssm-agent/agent/platform"
	testerPkg "github.com/aws/amazon-ssm-agent/agent/update/tester"
	"github.com/aws/amazon-ssm-agent/agent/updateutil"
	"github.com/aws/amazon-ssm-agent/agent/updateutil/updateconstants"
	"github.com/aws/amazon-ssm-agent/agent/updateutil/updateinfo"
	"github.com/aws/amazon-ssm-agent/agent/updateutil/updateprecondition"
	"github.com/aws/amazon-ssm-agent/agent/updateutil/updates3util"
	"github.com/aws/amazon-ssm-agent/agent/version"
	"github.com/aws/amazon-ssm-agent/agent/versionutil"
)

var minimumSupportedVersions map[string]string
var once sync.Once

var (
	downloadArtifact           = artifact.Download
	uncompress                 = fileutil.Uncompress
	getInstalledVersionRef     = getInstalledVersions
	getDirectoryNames          = fileutil.GetDirectoryNames
	deleteDirectory            = os.RemoveAll
	getStableManifestURL       = updateutil.GetStableURLFromManifestURL
	getFileNamesLaterThan      = fileutil.GetFileNamesUnsortedLaterThan
	moveFile                   = fileutil.MoveFile
	waitForCloudInit           = utility.WaitForCloudInit
	isWindowsServer2025OrLater = platform.IsWindowsServer2025OrLater
)

const (
	defaultStdoutFileName      = "stdout"
	defaultStderrFileName      = "stderr"
	defaultSSMAgentName        = "amazon-ssm-agent"
	defaultSelfUpdateMessageID = "aws.ssm.self-update-agent.i-instanceid"
	installationDirectory      = "installationDir"
	cloudInitWaitSeconds       = 600
	agentVersionPriorWin2025   = "3.3.1230.0"
)

// NewUpdater creates an instance of Updater and other services it requires
func NewUpdater(context context.T, info updateinfo.T, updateUtilRef *updateutil.Utility) *Updater {
	updater := &Updater{
		mgr: &updateManager{
			Context: context,
			Info:    info,
			util:    updateUtilRef,
			S3util:  updates3util.New(context),
			svc: &svcManager{
				context: context,
			},
			ctxMgr: &contextManager{
				context: context,
			},
			preconditions:       updateprecondition.GetPreconditions(context),
			initManifest:        initManifest,
			initSelfUpdate:      initSelfUpdate,
			determineTarget:     determineTarget,
			validateUpdateParam: validateUpdateParam,
			populateUrlHash:     populateUrlHash,
			downloadPackages:    downloadPackages,
			update:              proceedUpdate,
			verify:              verifyInstallation,
			rollback:            rollbackInstallation,
			uninstall:           uninstallAgent,
			install:             installAgent,
			download:            downloadAndUnzipArtifact,
			clean:               cleanAgentArtifacts,
			runTests:            testerPkg.StartTests,
			finalize:            finalizeUpdateAndSendReply,
			reportMetric:        reportIntermediateMetric,
		},
	}

	return updater
}

// StartOrResumeUpdate starts/resume update.
func (u *Updater) StartOrResumeUpdate(log log.T, updateDetail *UpdateDetail) (err error) {
	switch {
	case updateDetail.State == Initialized:
		return u.mgr.initManifest(u.mgr, log, updateDetail)
	case updateDetail.State == Staged:
		return u.mgr.update(u.mgr, log, updateDetail)
	case updateDetail.State == Installed:
		return u.mgr.verify(u.mgr, log, updateDetail, false)
	case updateDetail.State == Rollback:
		return u.mgr.rollback(u.mgr, log, updateDetail)
	case updateDetail.State == RolledBack:
		return u.mgr.verify(u.mgr, log, updateDetail, true)
	}

	return nil
}

// InitializeUpdate initializes update, populates update detail
func (u *Updater) InitializeUpdate(log log.T, updateDetail *UpdateDetail) (err error) {
	var pluginResult *updateutil.UpdatePluginResult

	// load plugin update result only if not self update
	if !updateDetail.SelfUpdate {
		// Check if updatePluginResultFilePath exists
		pluginResultPath := updateDetail.UpdateRoot
		_, err = os.Stat(updateutil.UpdatePluginResultFilePath(pluginResultPath))
		if os.IsNotExist(err) {
			// for backwards compatibility for when updating from <= 3.0.951.0 to a higher version on windows
			pluginResultPath = filepath.Join(os.Getenv("TEMP"), "Amazon\\SSM\\Update")
		}
		pluginResult, err = updateutil.LoadUpdatePluginResult(log, pluginResultPath)
		if err != nil {
			return fmt.Errorf("update failed, no rollback needed %v", err.Error())
		}
	} else {
		pluginResult = &updateutil.UpdatePluginResult{StartDateTime: time.Now()}
	}

	updateDetail.StandardOut = pluginResult.StandOut
	// if failed to read time from updateplugin file
	if !pluginResult.StartDateTime.Equal(time.Time{}) {
		updateDetail.StartDateTime = pluginResult.StartDateTime
	}

	if err = u.mgr.inProgress(updateDetail, log, Initialized); err != nil {
		return
	}

	return nil
}

// Failed sets update to failed with error messages
func (u *Updater) Failed(updateDetail *UpdateDetail, log log.T, code updateconstants.ErrorCode, errMessage string, noRollbackMessage bool) (err error) {
	return u.mgr.failed(updateDetail, log, code, errMessage, noRollbackMessage)
}

// validateUpdateVersion validates target version number base on the current platform
// to avoid accidentally downgrade agent to the earlier version that doesn't support current platform
func validateUpdateVersion(log log.T, detail *UpdateDetail, info updateinfo.T) error {
	minimumVersions := getMinimumVSupportedVersions()

	// check if current platform has minimum supported version
	if val, ok := (*minimumVersions)[info.GetPlatform()]; ok {
		// compare current agent version with minimum supported version
		if compareResult, err := versionutil.VersionCompare(detail.TargetVersion, val); err != nil {
			return err
		} else if compareResult < 0 {
			return fmt.Errorf("agent version %v is unsupported on current platform", detail.TargetVersion)
		}
	}

	if windows2025OrLater, err := isWindowsServer2025OrLater(info.GetPlatformVersion(), log); err != nil {
		return err
	} else if windows2025OrLater {
		// target agent version has to be greater than the lastest agent version that didn't have support for Win 2025,
		// in order to work with it or with any other Win released after
		if compareResult, err := versionutil.VersionCompare(detail.TargetVersion, agentVersionPriorWin2025); err != nil {
			return err
		} else if compareResult <= 0 {
			return fmt.Errorf("agent version %s is unsupported on current platform", detail.TargetVersion)
		}
	}

	return nil
}

func validateInactiveVersion(context context.T, info updateinfo.T, detail *UpdateDetail) (err error) {
	context.Log().Info("Validating inactive version for amazon ssm agent")
	var isActive bool
	isActive, err = detail.Manifest.IsVersionActive(appconfig.DefaultAgentName, detail.TargetVersion)

	if err != nil {
		return fmt.Errorf("failed to check if version is active: %v", err)
	}

	if !isActive {
		return fmt.Errorf("agent version %v is inactive", detail.TargetVersion)
	}

	if detail.TargetVersion == "2.3.772.0" {
		err := fmt.Errorf("agent version %v is inactive", detail.TargetVersion)
		return err
	}

	return nil
}

func validateTargetVersionCompatible(detail *UpdateDetail) (err error) {
	if detail.UpstreamServiceName == string(contracts.MessageGatewayService) {
		if versionutil.Compare(detail.TargetVersion, updateconstants.DowngradeThroughMGSMinVersion, true) < 0 {
			return fmt.Errorf("before downgrading to %s, first downgrade to any version from 3.1.821.0 to 3.2.923.0", detail.TargetVersion)
		}
	}
	return nil
}

// getMinimumVSupportedVersions returns a map of minimum supported version and it's platform
func getMinimumVSupportedVersions() (versions *map[string]string) {
	once.Do(func() {
		minimumSupportedVersions = make(map[string]string)
		minimumSupportedVersions[updateconstants.PlatformCentOS] = "1.0.187.0"
	})
	return &minimumSupportedVersions
}

// initManifest determines the manifest URL and downloads the manifest
func initManifest(mgr *updateManager, log log.T, updateDetail *UpdateDetail) (err error) {

	// Initialize manifest URL if not set
	// Manifest URL is not set by agents < v3 so we get the manifest url from source location
	if len(updateDetail.ManifestURL) == 0 {
		log.Infof("ManifestURL is not set, attempting to get url from source location")
		updateDetail.ManifestURL, err = updateutil.GetManifestURLFromSourceUrl(updateDetail.SourceLocation)

		if err != nil {
			return mgr.failed(updateDetail, log, updateconstants.ErrorManifestURLParse, fmt.Sprintf("Failed to resolve manifest url: %v", err), true)
		}
	}

	log.Infof("Initiating download manifest %v", updateDetail.ManifestURL)
	downloadErr := mgr.S3util.DownloadManifest(updateDetail.Manifest, updateDetail.ManifestURL)
	if downloadErr != nil && downloadErr.Error != nil {
		errorCode := updateutil.ConvertToUpdateErrorCode(string(updateconstants.ErrorDownloadManifest), "_", downloadErr.ErrorCode)
		return mgr.failed(updateDetail, log, errorCode, fmt.Sprintf("Failed to download manifest: %v", downloadErr.Error.Error()), true)
	}

	return mgr.initSelfUpdate(mgr, log, updateDetail)
}

// initSelfUpdate populates the update detail for self update
func initSelfUpdate(mgr *updateManager, logger log.T, updateDetail *UpdateDetail) (err error) {
	if updateDetail.SelfUpdate {
		logger.Infof("Starting self update preparation")

		updateDetail.StdoutFileName = defaultStdoutFileName
		updateDetail.StderrFileName = defaultStderrFileName
		updateDetail.PackageName = defaultSSMAgentName
		updateDetail.MessageID = defaultSelfUpdateMessageID

		var isDeprecated bool
		// Check if current/source version is deprecated
		if isDeprecated, err = updateDetail.Manifest.IsVersionDeprecated(appconfig.DefaultAgentName, updateDetail.SourceVersion); err != nil {
			return mgr.failed(updateDetail, logger, updateconstants.ErrorVersionNotFoundInManifest, fmt.Sprintf("Failed to check if version is deprecated: %v", err), true)
		}
		logger.Infof("Checking if version %s is deprecated: %v", updateDetail.SourceVersion, isDeprecated)

		if isDeprecated {
			var targetVersion string
			if targetVersion, err = updateDetail.Manifest.GetLatestActiveVersion(appconfig.DefaultAgentName); err != nil {
				return mgr.failed(updateDetail, logger, updateconstants.ErrorGetLatestActiveVersionManifest, fmt.Sprintf("Failed to get latest active version from manifest: %v", err), true)
			}
			updateDetail.TargetVersion = targetVersion
			updateDetail.TargetResolver = updateconstants.TargetVersionSelfUpdate
			logger.Infof("Source version %s is deprecated, Target version has been set to %s", updateDetail.SourceVersion, updateDetail.TargetVersion)
		} else {
			// Return if version is not deprecated, nothing else to do for selfupdate
			logger.Infof("Current version %v is not deprecated, skipping self update", updateDetail.SourceVersion)
			return mgr.finalize(mgr, updateDetail, "")
		}
	}

	return mgr.determineTarget(mgr, logger, updateDetail)
}

// determineTarget resolves the target version if not yet defined
func determineTarget(mgr *updateManager, logger log.T, updateDetail *UpdateDetail) (err error) {

	// "None" passed as TargetVersion by the plugin if the customer does not define a version
	if updateDetail.TargetVersion == "None" {
		logger.Info("TargetVersion is empty string, defaulting to 'latest'")
		updateDetail.TargetVersion = "latest"
	}

	if updateDetail.TargetVersion == "latest" {
		updateDetail.TargetResolver = updateconstants.TargetVersionLatest
		logger.Info("TargetVersion is 'latest', attempting to find the latest active version")
		updateDetail.TargetVersion, err = updateDetail.Manifest.GetLatestActiveVersion(updateDetail.PackageName)

		if err != nil {
			return mgr.failed(updateDetail, logger, updateconstants.ErrorGetLatestActiveVersionManifest, fmt.Sprintf("Failed to get latest active version from manifest: %v", err), true)
		}
	} else if strings.ToLower(updateDetail.TargetVersion) == "stable" {
		var stableVersionUrl string
		updateDetail.TargetResolver = updateconstants.TargetVersionStable
		logger.Info("TargetVersion is 'stable', attempting to get the stable version from s3")
		stableVersionUrl, err = getStableManifestURL(updateDetail.ManifestURL, mgr.Context.Identity())
		if err != nil {
			// Should never happen because manifest has already been downloaded using the manifest url at this point
			return mgr.failed(updateDetail, logger, updateconstants.ErrorGetStableVersionS3, fmt.Sprintf("Failed to generate stable version from manifest url: %v", err), true)
		}

		updateDetail.TargetVersion, err = mgr.S3util.GetStableVersion(stableVersionUrl)
		if err != nil {
			return mgr.failed(updateDetail, logger, updateconstants.ErrorGetStableVersionS3, fmt.Sprintf("Failed to get stable version form s3: %v", err), true)
		}
	} else {
		updateDetail.TargetResolver = updateconstants.TargetVersionCustomerDefined
	}

	// TODO: Support version wild-cards e.g. 3.0.* to get latest of a minor version

	if !versionutil.IsValidVersion(updateDetail.TargetVersion) {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorInvalidTargetVersion, fmt.Sprintf("Invalid target version: %s", updateDetail.TargetVersion), true)
	}

	return mgr.validateUpdateParam(mgr, logger, updateDetail)
}

// validateUpdateParam populates the update detail for self update
func validateUpdateParam(mgr *updateManager, logger log.T, updateDetail *UpdateDetail) (err error) {
	// Only write this to console if source version has other than v1 update plugin
	if !updateutil.IsV1UpdatePlugin(updateDetail.SourceVersion) {
		updateDetail.AppendInfo(logger, "Updating %v from %v to %v",
			updateDetail.PackageName,
			updateDetail.SourceVersion,
			updateDetail.TargetVersion)
	}

	logger.Infof("Comparing source version %s and target version %s", updateDetail.SourceVersion, updateDetail.TargetVersion)

	var comp int
	comp, err = versionutil.VersionCompare(updateDetail.SourceVersion, updateDetail.TargetVersion)
	if err != nil {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorVersionCompare, fmt.Sprintf("Failed to compare versions %s and %s: %v", updateDetail.SourceVersion, updateDetail.TargetVersion, err), true)
	}

	if comp == 0 {
		updateDetail.AppendInfo(
			logger,
			"%v %v has already been installed",
			updateDetail.PackageName,
			updateDetail.SourceVersion)

		return mgr.skipped(updateDetail, logger)
	}

	// If SourceVersion is higher
	if comp > 0 {
		// Downgrade requires uninstall
		updateDetail.RequiresUninstall = true
		logger.Infof("Source version is higher than target version, will require a downgrade")

		if !updateDetail.AllowDowngrade && !allowDowngradeForVersionNotActive(mgr, logger, updateDetail) {
			logger.Warnf("Downgrade is not enabled, please enable downgrade to perform this update")
			return mgr.failed(updateDetail, logger, updateconstants.ErrorAttemptToDowngrade, fmt.Sprintf("Updating %v to an older version %v, please enable allow downgrade to proceed", updateDetail.SourceVersion, updateDetail.TargetVersion), true)
		}
	}

	logger.Infof("Verifying source version %s and target version %s are available in the manifest from %s", updateDetail.SourceVersion, updateDetail.TargetVersion, updateDetail.ManifestURL)
	if !updateDetail.Manifest.HasVersion(updateDetail.PackageName, updateDetail.SourceVersion) {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorInvalidSourceVersion, fmt.Sprintf("%v source version %v is unsupported on current platform", updateDetail.PackageName, updateDetail.SourceVersion), true)
	}

	if !updateDetail.Manifest.HasVersion(updateDetail.PackageName, updateDetail.TargetVersion) {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorInvalidTargetVersion, fmt.Sprintf("%v target version %v is unsupported on current platform", updateDetail.PackageName, updateDetail.TargetVersion), true)
	}

	// Validate target version is supported
	if err = validateUpdateVersion(logger, updateDetail, mgr.Info); err != nil {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorUnsupportedVersion, err.Error(), true)
	}

	// Validate target version is not inactive
	if err = validateInactiveVersion(mgr.Context, mgr.Info, updateDetail); err != nil {
		return mgr.inactive(updateDetail, logger, updateconstants.WarnInactiveVersion)
	}

	// Validate target version is compatible with update coming from upstream messaging service
	if err = validateTargetVersionCompatible(updateDetail); err != nil {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorIncompatibleTargetVersion, err.Error(), true)
	}

	// Checking target version update preconditions
	for _, condition := range mgr.preconditions {
		logger.Infof("Checking update precondition %s", condition.GetPreconditionName())
		if err = condition.CheckPrecondition(updateDetail.TargetVersion); err != nil {
			return mgr.failed(updateDetail, logger, updateconstants.ErrorFailedPrecondition, fmt.Sprintf("Failed update precondition check: %s", err.Error()), true)
		}
	}

	return mgr.populateUrlHash(mgr, logger, updateDetail)
}

// Allow rollback if current version installed is not active
// And that customer are not manually specifying a version.
func allowDowngradeForVersionNotActive(mgr *updateManager, logger log.T, updateDetail *UpdateDetail) (allowed bool) {
	isSourceVersionActive, err := updateDetail.Manifest.IsVersionActive(appconfig.DefaultAgentName, updateDetail.SourceVersion)
	if err != nil {
		// If we cannot read manifest, err on the safe side and do not change behavior
		logger.Errorf("Unable to read manifest file to verify active version: ", err)
		return
	}
	if updateDetail.TargetResolver != updateconstants.TargetVersionCustomerDefined && !isSourceVersionActive {
		logger.Infof("Current version %s is not active, allow downgrade", updateDetail.SourceVersion)
		allowed = true
	}
	return
}

// populateUrlHash continues initializing after self update has been handled
func populateUrlHash(mgr *updateManager, logger log.T, updateDetail *UpdateDetail) (err error) {
	logger.Infof("Attempting to get download url and artifact hashes from manifest")

	// target version download url and hash
	if updateDetail.TargetLocation, updateDetail.TargetHash, err = updateDetail.Manifest.GetDownloadURLAndHash(appconfig.DefaultAgentName, updateDetail.TargetVersion); err != nil {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorTargetPkgDownload, fmt.Sprintf("Failed to get target hash/url: %v", err), true)
	}

	// source version download url and hash
	if updateDetail.SourceLocation, updateDetail.SourceHash, err = updateDetail.Manifest.GetDownloadURLAndHash(appconfig.DefaultAgentName, updateDetail.SourceVersion); err != nil {
		return mgr.failed(updateDetail, logger, updateconstants.ErrorSourcePkgDownload, fmt.Sprintf("Failed to get source hash/url: %v", err), true)
	}

	return mgr.downloadPackages(mgr, logger, updateDetail)
}

// downloadPackages downloads artifacts from public s3 storage and sets update to status Staged
func downloadPackages(mgr *updateManager, log log.T, updateDetail *UpdateDetail) (err error) {
	log.Infof("Initiating download %v", updateDetail.PackageName)

	updateDownload := ""

	if updateDownload, err = mgr.util.CreateUpdateDownloadFolder(); err != nil {
		message := updateutil.BuildMessage(
			err,
			"failed to create download folder %v %v",
			updateDetail.PackageName,
			updateDetail.TargetVersion)
		return mgr.failed(updateDetail, log, updateconstants.ErrorCreateUpdateFolder, message, true)
	}

	// Download source
	downloadInput := artifact.DownloadInput{
		SourceURL: updateDetail.SourceLocation,
		SourceChecksums: map[string]string{
			updateconstants.HashType: updateDetail.SourceHash,
		},
		DestinationDirectory: updateDownload,
	}
	if err = mgr.download(mgr, log, downloadInput, updateDetail, updateDetail.SourceVersion); err != nil {
		return mgr.failed(updateDetail, log, updateconstants.ErrorSourcePkgDownload, err.Error(), true)
	}
	// Download target
	downloadInput = artifact.DownloadInput{
		SourceURL: updateDetail.TargetLocation,
		SourceChecksums: map[string]string{
			updateconstants.HashType: updateDetail.TargetHash,
		},
		DestinationDirectory: updateDownload,
	}
	if err = mgr.download(mgr, log, downloadInput, updateDetail, updateDetail.TargetVersion); err != nil {
		return mgr.failed(updateDetail, log, updateconstants.ErrorTargetPkgDownload, err.Error(), true)
	}
	// Update stdout
	updateDetail.AppendInfo(
		log,
		"Initiating %v update to %v",
		updateDetail.PackageName,
		updateDetail.TargetVersion)

	updateDetail.State = Staged
	updateDetail.Result = contracts.ResultStatusInProgress

	// Process update
	return mgr.update(mgr, log, updateDetail)
}

// proceedUpdate starts update process
func proceedUpdate(mgr *updateManager, log log.T, updateDetail *UpdateDetail) (err error) {
	if err := waitForCloudInit(log, cloudInitWaitSeconds); err != nil {
		log.Warnf("error waiting for cloud-init: %v", err)
	}

	log.Infof(
		"Attempting to upgrade from %v to %v",
		updateDetail.SourceVersion,
		updateDetail.TargetVersion)

	// Uninstall only when the target version is lower than the source version
	if updateDetail.RequiresUninstall {
		if exitCode, err := mgr.uninstall(mgr, log, updateDetail.SourceVersion, updateDetail); err != nil {
			message := updateutil.BuildMessage(
				err,
				"failed to uninstall %v %v",
				updateDetail.PackageName,
				updateDetail.SourceVersion)
			mgr.subStatus = updateconstants.Downgrade

			// Command never executes, no need to rollback
			if exitCode == updateconstants.ExitCodeErrorPrepareUpdateCommand {
				errorCode := updateconstants.ErrorUninstallFailed + updateconstants.ErrorCode(updateconstants.ErrorPrepareUpdateCommandSuffix)
				return mgr.failed(updateDetail, log, errorCode, message, true)
			}

			if exitCode == updateconstants.ExitCodeUnsupportedPlatform {
				return mgr.failed(updateDetail, log, updateconstants.ErrorUnsupportedServiceManager, message, true)
			}

			if slices.Contains(updateconstants.ExitCodesUpdateErrorUsingPkgMgr, exitCode) {
				updateDetail.AppendError(log, message)
				mgr.reportMetric(mgr, updateDetail, insertPkgMgrErrorCode(updateconstants.ErrorUninstallFailed, exitCode))
			} else {
				return mgr.failed(updateDetail, log, updateconstants.ErrorUninstallFailed, message, true)
			}
		}
	}

	mgr.runTests(mgr.Context, mgr.reportTestResultGenerator(updateDetail, log))

	if err = mgr.util.UpdateInstallDelayer(mgr.Context, updateDetail.UpdateRoot); err != nil {
		log.Errorf("error while executing install delayer %v", err)
	}

	if exitCode, err := mgr.install(mgr, log, updateDetail.TargetVersion, updateDetail); err != nil {
		// Install target failed with err
		// log the error and initiating rollback to the source version
		message := updateutil.BuildMessage(err,
			"failed to install %v %v",
			updateDetail.PackageName,
			updateDetail.TargetVersion)
		updateDetail.AppendError(log, message)

		// Command never executes, no need to rollback
		if exitCode == updateconstants.ExitCodeErrorPrepareUpdateCommand {
			errorCode := updateconstants.ErrorInstallFailed + updateconstants.ErrorCode(updateconstants.ErrorPrepareUpdateCommandSuffix)
			return mgr.failed(updateDetail, log, errorCode, message, true)
		}

		if exitCode == updateconstants.ExitCodeUnsupportedPlatform {
			return mgr.failed(updateDetail, log, updateconstants.ErrorUnsupportedServiceManager, message, true)
		}

		if exitCode == updateconstants.ExitCodeUpdateFailedDueToSnapd {
			return mgr.failed(updateDetail, log, updateconstants.ErrorInstallFailureDueToSnapd, message, true)
		}

		if exitCode == updateconstants.ExitCodeInstallFailedDueToSigningIssue {
			mgr.reportMetric(mgr, updateDetail, updateconstants.ErrorInstallFailedDueToSigningIssue)
		}

		if slices.Contains(updateconstants.ExitCodesUpdateErrorUsingPkgMgr, exitCode) {
			mgr.reportMetric(mgr, updateDetail, insertPkgMgrErrorCode(updateconstants.ErrorInstallFailed, exitCode))
		}

		updateDetail.AppendInfo(
			log,
			"Initiating rollback %v to %v",
			updateDetail.PackageName,
			updateDetail.SourceVersion)
		mgr.subStatus = updateconstants.InstallRollback
		// Update state to Rollback to indicate updater has initiated the rollback process
		if err = mgr.inProgress(updateDetail, log, Rollback); err != nil {
			return err
		}
		// Rollback
		return mgr.rollback(mgr, log, updateDetail)
	}

	updateDetail.State = Installed
	updateDetail.Result = contracts.ResultStatusInProgress

	// verify target version
	return mgr.verify(mgr, log, updateDetail, false)
}

// verifyInstallation checks installation result, verifies if agent is running
func verifyInstallation(mgr *updateManager, log log.T, updateDetail *UpdateDetail, isRollback bool) (err error) {
	// Check if agent is running
	var isRunning = false

	version := updateDetail.TargetVersion
	if isRollback {
		version = updateDetail.SourceVersion
	}
	log.Info("Initiating Agent service check")
	if isRunning, err = mgr.util.WaitForServiceToStart(log, mgr.Info, version); err != nil || !isRunning {
		if !isRollback {
			message := updateutil.BuildMessage(err,
				"failed to update %v to %v, %v",
				updateDetail.PackageName,
				updateDetail.TargetVersion,
				"failed to start the agent")

			updateDetail.AppendError(log, message)
			updateDetail.AppendInfo(
				log,
				"Initiating rollback %v to %v",
				updateDetail.PackageName,
				updateDetail.SourceVersion)
			mgr.subStatus = updateconstants.VerificationRollback
			// Update state to rollback
			if err = mgr.inProgress(updateDetail, log, Rollback); err != nil {
				return err
			}
			return mgr.rollback(mgr, log, updateDetail)
		}

		message := updateutil.BuildMessage(err,
			"failed to rollback %v to %v, %v",
			updateDetail.PackageName,
			updateDetail.SourceVersion,
			"failed to start the agent")
		// Rolled back, but service cannot start, Update failed.
		return mgr.failed(updateDetail, log, updateconstants.ErrorCannotStartService, message, false)
	}

	if !isRollback {
		// initiate rollback when agent installation is not complete
		if versionInstalledErrCode := mgr.util.VerifyInstalledVersion(log, version); versionInstalledErrCode != "" {
			message := updateutil.BuildMessage(nil,
				"failed to identify installed target agent version: %v",
				updateDetail.TargetVersion)
			updateDetail.AppendError(log, message)
			// we will receive only 1 error code - ErrorInstTargetVersionNotFoundViaReg
			return mgr.failed(updateDetail, log, versionInstalledErrCode, message, false)
		}
		log.Infof("%v is running", updateDetail.PackageName)
		return mgr.succeeded(updateDetail, log)
	}

	message := fmt.Sprintf("rolledback %v to %v", updateDetail.PackageName, updateDetail.SourceVersion)
	log.Infof("message is %v", message)
	return mgr.failed(updateDetail, log, updateconstants.ErrorUpdateFailRollbackSuccess, message, false)
}

// rollbackInstallation rollback installation to the source version
func rollbackInstallation(mgr *updateManager, log log.T, updateDetail *UpdateDetail) (err error) {
	if exitCode, err := mgr.uninstall(mgr, log, updateDetail.TargetVersion, updateDetail); err != nil {
		// Fail the rollback process as a result of target version cannot be uninstalled
		message := updateutil.BuildMessage(
			err,
			"failed to uninstall %v %v",
			updateDetail.PackageName,
			updateDetail.TargetVersion)

		if exitCode == updateconstants.ExitCodeErrorPrepareUpdateCommand {
			errorCode := updateconstants.ErrorUninstallFailed + updateconstants.ErrorCode(updateconstants.ErrorPrepareUpdateCommandSuffix)
			return mgr.failed(updateDetail, log, errorCode, message, false)
		}

		// this case is not possible at all as we would have caught it in the earlier uninstall/install
		// if this happens, something else is wrong so it is better to have this code for differentiation
		if exitCode == updateconstants.ExitCodeUnsupportedPlatform {
			return mgr.failed(updateDetail, log, updateconstants.ErrorUnsupportedServiceManager, message, false)
		}

		if slices.Contains(updateconstants.ExitCodesUpdateErrorUsingPkgMgr, exitCode) {
			updateDetail.AppendError(log, message)
			mgr.reportMetric(mgr, updateDetail, insertPkgMgrErrorCode(updateconstants.ErrorUninstallFailed, exitCode))
		} else {
			return mgr.failed(updateDetail, log, updateconstants.ErrorUninstallFailed, message, false)
		}
	}

	if exitCode, err := mgr.install(mgr, log, updateDetail.SourceVersion, updateDetail); err != nil {
		// Fail the rollback process as a result of source version cannot be reinstalled
		message := updateutil.BuildMessage(
			err,
			"failed to install %v %v",
			updateDetail.PackageName,
			updateDetail.SourceVersion)

		if exitCode == updateconstants.ExitCodeErrorPrepareUpdateCommand {
			errorCode := updateconstants.ErrorInstallFailed + updateconstants.ErrorCode(updateconstants.ErrorPrepareUpdateCommandSuffix)
			return mgr.failed(updateDetail, log, errorCode, message, false)
		}

		// this case is not possible at all as we would have caught it in the earlier uninstall/install
		// if this happens, something else is wrong and it is better to have this code for differentiation
		if exitCode == updateconstants.ExitCodeUnsupportedPlatform {
			return mgr.failed(updateDetail, log, updateconstants.ErrorUnsupportedServiceManager, message, false)
		}

		if exitCode == updateconstants.ExitCodeInstallFailedDueToSigningIssue {
			return mgr.failed(updateDetail, log, updateconstants.ErrorInstallFailedDueToSigningIssue, message, false)
		}

		return mgr.failed(updateDetail, log, insertPkgMgrErrorCode(updateconstants.ErrorInstallFailed, exitCode), message, false)
	}

	if err = mgr.inProgress(updateDetail, log, RolledBack); err != nil {
		return err
	}
	return mgr.verify(mgr, log, updateDetail, true)
}

// uninstall executes the uninstall script for the specific version of agent
func uninstallAgent(mgr *updateManager, log log.T, version string, updateDetail *UpdateDetail) (exitCode updateconstants.UpdateScriptExitCode, err error) {
	log.Infof("Initiating %v %v uninstallation", updateDetail.PackageName, version)

	// find the path for the uninstall script
	uninstallPath := updateutil.UnInstallerFilePath(
		updateDetail.UpdateRoot,
		updateDetail.PackageName,
		version,
		mgr.Info.GetUninstallScriptName())

	// calculate work directory
	workDir := updateutil.UpdateArtifactFolder(
		updateDetail.UpdateRoot,
		updateDetail.PackageName,
		version)

	uninstallRetryCount := 3
	uninstallRetryDelay := 1000     // 1 second
	uninstallRetryDelayBase := 2000 // 2 seconds
	input := &updateutil.CommandExecutionSettings{
		Log:         log,
		Cmd:         strings.Fields(uninstallPath),
		WorkingDir:  workDir,
		UpdaterRoot: updateDetail.UpdateRoot,
		StdOut:      updateDetail.StdoutFileName,
		StdErr:      updateDetail.StderrFileName,
		IsAsync:     false,
		Env:         getCommandEnv(log, updateDetail.UpdateRoot),
	}
	// Uninstall version - TODO - move the retry logic to ExeCommand while cleaning that function
	for retryCounter := 1; retryCounter <= uninstallRetryCount; retryCounter++ {
		_, exitCode, err = mgr.util.ExeCommand(input)
		if err == nil {
			break
		}
		if retryCounter < uninstallRetryCount {
			time.Sleep(time.Duration(uninstallRetryDelayBase+rand.Intn(uninstallRetryDelay)) * time.Millisecond)
		}
	}
	if err != nil {
		return exitCode, err
	}
	log.Infof("%v %v uninstalled successfully", updateDetail.PackageName, version)
	return exitCode, nil
}

// install executes the install script for the specific version of agent
func installAgent(mgr *updateManager, log log.T, version string, updateDetail *UpdateDetail) (exitCode updateconstants.UpdateScriptExitCode, err error) {
	log.Infof("Initiating %v %v installation", updateDetail.PackageName, version)

	// find the path for the install script
	installerPath := updateutil.InstallerFilePath(
		updateDetail.UpdateRoot,
		updateDetail.PackageName,
		version,
		mgr.Info.GetInstallScriptName())
	// calculate work directory
	workDir := updateutil.UpdateArtifactFolder(
		updateDetail.UpdateRoot,
		updateDetail.PackageName,
		version)

	// Install version - TODO - move the retry logic to ExeCommand while cleaning that function
	installRetryCount := 3
	if updateDetail.State == Staged {
		installRetryCount = 4 // this value is taken because previous updater version had total 4 retries (2 target install + 2 rollback install)
	}
	defaultTimeOut := mgr.util.GetExecutionTimeOut()

	input := &updateutil.CommandExecutionSettings{
		Log:         log,
		Cmd:         strings.Fields(installerPath),
		WorkingDir:  workDir,
		UpdaterRoot: updateDetail.UpdateRoot,
		StdOut:      updateDetail.StdoutFileName,
		StdErr:      updateDetail.StderrFileName,
		IsAsync:     false,
		Env:         getCommandEnv(log, updateDetail.UpdateRoot),
	}
	for retryCounter := 1; retryCounter <= installRetryCount; retryCounter++ {
		updateExecutionTimeoutIfNeeded(retryCounter, defaultTimeOut, mgr.util)
		if retryCounter == installRetryCount {
			log.Info("execute command and fetch error output for agent install")
			var outBytes, errBytes *bytes.Buffer
			_, exitCode, outBytes, errBytes, err = mgr.util.ExecCommandWithOutput(input)

			if strings.Contains(mgr.Info.GetInstallScriptName(), "snap") {
				if err != nil && errBytes != nil && errBytes.Len() != 0 {
					if strings.Contains(errBytes.String(), "snap \"amazon-ssm-agent\" has running apps") {
						log.Errorf("command failure for agent installed using snap: %v", err)
						return updateconstants.ExitCodeUpdateFailedDueToSnapd, err
					}
				}
			}

			if outBytes != nil && outBytes.Len() != 0 && strings.Contains(outBytes.String(), "does not verify: no digest") {
				log.Errorf("command failure for agent installed with no header: %v", err)
				return updateconstants.ExitCodeInstallFailedDueToSigningIssue, err
			}

		} else {
			_, exitCode, err = mgr.util.ExeCommand(input)
		}
		if err == nil {
			break
		}
		if retryCounter < installRetryCount {
			backOff := getNextBackOff(retryCounter)

			// Increase backoff by 30 seconds if package manager fails
			if slices.Contains(updateconstants.ExitCodesUpdateErrorUsingPkgMgr, exitCode) {
				backOff += time.Duration(30) * time.Second // 30 seconds
			}

			time.Sleep(backOff)
		}
	}
	if err != nil {
		return exitCode, err
	}
	log.Infof("%v %v installed successfully", updateDetail.PackageName, version)
	return exitCode, nil
}

// getNextBackOff gets back-off in milli-seconds based on retry counter
// (4*retryCounter) ms + 2000 ms random delay
func getNextBackOff(retryCounter int) (backOff time.Duration) {
	backOffMultiplier := 4
	maxBackOffSeconds := 20 // 20 seconds
	randomDelay := 2000     // 2 seconds

	backOffSeconds := backOffMultiplier * retryCounter
	if backOffSeconds > maxBackOffSeconds {
		backOffSeconds = maxBackOffSeconds
	}
	return time.Duration((backOffSeconds*1000)+rand.Intn(randomDelay)) * time.Millisecond
}

// getInstalledVersions returns the version which needs to be retained after deletion
// returns empty for other folders except amazon-ssm-agent, amazon-ssm-agent-updater download folders
func getInstalledVersions(updateDetail *UpdateDetail, path string) string {
	var validVersion string
	updaterBasePath := filepath.Base(path)
	if updaterBasePath == updateconstants.UpdateAmazonSSMAgentDir {
		switch updateDetail.Result {
		case contracts.ResultStatusSuccess:
			validVersion = updateDetail.TargetVersion
		case contracts.ResultStatusFailed:
			validVersion = updateDetail.SourceVersion
		}
	} else if updaterBasePath == updateconstants.UpdateAmazonSSMAgentUpdaterDir {
		validVersion = version.Version
	}
	return validVersion
}

// cleanAgentArtifacts deletes leftover files from update folders (amazon-ssm-agent, amazon-ssm-agent-updater and downloads folder)
func cleanAgentArtifacts(log log.T, updateDetail *UpdateDetail) {
	moveCleanInstallationDir(log, updateDetail)
	_ = cleanAgentUpdaterDir(log, updateDetail)
	cleanUpdateDownloadDir(log)
}

// cleanAgentUpdaterDir deletes files from update folders (amazon-ssm-agent and amazon-ssm-agent-updater folder)
func cleanAgentUpdaterDir(log log.T, updateDetail *UpdateDetail) (rmVersions map[string][]string) {
	log.Infof("initiating cleanup of other versions in amazon-ssm-agent and amazon-ssm-agent-updater folder")
	var installedVersion string
	var combinedErrors bytes.Buffer
	removedVersions := make(map[string][]string, 2)
	ssmAgentDownloadDir := filepath.Join(appconfig.UpdaterArtifactsRoot, updateconstants.UpdateAmazonSSMAgentDir)
	ssmAgentUpdaterDir := filepath.Join(appconfig.UpdaterArtifactsRoot, updateconstants.UpdateAmazonSSMAgentUpdaterDir)

	artifactDirPaths := []string{ssmAgentDownloadDir, ssmAgentUpdaterDir}
	for _, artifactDirPath := range artifactDirPaths {
		log.Infof("removing artifacts in the folder: %s", artifactDirPath)
		installedVersion = getInstalledVersionRef(updateDetail, artifactDirPath)
		artifactNames, dirErr := getDirectoryNames(artifactDirPath)
		if dirErr == nil {
			for _, artifactName := range artifactNames {
				artifactDir := filepath.Join(artifactDirPath, artifactName)
				if artifactName == installedVersion {
					// we skip installed/running versions to avoid race conditions while deleting
					continue
				} else if err := deleteDirectory(artifactDir); err != nil {
					combinedErrors.WriteString(err.Error())
					combinedErrors.WriteString("\n")
				} else {
					removedVersions[artifactDirPath] = append(removedVersions[artifactDirPath], artifactName)
				}
			}
			log.Infof("removed files and folders: %v", strings.Join(removedVersions[artifactDirPath], ", "))
		} else {
			log.Warnf("failed to list directory names: %v", dirErr)
		}
	}
	if combinedErrors.String() != "" {
		log.Warnf("combined errors while deleting files in updater and agent download folders: %v", combinedErrors.String())
	}
	return removedVersions
}

// cleanUpdateDownloadDir deletes files from update download folders (update download folder)
func cleanUpdateDownloadDir(log log.T) {
	log.Infof("initiating cleanup of files in update download folder")
	updateDownloadDir := filepath.Join(appconfig.DownloadRoot, "update")
	legacyUpdateDownloadDir := filepath.Join(appconfig.LegacyUpdateDownloadFolder, "update")
	if fileutil.Exists(legacyUpdateDownloadDir) {
		if err := deleteDirectory(legacyUpdateDownloadDir); err != nil {
			log.Warnf("error while deleting the update legacy download folder: %v", err)
		}
	}
	if err := deleteDirectory(updateDownloadDir); err != nil {
		log.Warnf("error while deleting the update download folder: %v", err)
	}
}

// downloadAndUnzipArtifact downloads installation package and unzips it
func downloadAndUnzipArtifact(
	mgr *updateManager,
	log log.T,
	downloadInput artifact.DownloadInput,
	updateDetail *UpdateDetail,
	version string) (err error) {

	log.Infof("Preparing artifacts for version %v", version)
	// download installation zip files
	downloadOutput, err := downloadArtifact(mgr.Context, downloadInput)
	if err != nil ||
		downloadOutput.IsHashMatched == false ||
		downloadOutput.LocalFilePath == "" {
		if err != nil {
			return fmt.Errorf("failed to download file reliably, %v, %v", downloadInput.SourceURL, err.Error())
		}
		return fmt.Errorf("failed to download file reliably, %v", downloadInput.SourceURL)
	}

	// downloaded successfully, append message
	updateDetail.AppendInfo(log, "Successfully downloaded %v", downloadInput.SourceURL)

	// uncompress installation package
	if err = uncompress(
		log,
		downloadOutput.LocalFilePath,
		updateutil.UpdateArtifactFolder(updateDetail.UpdateRoot, updateDetail.PackageName, version)); err != nil {
		return fmt.Errorf("failed to uncompress installation package, %v", err.Error())
	}

	return nil
}

// insertPkgMgrErrorCode inserts package manager error code suffix when applies
func insertPkgMgrErrorCode(errorCode updateconstants.ErrorCode, exitCode updateconstants.UpdateScriptExitCode) (result updateconstants.ErrorCode) {
	switch exitCode {
	default:
		result = errorCode
	case updateconstants.ExitCodeUpdateErrorUsingYumAndRpm:
		result = errorCode + updateconstants.ErrorCode(updateconstants.ErrorUsingYumAndRpmSuffix)
	case updateconstants.ExitCodeUpdateErrorUsingDpkg:
		result = errorCode + updateconstants.ErrorCode(updateconstants.ErrorUsingDpkgSuffix)
	case updateconstants.ExitCodeUpdateErrorUsingSnap:
		result = errorCode + updateconstants.ErrorCode(updateconstants.ErrorUsingSnapSuffix)
	case updateconstants.ExitCodeUpdateErrorUsingPkgMgrLegacy:
		result = errorCode + updateconstants.ErrorCode(updateconstants.ErrorUsingPkgMgrLegacySuffix)
	}
	return
}