core/app/selfupdate/fileutil/artifact/artifact.go

// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may not
// use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package artifact contains utilities for working downloading files.
package artifact

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/aws/amazon-ssm-agent/agent/appconfig"
	"github.com/aws/amazon-ssm-agent/agent/log"
	"github.com/aws/amazon-ssm-agent/agent/network"
	"github.com/aws/amazon-ssm-agent/core/app/selfupdate/fileutil"
)

// DownloadOutput holds the result of file download operation.
type DownloadOutput struct {
	LocalFilePath string
	IsUpdated     bool
	IsHashMatched bool
}

// DownloadInput specifies the input to file download operation
type DownloadInput struct {
	SourceURL            string
	DestinationDirectory string
	SourceChecksums      map[string]string
}

type Artifact struct {
	log       log.T
	appConfig appconfig.SsmagentConfig
	fileutil  *fileutil.Fileutil
}

type IArtifact interface {
	Download(input DownloadInput) (output DownloadOutput, err error)
	VerifyHash(input DownloadInput, output DownloadOutput) (bool, error)
	Uncompress(src, dest string) error
}

func NewSelfUpdateArtifact(log log.T, appConfig appconfig.SsmagentConfig) *Artifact {
	log.Debugf("Initializing self update artifact")
	futl := fileutil.NewFileUtil(log)
	return &Artifact{
		log:       log,
		appConfig: appConfig,
		fileutil:  futl,
	}
}

// Download is a generic utility which attempts to download smartly.
func (artifact *Artifact) Download(input DownloadInput) (output DownloadOutput, err error) {
	// parse the url
	var fileURL *url.URL
	fileURL, err = url.Parse(input.SourceURL)
	if err != nil {
		err = fmt.Errorf("url parsing failed. %v", err)
		return
	}

	// create destination directory
	var destinationDir = input.DestinationDirectory
	if destinationDir == "" {
		destinationDir = appconfig.DownloadRoot
	}

	// create directory where artifacts are downloaded.
	err = artifact.fileutil.MakeDirs(destinationDir)
	if err != nil {
		err = fmt.Errorf("failed to create directory=%v, err=%v", destinationDir, err)
		return
	}

	// process if the url is local file or it has already been downloaded.
	var isLocalFile = false
	isLocalFile, _ = artifact.fileutil.LocalFileExist(input.SourceURL)

	if isLocalFile {
		// if local file exist, remove the downloaded artifacts and re-download it again.
		artifact.log.Debugf("source is a local file, start removing existing artifacts. %v", input.SourceURL)
		if err := artifact.fileutil.DeleteFile(input.SourceURL); err != nil {
			artifact.log.Warnf("source is a local file, failed to remove existing local file %v", input.SourceURL)
		} else {
			output.IsUpdated = false
		}
	}
	artifact.log.Debugf("attempt to get the source file as web download. %v", input.SourceURL)
	// compute the local filename which is hash of url_filename
	// Generating a hash_filename will also help against attackers
	// from specifying a directory and filename to overwrite any ami/built-in files.
	urlHash := sha1.Sum([]byte(fileURL.String()))
	output.LocalFilePath = filepath.Join(destinationDir, fmt.Sprintf("%x", urlHash))

	var tempOutput DownloadOutput

	artifact.log.Debugf("Try to download from http/https")
	tempOutput, err = artifact.httpDownload(input.SourceURL, output.LocalFilePath)
	output = tempOutput
	if err != nil {
		return
	}

	isLocalFile, err = artifact.fileutil.LocalFileExist(output.LocalFilePath)
	if isLocalFile {
		output.IsHashMatched, err = artifact.VerifyHash(input, output)
	}

	return
}

// httpDownload attempts to download a file via http/s call
func (artifact *Artifact) httpDownload(fileURL string, destFile string) (output DownloadOutput, err error) {
	artifact.log.Debugf("attempting to download as http/https download %v", destFile)
	eTagFile := destFile + ".etag"
	var request *http.Request
	request, err = http.NewRequest("GET", fileURL, nil)
	if err != nil {
		artifact.log.Errorf("Failed to create http request for artifact download %s", err)
		return
	}
	if artifact.fileutil.Exists(destFile) && artifact.fileutil.Exists(eTagFile) {
		var existingETag string
		existingETag, err = artifact.fileutil.ReadAllText(eTagFile)
		if err != nil {
			artifact.log.Errorf("Fail to read contents from exist file", err)
		}
		request.Header.Add("If-None-Match", existingETag)
	}
	check := http.Client{
		CheckRedirect: network.DisableHTTPDowngrade,
	}

	var resp *http.Response
	resp, err = check.Do(request)
	if err != nil {
		artifact.log.Debug("failed to download from http/https, ", err)
		artifact.fileutil.DeleteFile(destFile)
		artifact.fileutil.DeleteFile(eTagFile)
		return
	}

	if resp.StatusCode == http.StatusNotModified {
		artifact.log.Debug("Unchanged file.")
		output.IsUpdated = false
		output.LocalFilePath = destFile
		return output, nil
	} else if resp.StatusCode != http.StatusOK {
		artifact.log.Debug("failed to download from http/https, ", err)
		artifact.fileutil.DeleteFile(destFile)
		artifact.fileutil.DeleteFile(eTagFile)
		err = fmt.Errorf("http request failed. status:%v statuscode:%v", resp.Status, resp.StatusCode)
		return
	}
	defer resp.Body.Close()
	eTagValue := resp.Header.Get("Etag")
	if eTagValue != "" {
		artifact.log.Debug("file eTagValue is ", eTagValue)
		err = artifact.fileutil.WriteAllText(eTagFile, eTagValue)
		if err != nil {
			artifact.log.Errorf("failed to write eTagfile %v, %v ", eTagFile, err)
			return
		}
	}
	_, err = artifact.fileCopy(destFile, resp.Body)
	if err == nil {
		output.LocalFilePath = destFile
		output.IsUpdated = true
	} else {
		artifact.log.Errorf("failed to write destFile %v, %v ", destFile, err)
	}
	return
}

// FileCopy copies the content from reader to destinationPath file
func (artifact *Artifact) fileCopy(destinationPath string, src io.Reader) (written int64, err error) {

	var file *os.File
	file, err = os.Create(destinationPath)
	if err != nil {
		artifact.log.Errorf("failed to create file. %v", err)
		return
	}
	defer file.Close()
	var size int64
	size, err = io.Copy(file, src)
	artifact.log.Debugf("%s with %v bytes downloaded", destinationPath, size)
	return
}

// VerifyHash verifies the hash of the url file as per specified hash algorithm type and its value
func (artifact *Artifact) VerifyHash(input DownloadInput, output DownloadOutput) (bool, error) {
	hasMatchingHash := false

	// check and set default hashing algorithm
	checksums := input.SourceChecksums

	if len(checksums) == 0 {
		return true, nil
	}

	//backwards compatibility for empty HashValues and HashTypes
	if len(checksums) == 1 {
		for _, hashValue := range checksums {
			// this is the only pair in the map
			if hashValue == "" {
				return true, nil
			}
		}
	}

	for hashAlgorithm, hashValue := range checksums {
		var computedHashValue string
		var err error
		// check the sha256 algorithm by default
		if hashAlgorithm == "" || strings.EqualFold(hashAlgorithm, "sha256") {
			computedHashValue, err = artifact.sha256HashValue(output.LocalFilePath)
		} else if strings.EqualFold(hashAlgorithm, "md5") {
			computedHashValue, err = artifact.md5HashValue(output.LocalFilePath)
		} else {
			continue
		}

		if err != nil {
			return false, fmt.Errorf("the algorithm returned an error when trying to compute the checksum %v", input)
		}

		if !strings.EqualFold(hashValue, computedHashValue) {
			return false, fmt.Errorf("failed to verify hash of downloadinput %v", input)
		}

		hasMatchingHash = true
	}

	//if a supported hash algorithm was not provided, jut return an error
	if !hasMatchingHash {
		return false, fmt.Errorf("no supported algorithm was provided for downloadinput %v", input)
	}

	return true, nil
}

// Sha256HashValue gets the sha256 hash value
func (artifact *Artifact) sha256HashValue(filePath string) (hash string, err error) {
	var exists = false
	exists, err = artifact.fileutil.LocalFileExist(filePath)
	if err != nil || !exists {
		return
	}

	var f *os.File
	f, err = os.Open(filePath)
	if err != nil {
		artifact.log.Error(err)
	}
	defer f.Close()
	hasher := sha256.New()
	if _, err = io.Copy(hasher, f); err != nil {
		artifact.log.Error(err)
	}
	hash = hex.EncodeToString(hasher.Sum(nil))
	artifact.log.Debugf("Hash=%v, FilePath=%v", hash, filePath)
	return
}

// Md5HashValue gets the md5 hash value
func (artifact *Artifact) md5HashValue(filePath string) (hash string, err error) {
	var exists = false
	exists, err = artifact.fileutil.LocalFileExist(filePath)
	if err != nil || !exists {
		return
	}

	var f *os.File
	f, err = os.Open(filePath)
	if err != nil {
		artifact.log.Error(err)
	}
	defer f.Close()
	hasher := md5.New()
	if _, err = io.Copy(hasher, f); err != nil {
		artifact.log.Error(err)
	}
	hash = hex.EncodeToString(hasher.Sum(nil))
	artifact.log.Debugf("Hash=%v, FilePath=%v", hash, filePath)
	return
}

func (artifact *Artifact) Uncompress(src, dest string) error {
	return artifact.fileutil.Uncompress(artifact.log, src, dest)
}