pkg/providers/vsphere/validator.go (577 lines of code) (raw):
package vsphere
import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"net"
"path/filepath"
anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1"
"github.com/aws/eks-anywhere/pkg/collection"
"github.com/aws/eks-anywhere/pkg/config"
"github.com/aws/eks-anywhere/pkg/features"
"github.com/aws/eks-anywhere/pkg/govmomi"
"github.com/aws/eks-anywhere/pkg/logger"
"github.com/aws/eks-anywhere/pkg/types"
)
const (
vsphereRootPath = "/"
)
type PrivAssociation struct {
objectType string
privsContent string
path string
}
type missingPriv struct {
Username string `yaml:"username"`
ObjectType string `yaml:"objectType"`
Path string `yaml:"path"`
Permissions []string `yaml:"permissions"`
}
type VSphereClientBuilder interface {
Build(ctx context.Context, host string, username string, password string, insecure bool, datacenter string) (govmomi.VSphereClient, error)
}
// ResourcePaths defines an interface for objects that contain vSphere resource path information.
type ResourcePaths interface {
ResourcePaths() map[string]string
}
type Validator struct {
govc ProviderGovcClient
vSphereClientBuilder VSphereClientBuilder
}
// NewValidator initializes the client for VSphere provider validations.
func NewValidator(govc ProviderGovcClient, vscb VSphereClientBuilder) *Validator {
return &Validator{
govc: govc,
vSphereClientBuilder: vscb,
}
}
func (v *Validator) validateVCenterAccess(ctx context.Context, server string) error {
if err := v.govc.ValidateVCenterConnection(ctx, server); err != nil {
return fmt.Errorf("failed validating connection to vCenter: %v", err)
}
logger.MarkPass("Connected to server")
if err := v.govc.ValidateVCenterAuthentication(ctx); err != nil {
return fmt.Errorf("failed validating credentials for vCenter: %v", err)
}
logger.MarkPass("Authenticated to vSphere")
return nil
}
func (v *Validator) ValidateVCenterConfig(ctx context.Context, datacenterConfig *anywherev1.VSphereDatacenterConfig) error {
if err := v.validateVCenterAccess(ctx, datacenterConfig.Spec.Server); err != nil {
return err
}
if err := v.validateThumbprint(ctx, datacenterConfig); err != nil {
return err
}
if err := v.validateDatacenter(ctx, datacenterConfig.Spec.Datacenter); err != nil {
return err
}
if err := v.validateNetwork(ctx, datacenterConfig.Spec.Network); err != nil {
return err
}
return nil
}
// ValidateFailureDomains validates the provided list of failure domains.
func (v *Validator) ValidateFailureDomains(ctx context.Context, vsphereClusterSpec *Spec) error {
failureDomains := vsphereClusterSpec.VSphereDatacenter.Spec.FailureDomains
if !features.IsActive(features.VsphereFailureDomainEnabled()) {
if len(failureDomains) > 0 {
return fmt.Errorf("failure domains feature is not enabled. Please set the env variable %v", features.VSphereFailureDomainEnabledEnvVar)
}
return nil
}
providedFailureDomains := collection.MapSet(failureDomains, func(fd anywherev1.FailureDomain) string {
return fd.Name
})
failureDomainsAssigned, err := v.validateWorkerNodeGroupDomains(vsphereClusterSpec, providedFailureDomains)
if err != nil {
return err
}
if !failureDomainsAssigned {
// TODO: Error message here if Failure Domain not being used by workernodegroups?
// Skipping further validation currently
// return fmt.Errorf("failure domain defined, but no worker node group references")
return nil
}
if err := v.validateFailureDomainResources(ctx, vsphereClusterSpec, failureDomains); err != nil {
return err
}
return nil
}
func (v *Validator) validateWorkerNodeGroupDomains(vsphereClusterSpec *Spec, providedFailureDomains collection.Set[string]) (bool, error) {
failureDomainsAssigned := false
for _, wng := range vsphereClusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations {
if len(wng.FailureDomains) == 0 {
continue
}
if len(wng.FailureDomains) > 1 {
return failureDomainsAssigned, fmt.Errorf("multiple failure domains provided in the worker node group: %s. Please provide only one failure domain", wng.Name)
}
assignedFailureDomain := wng.FailureDomains[0]
if !providedFailureDomains.Contains(assignedFailureDomain) {
return failureDomainsAssigned, fmt.Errorf("provided invalid failure domain %s in the worker node group %s", assignedFailureDomain, wng.Name)
}
failureDomainsAssigned = true
}
return failureDomainsAssigned, nil
}
func (v *Validator) validateFailureDomainResources(ctx context.Context, vsphereClusterSpec *Spec, failureDomains []anywherev1.FailureDomain) error {
for index, fd := range failureDomains {
message := fmt.Sprintf("Start failure domain validation for '%s' ", failureDomains[index].Name)
logger.Info(message)
if err := v.validateNetwork(ctx, fd.Network); err != nil {
return err
}
if err := v.govc.ValidateFailureDomainConfig(ctx, vsphereClusterSpec.VSphereDatacenter, &fd); err != nil {
return err
}
}
logger.Info("Finished failure domain validations")
return nil
}
func (v *Validator) validateMachineConfigTagsExist(ctx context.Context, machineConfigs []*anywherev1.VSphereMachineConfig) error {
tags, err := v.govc.ListTags(ctx)
if err != nil {
return fmt.Errorf("failed to check if tags exists in vSphere: %v", err)
}
tagIDs := make([]string, 0, len(tags))
for _, t := range tags {
tagIDs = append(tagIDs, t.Id)
}
idLookup := types.SliceToLookup(tagIDs)
for _, machineConfig := range machineConfigs {
for _, tagID := range machineConfig.Spec.TagIDs {
if !idLookup.IsPresent(tagID) {
return fmt.Errorf("tag (%s) does not exist in vSphere. please provide a valid tag id in the urn format (example: urn:vmomi:InventoryServiceTag:8e0ce079-0677-48d6-8865-19ada4e6dabd:GLOBAL)", tagID)
}
}
}
logger.MarkPass("Machine config tags validated")
return nil
}
// ValidateClusterMachineConfigs validates all the attributes of etcd, control plane, and worker node VSphereMachineConfigs.
func (v *Validator) ValidateClusterMachineConfigs(ctx context.Context, vsphereClusterSpec *Spec) error {
var etcdMachineConfig *anywherev1.VSphereMachineConfig
controlPlaneMachineConfig := vsphereClusterSpec.controlPlaneMachineConfig()
if controlPlaneMachineConfig == nil {
return fmt.Errorf("cannot find VSphereMachineConfig %v for control plane", vsphereClusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
}
for _, workerNodeGroupConfiguration := range vsphereClusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations {
workerNodeGroupMachineConfig := vsphereClusterSpec.workerMachineConfig(workerNodeGroupConfiguration)
if workerNodeGroupMachineConfig == nil {
return fmt.Errorf("cannot find VSphereMachineConfig %v for worker nodes", workerNodeGroupConfiguration.MachineGroupRef.Name)
}
}
if vsphereClusterSpec.Cluster.Spec.ExternalEtcdConfiguration != nil {
etcdMachineConfig = vsphereClusterSpec.etcdMachineConfig()
if etcdMachineConfig == nil {
return fmt.Errorf("cannot find VSphereMachineConfig %v for etcd machines", vsphereClusterSpec.Cluster.Spec.ExternalEtcdConfiguration.MachineGroupRef.Name)
}
if !v.sameOSFamily(vsphereClusterSpec.VSphereMachineConfigs) {
return errors.New("all VSphereMachineConfigs must have the same osFamily specified")
}
if etcdMachineConfig.Spec.HostOSConfiguration != nil && etcdMachineConfig.Spec.HostOSConfiguration.BottlerocketConfiguration != nil && etcdMachineConfig.Spec.HostOSConfiguration.BottlerocketConfiguration.Kubernetes != nil {
logger.Info("Bottlerocket Kubernetes settings are not supported for etcd machines. Ignoring Kubernetes settings for etcd machines.", "etcdMachineConfig", etcdMachineConfig.Name)
}
}
// TODO: move this to api Cluster validations
if err := v.validateControlPlaneIp(vsphereClusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host); err != nil {
return err
}
for _, config := range vsphereClusterSpec.VSphereMachineConfigs {
var b bool // Temporary until we remove the need to pass a bool pointer
err := v.govc.ValidateVCenterSetupMachineConfig(ctx, vsphereClusterSpec.VSphereDatacenter, config, &b) // TODO: remove side effects from this implementation or directly move it to set defaults (pointer to bool is not needed)
if err != nil {
return fmt.Errorf("validating vCenter setup for VSphereMachineConfig %v: %v", config.Name, err)
}
}
if err := v.validateTemplates(ctx, vsphereClusterSpec); err != nil {
return err
}
if err := v.validateMachineConfigTagsExist(ctx, vsphereClusterSpec.machineConfigs()); err != nil {
return err
}
logger.MarkPass("Control plane and Workload templates validated")
for _, mc := range vsphereClusterSpec.VSphereMachineConfigs {
if mc.OSFamily() == anywherev1.Bottlerocket {
if err := v.validateBRHardDiskSize(ctx, vsphereClusterSpec, mc); err != nil {
return fmt.Errorf("failed validating BR Hard Disk size: %v", err)
}
}
}
return nil
}
func (v *Validator) validateControlPlaneIp(ip string) error {
// check if controlPlaneEndpointIp is valid
parsedIp := net.ParseIP(ip)
if parsedIp == nil {
return fmt.Errorf("cluster controlPlaneConfiguration.Endpoint.Host is invalid: %s", ip)
}
return nil
}
func (v *Validator) validateTemplates(ctx context.Context, spec *Spec) error {
tagsForTemplates := make(map[string][]string)
rootVersionsBundle := spec.RootVersionsBundle()
for _, m := range sliceIfNotNil(spec.controlPlaneMachineConfig(), spec.etcdMachineConfig()) {
currentTags := tagsForTemplates[m.Spec.Template]
tagsForTemplates[m.Spec.Template] = append(
currentTags,
requiredTemplateTags(m, rootVersionsBundle)...,
)
}
for _, w := range spec.Cluster.Spec.WorkerNodeGroupConfigurations {
machineConfig := spec.VSphereMachineConfigs[w.MachineGroupRef.Name]
versionsBundle := spec.WorkerNodeGroupVersionsBundle(w)
currentTags := tagsForTemplates[machineConfig.Spec.Template]
tagsForTemplates[machineConfig.Spec.Template] = append(
currentTags,
requiredTemplateTags(machineConfig, versionsBundle)...,
)
}
for template, requiredTags := range tagsForTemplates {
datacenter := spec.VSphereDatacenter.Spec.Datacenter
templatePath, err := v.getTemplatePath(ctx, datacenter, template)
if err != nil {
return err
}
if err := v.validateTemplateTags(ctx, templatePath, requiredTags); err != nil {
return err
}
}
return nil
}
func (v *Validator) getTemplatePath(ctx context.Context, datacenter, templatePath string) (string, error) {
templateFullPath, err := v.govc.SearchTemplate(ctx, datacenter, templatePath)
if err != nil {
return "", fmt.Errorf("validating template: %v", err)
}
if len(templateFullPath) <= 0 {
return "", fmt.Errorf("template <%s> not found. Has the template been imported?", templatePath)
}
return templateFullPath, nil
}
func (v *Validator) validateTemplateTags(ctx context.Context, templatePath string, requiredTags []string) error {
tags, err := v.govc.GetTags(ctx, templatePath)
if err != nil {
return fmt.Errorf("validating template tags: %v", err)
}
tagsLookup := types.SliceToLookup(tags)
for _, t := range requiredTags {
if !tagsLookup.IsPresent(t) {
// TODO: maybe add help text about to how to tag a template?
return fmt.Errorf("template %s is missing tag %s", templatePath, t)
}
}
return nil
}
func (v *Validator) validateBRHardDiskSize(ctx context.Context, spec *Spec, machineConfigSpec *anywherev1.VSphereMachineConfig) error {
dataCenter := spec.Config.VSphereDatacenter.Spec.Datacenter
template := machineConfigSpec.Spec.Template
hardDiskMap, err := v.govc.GetHardDiskSize(ctx, template, dataCenter)
if err != nil {
return fmt.Errorf("validating hard disk size: %v", err)
}
if len(hardDiskMap) == 0 {
return fmt.Errorf("no hard disks found for template: %v", template)
} else if len(hardDiskMap) > 1 {
if hardDiskMap[disk1] != 2097152 { // 2GB in KB to avoid roundoff errors
return fmt.Errorf("Incorrect disk size for disk1 - expected: 2097152 kB got: %v", hardDiskMap[disk1])
} else if hardDiskMap[disk2] != 20971520 { // 20GB in KB to avoid roundoff errors
return fmt.Errorf("Incorrect disk size for disk2 - expected: 20971520 kB got: %v", hardDiskMap[disk2])
}
} else if hardDiskMap[disk1] != 23068672 { // 22GB in KB to avoid roundoff errors
return fmt.Errorf("Incorrect disk size for disk1 - expected: 23068672 kB got: %v", hardDiskMap[disk1])
}
logger.V(5).Info("Bottlerocket Disk size validated: ", "diskMap", hardDiskMap)
return nil
}
func (v *Validator) validateThumbprint(ctx context.Context, datacenterConfig *anywherev1.VSphereDatacenterConfig) error {
// No need to validate thumbprint in insecure mode
if datacenterConfig.Spec.Insecure {
return nil
}
// If cert is not self signed, thumbprint is ignored
if !v.govc.IsCertSelfSigned(ctx) {
return nil
}
if datacenterConfig.Spec.Thumbprint == "" {
return fmt.Errorf("thumbprint is required for secure mode with self-signed certificates")
}
thumbprint, err := v.govc.GetCertThumbprint(ctx)
if err != nil {
return err
}
if thumbprint != datacenterConfig.Spec.Thumbprint {
return fmt.Errorf("thumbprint mismatch detected, expected: %s, actual: %s", datacenterConfig.Spec.Thumbprint, thumbprint)
}
return nil
}
func (v *Validator) validateDatacenter(ctx context.Context, datacenter string) error {
exists, err := v.govc.DatacenterExists(ctx, datacenter)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("datacenter %s not found", datacenter)
}
logger.MarkPass("Datacenter validated")
return nil
}
func (v *Validator) validateNetwork(ctx context.Context, network string) error {
exists, err := v.govc.NetworkExists(ctx, network)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("network %s not found", network)
}
logger.MarkPass("Network validated")
return nil
}
func (v *Validator) collectResourcePathConfig(_ context.Context, spec *Spec) ([]ResourcePaths, error) {
resourcePaths := []ResourcePaths{}
controlPlaneMachineConfig := spec.controlPlaneMachineConfig()
resourcePaths = append(resourcePaths, controlPlaneMachineConfig)
for _, workerNodeGroupConfiguration := range spec.Cluster.Spec.WorkerNodeGroupConfigurations {
workerNodeGroupMachineConfig := spec.workerMachineConfig(workerNodeGroupConfiguration)
resourcePaths = append(resourcePaths, workerNodeGroupMachineConfig)
}
if spec.Cluster.Spec.ExternalEtcdConfiguration != nil {
etcdMachineConfig := spec.etcdMachineConfig()
resourcePaths = append(resourcePaths, etcdMachineConfig)
}
if spec.VSphereDatacenter.Spec.FailureDomains != nil {
for _, failureDomain := range spec.VSphereDatacenter.Spec.FailureDomains {
resourcePaths = append(resourcePaths, &failureDomain)
}
}
return resourcePaths, nil
}
func (v *Validator) validateVsphereUserPrivs(ctx context.Context, vSphereClusterSpec *Spec) error {
var passed bool
var err error
vuc := config.NewVsphereUserConfig()
if passed, err = v.validateUserPrivs(ctx, vSphereClusterSpec, vuc); err != nil {
return err
}
markPrivsValidationPass(passed, vuc.EksaVsphereUsername)
if len(vuc.EksaVsphereCPUsername) > 0 && vuc.EksaVsphereCPUsername != vuc.EksaVsphereUsername {
if passed, err = v.validateCPUserPrivs(ctx, vSphereClusterSpec, vuc); err != nil {
return err
}
markPrivsValidationPass(passed, vuc.EksaVsphereCPUsername)
}
return nil
}
func markPrivsValidationPass(passed bool, username string) {
if passed {
s := fmt.Sprintf("%s user vSphere privileges validated", username)
logger.MarkPass(s)
}
}
func (v *Validator) validateUserPrivs(ctx context.Context, spec *Spec, vuc *config.VSphereUserConfig) (bool, error) {
resourcePaths, err := v.collectResourcePathConfig(ctx, spec)
if err != nil {
return false, err
}
requiredPrivAssociations := []PrivAssociation{
// validate global root priv settings are correct
{
objectType: govmomi.VSphereTypeFolder,
privsContent: config.VSphereGlobalPrivsFile,
path: vsphereRootPath,
},
{
objectType: govmomi.VSphereTypeNetwork,
privsContent: config.VSphereUserPrivsFile,
path: spec.VSphereDatacenter.Spec.Network,
},
}
seen := map[string]interface{}{}
for _, mc := range resourcePaths {
datastore := mc.ResourcePaths()["datastore"]
if _, ok := seen[datastore]; !ok {
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeDatastore,
privsContent: config.VSphereUserPrivsFile,
path: datastore,
},
)
seen[datastore] = 1
}
resourcePool := mc.ResourcePaths()["resourcePool"]
if _, ok := seen[resourcePool]; !ok {
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeResourcePool,
privsContent: config.VSphereUserPrivsFile,
path: resourcePool,
})
seen[resourcePool] = 1
}
folder := mc.ResourcePaths()["folder"]
if _, ok := seen[folder]; !ok {
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeFolder,
privsContent: config.VSphereAdminPrivsFile,
path: folder,
})
seen[folder] = 1
}
switch v := mc.(type) {
case *anywherev1.FailureDomain:
computecluster := v.ResourcePaths()["computeCluster"]
if _, ok := seen[computecluster]; !ok {
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeComputeCluster,
privsContent: config.VSphereAdminPrivsFile,
path: computecluster,
})
seen[computecluster] = 1
}
case *anywherev1.VSphereMachineConfig:
template := v.ResourcePaths()["template"]
if _, ok := seen[template]; !ok {
// ToDo: add more sophisticated validation around a scenario where someone has uploaded templates
// on their own and does not want to allow EKSA user write access to templates
// Verify privs on the template
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeVirtualMachine,
privsContent: config.VSphereAdminPrivsFile,
path: template,
})
seen[template] = 1
}
if _, ok := seen[filepath.Dir(template)]; !ok {
requiredPrivAssociations = append(requiredPrivAssociations, PrivAssociation{
objectType: govmomi.VSphereTypeFolder,
privsContent: config.VSphereAdminPrivsFile,
path: filepath.Dir(template),
})
seen[filepath.Dir(template)] = 1
}
default:
return false, errors.New("unexpected type in missing validateUserPrivs")
}
}
host := spec.VSphereDatacenter.Spec.Server
datacenter := spec.VSphereDatacenter.Spec.Datacenter
vsc, err := v.vSphereClientBuilder.Build(
ctx,
host,
vuc.EksaVsphereUsername,
vuc.EksaVspherePassword,
spec.VSphereDatacenter.Spec.Insecure,
datacenter,
)
if err != nil {
return false, err
}
return v.validatePrivs(ctx, requiredPrivAssociations, vsc)
}
func (v *Validator) validateCPUserPrivs(ctx context.Context, spec *Spec, vuc *config.VSphereUserConfig) (bool, error) {
// CP role just needs read only
privObjs := []PrivAssociation{
{
objectType: govmomi.VSphereTypeFolder,
privsContent: config.VSphereReadOnlyPrivs,
path: vsphereRootPath,
},
}
host := spec.VSphereDatacenter.Spec.Server
datacenter := spec.VSphereDatacenter.Spec.Datacenter
vsc, err := v.vSphereClientBuilder.Build(
ctx,
host,
vuc.EksaVsphereCPUsername,
vuc.EksaVsphereCPPassword,
spec.VSphereDatacenter.Spec.Insecure,
datacenter,
)
if err != nil {
return false, err
}
return v.validatePrivs(ctx, privObjs, vsc)
}
func (v *Validator) validatePrivs(ctx context.Context, privObjs []PrivAssociation, vsc govmomi.VSphereClient) (bool, error) {
var privs []string
var err error
missingPrivs := []missingPriv{}
passed := false
username := vsc.Username()
for _, obj := range privObjs {
path := obj.path
privsContent := obj.privsContent
t := obj.objectType
privs, err = v.getMissingPrivs(ctx, vsc, path, t, privsContent, username)
if err != nil {
return passed, fmt.Errorf("failed to get missing privileges: %v", err)
} else if len(privs) > 0 {
mp := missingPriv{
Username: username,
ObjectType: t,
Path: path,
Permissions: privs,
}
missingPrivs = append(missingPrivs, mp)
}
}
if len(missingPrivs) != 0 {
_, err := json.Marshal(missingPrivs)
if err != nil {
return passed, fmt.Errorf("failed to marshal missing permissions: %v", err)
}
errMsg := fmt.Sprintf("user %s missing vSphere permissions", username)
logger.V(3).Info(errMsg, "Missing Permissions (JSON)", missingPrivs)
return passed, fmt.Errorf("user %s missing vSphere permissions", username)
}
passed = true
return passed, nil
}
func checkRequiredPrivs(requiredPrivs []string, hasPrivs []string) []string {
hp := map[string]interface{}{}
for _, val := range hasPrivs {
hp[val] = 1
}
missingPrivs := []string{}
for _, p := range requiredPrivs {
if _, ok := hp[p]; !ok {
missingPrivs = append(missingPrivs, p)
}
}
return missingPrivs
}
func (v *Validator) getMissingPrivs(ctx context.Context, vsc govmomi.VSphereClient, path string, objType string, requiredPrivsContent string, username string) ([]string, error) {
var requiredPrivs []string
err := json.Unmarshal([]byte(requiredPrivsContent), &requiredPrivs)
if err != nil {
return nil, err
}
hasPrivs, err := vsc.GetPrivsOnEntity(ctx, path, objType, username)
if err != nil {
return nil, err
}
missingPrivs := checkRequiredPrivs(requiredPrivs, hasPrivs)
return missingPrivs, nil
}
func (v *Validator) sameOSFamily(configs map[string]*anywherev1.VSphereMachineConfig) bool {
c := getRandomMachineConfig(configs)
osFamily := c.Spec.OSFamily
for _, machineConfig := range configs {
if machineConfig.Spec.OSFamily != osFamily {
return false
}
}
return true
}
func (v *Validator) sameTemplate(configs map[string]*anywherev1.VSphereMachineConfig) bool {
c := getRandomMachineConfig(configs)
template := c.Spec.Template
for _, machineConfig := range configs {
if machineConfig.Spec.Template != template {
return false
}
}
return true
}
func getRandomMachineConfig(configs map[string]*anywherev1.VSphereMachineConfig) *anywherev1.VSphereMachineConfig {
var machineConfig *anywherev1.VSphereMachineConfig
for _, c := range configs {
machineConfig = c
break
}
return machineConfig
}
func sliceIfNotNil(machines ...*anywherev1.VSphereMachineConfig) []*anywherev1.VSphereMachineConfig {
var notNil []*anywherev1.VSphereMachineConfig
for _, m := range machines {
if m != nil {
notNil = append(notNil, m)
}
}
return notNil
}