in pkg/deploy/generator/resources_rp.go [356:631]
func (g *generator) rpVMSS() *arm.Resource {
// TODO: there is a lot of duplication with gatewayVMSS() (and other places)
parts := []string{
fmt.Sprintf("base64ToString('%s')", base64.StdEncoding.EncodeToString([]byte("set -ex\n\n"))),
}
for _, variable := range []string{
"acrResourceId",
"adminApiClientCertCommonName",
"armApiClientCertCommonName",
"armClientId",
"azureCloudName",
"azureSecPackQualysUrl",
"azureSecPackVSATenantId",
"clusterMdmAccount",
"clusterMdsdAccount",
"clusterMdsdConfigVersion",
"clusterMdsdNamespace",
"clusterParentDomainName",
"databaseAccountName",
"fluentbitImage",
"fpClientId",
"fpTenantId",
"fpServicePrincipalId",
"gatewayDomains",
"gatewayResourceGroupName",
"gatewayServicePrincipalId",
"keyvaultDNSSuffix",
"keyvaultPrefix",
"mdmFrontendUrl",
"mdsdEnvironment",
"msiRpEndpoint",
"portalAccessGroupIds",
"portalClientId",
"portalElevatedGroupIds",
"rpFeatures",
"rpImage",
"rpMdmAccount",
"rpMdsdAccount",
"rpMdsdConfigVersion",
"rpMdsdNamespace",
"rpParentDomainName",
"oidcStorageAccountName",
"otelAuditQueueSize",
// TODO: Replace with Live Service Configuration in KeyVault
"clustersInstallViaHive",
"clustersAdoptByHive",
"clusterDefaultInstallerPullspec",
} {
parts = append(parts,
fmt.Sprintf("'%s=$(base64 -d <<<'''", strings.ToUpper(variable)),
fmt.Sprintf("base64(parameters('%s'))", variable),
"''')\n'",
)
}
// convert array variables to string using ARM string() function to be passed via customScript later
for _, variable := range []string{
"miseValidAudiences",
"miseValidAppIDs",
} {
parts = append(parts,
fmt.Sprintf("'%s=$(base64 -d <<<'''", strings.ToUpper(variable)),
fmt.Sprintf("base64(string(parameters('%s')))", variable),
"''')\n'",
)
}
for _, variable := range []string{
"adminApiCaBundle",
"armApiCaBundle",
} {
parts = append(parts,
fmt.Sprintf("'%s='''", strings.ToUpper(variable)),
fmt.Sprintf("parameters('%s')", variable),
"'''\n'",
)
}
parts = append(parts,
"'MDMIMAGE=''"+version.MdmImage("")+"''\n'",
)
parts = append(parts,
"'OTELIMAGE=''"+version.OTelImage("")+"''\n'",
)
parts = append(parts,
"'MISEIMAGE=''"+version.MiseImage("")+"''\n'",
)
parts = append(parts,
"'LOCATION=$(base64 -d <<<'''",
"base64(resourceGroup().location)",
"''')\n'",
)
parts = append(parts,
"'SUBSCRIPTIONID=$(base64 -d <<<'''",
"base64(subscription().subscriptionId)",
"''')\n'",
)
parts = append(parts,
"'RESOURCEGROUPNAME=$(base64 -d <<<'''",
"base64(resourceGroup().name)",
"''')\n'",
)
// VMSS extensions only support one custom script
// Because of this, the util-*.sh scripts are prefixed to the bootstrapping script
// main is called at the end of the bootstrapping script, so appending them will not work
bootstrapScript := scriptUtilCommon +
scriptUtilPackages +
scriptUtilServices +
scriptUtilSystem +
scriptRpVMSS
trailer := base64.StdEncoding.EncodeToString([]byte(bootstrapScript))
parts = append(parts, "'\n'", fmt.Sprintf("base64ToString('%s')", trailer))
customScript := fmt.Sprintf("[base64(concat(%s))]", strings.Join(parts, ","))
return &arm.Resource{
Resource: &mgmtcompute.VirtualMachineScaleSet{
Sku: &mgmtcompute.Sku{
Name: to.StringPtr("[parameters('vmSize')]"),
Tier: to.StringPtr("Standard"),
Capacity: to.Int64Ptr(1338),
},
Tags: map[string]*string{},
VirtualMachineScaleSetProperties: &mgmtcompute.VirtualMachineScaleSetProperties{
// Reference: https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-automatic-upgrade#arm-templates
UpgradePolicy: &mgmtcompute.UpgradePolicy{
Mode: mgmtcompute.UpgradeModeAutomatic,
RollingUpgradePolicy: &mgmtcompute.RollingUpgradePolicy{
// Percentage equates to 1.02 instances out of 3
MaxBatchInstancePercent: to.Int32Ptr(34),
MaxUnhealthyInstancePercent: to.Int32Ptr(34),
MaxUnhealthyUpgradedInstancePercent: to.Int32Ptr(34),
PauseTimeBetweenBatches: to.StringPtr("PT10M"),
},
AutomaticOSUpgradePolicy: &mgmtcompute.AutomaticOSUpgradePolicy{
EnableAutomaticOSUpgrade: to.BoolPtr(true),
},
},
VirtualMachineProfile: &mgmtcompute.VirtualMachineScaleSetVMProfile{
OsProfile: &mgmtcompute.VirtualMachineScaleSetOSProfile{
ComputerNamePrefix: to.StringPtr("[concat('rp-', parameters('vmssName'), '-')]"),
AdminUsername: to.StringPtr("cloud-user"),
LinuxConfiguration: &mgmtcompute.LinuxConfiguration{
DisablePasswordAuthentication: to.BoolPtr(true),
SSH: &mgmtcompute.SSHConfiguration{
PublicKeys: &[]mgmtcompute.SSHPublicKey{
{
Path: to.StringPtr("/home/cloud-user/.ssh/authorized_keys"),
KeyData: to.StringPtr("[parameters('sshPublicKey')]"),
},
},
},
},
},
StorageProfile: &mgmtcompute.VirtualMachineScaleSetStorageProfile{
// https://eng.ms/docs/products/azure-linux/gettingstarted/azurevm/azurevm
ImageReference: &mgmtcompute.ImageReference{
// cbl-mariner-2-gen2-fips is not supported by Automatic OS Updates
// therefore the non fips image is used, and fips is configured manually
// Reference: https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-automatic-upgrade
// https://eng.ms/docs/cloud-ai-platform/azure-core/azure-compute/compute-platform-arunki/azure-compute-artifacts/azure-compute-artifacts-docs/project-standard/1pgalleryusageinstructions#vmss-deployment-with-1p-image-galleryarm-template
// https://eng.ms/docs/cloud-ai-platform/azure-core/core-compute-and-host/compute-platform-arunki/azure-compute-artifacts/azure-compute-artifacts-docs/project-standard/1pgalleryimagereference#cbl-mariner-2-images
SharedGalleryImageID: to.StringPtr("/sharedGalleries/CblMariner.1P/images/cbl-mariner-2-gen2/versions/latest"),
},
OsDisk: &mgmtcompute.VirtualMachineScaleSetOSDisk{
CreateOption: mgmtcompute.DiskCreateOptionTypesFromImage,
ManagedDisk: &mgmtcompute.VirtualMachineScaleSetManagedDiskParameters{
StorageAccountType: mgmtcompute.StorageAccountTypesPremiumLRS,
},
DiskSizeGB: to.Int32Ptr(1024),
},
},
NetworkProfile: &mgmtcompute.VirtualMachineScaleSetNetworkProfile{
HealthProbe: &mgmtcompute.APIEntityReference{
ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/probes', 'rp-lb', 'rp-probe')]"),
},
NetworkInterfaceConfigurations: &[]mgmtcompute.VirtualMachineScaleSetNetworkConfiguration{
{
Name: to.StringPtr("rp-vmss-nic"),
VirtualMachineScaleSetNetworkConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetNetworkConfigurationProperties{
Primary: to.BoolPtr(true),
IPConfigurations: &[]mgmtcompute.VirtualMachineScaleSetIPConfiguration{
{
Name: to.StringPtr("rp-vmss-ipconfig"),
VirtualMachineScaleSetIPConfigurationProperties: &mgmtcompute.VirtualMachineScaleSetIPConfigurationProperties{
Subnet: &mgmtcompute.APIEntityReference{
ID: to.StringPtr("[resourceId('Microsoft.Network/virtualNetworks/subnets', 'rp-vnet', 'rp-subnet')]"),
},
Primary: to.BoolPtr(true),
PublicIPAddressConfiguration: &mgmtcompute.VirtualMachineScaleSetPublicIPAddressConfiguration{
Name: to.StringPtr("rp-vmss-pip"),
},
LoadBalancerBackendAddressPools: &[]mgmtcompute.SubResource{
{
ID: to.StringPtr("[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', 'rp-lb', 'rp-backend')]"),
},
},
},
},
},
},
},
},
},
ExtensionProfile: &mgmtcompute.VirtualMachineScaleSetExtensionProfile{
Extensions: &[]mgmtcompute.VirtualMachineScaleSetExtension{
{
Name: to.StringPtr("rp-vmss-cse"),
VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.Azure.Extensions"),
Type: to.StringPtr("CustomScript"),
TypeHandlerVersion: to.StringPtr("2.0"),
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"script": customScript,
},
},
},
{
// az-secmonitor package no longer needs to be manually installed
// References:
// https://eng.ms/docs/products/azure-linux/gettingstarted/aks/monitoring
// https://msazure.visualstudio.com/ASMDocs/_wiki/wikis/ASMDocs.wiki/179541/Linux-AzSecPack-AutoConfig-Onboarding-(manual-for-C-AI)?anchor=3.1.1-using-arm-template-resource-elements
Name: to.StringPtr("AzureMonitorLinuxAgent"),
VirtualMachineScaleSetExtensionProperties: &mgmtcompute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.Azure.Monitor"),
EnableAutomaticUpgrade: to.BoolPtr(true),
AutoUpgradeMinorVersion: to.BoolPtr(true),
TypeHandlerVersion: to.StringPtr("1.0"),
Type: to.StringPtr("AzureMonitorLinuxAgent"),
Settings: map[string]interface{}{
"GCS_AUTO_CONFIG": true,
},
},
},
},
},
DiagnosticsProfile: &mgmtcompute.DiagnosticsProfile{
BootDiagnostics: &mgmtcompute.BootDiagnostics{
Enabled: to.BoolPtr(true),
},
},
SecurityProfile: &mgmtcompute.SecurityProfile{
// Required for 1P Image Gallery Use
// https://eng.ms/docs/cloud-ai-platform/azure-core/azure-compute/compute-platform-arunki/azure-compute-artifacts/azure-compute-artifacts-docs/project-standard/1pgalleryusageinstructions#enable-trusted-launch-for-vmss
SecurityType: mgmtcompute.SecurityTypesTrustedLaunch,
},
},
Overprovision: to.BoolPtr(false),
},
Identity: &mgmtcompute.VirtualMachineScaleSetIdentity{
Type: mgmtcompute.ResourceIdentityTypeUserAssigned,
UserAssignedIdentities: map[string]*mgmtcompute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', concat('aro-rp-', resourceGroup().location))]": {},
},
},
Name: to.StringPtr("[concat('rp-vmss-', parameters('vmssName'))]"),
Type: to.StringPtr("Microsoft.Compute/virtualMachineScaleSets"),
Location: to.StringPtr("[resourceGroup().location]"),
},
APIVersion: azureclient.APIVersion("Microsoft.Compute"),
DependsOn: []string{
"[resourceId('Microsoft.Authorization/roleAssignments', guid(resourceGroup().id, parameters('rpServicePrincipalId'), 'RP / Reader'))]",
"[resourceId('Microsoft.Network/loadBalancers', 'rp-lb')]",
},
}
}