in cli/src/pcluster/templates/cluster_stack.py [0:0]
def _add_head_node(self):
head_node = self.config.head_node
head_lt_security_groups = self._get_head_node_security_groups_full()
# LT network interfaces
head_lt_nw_interfaces = [
ec2.CfnLaunchTemplate.NetworkInterfaceProperty(
device_index=0,
network_interface_id=self._head_eni.ref,
)
]
for device_index in range(1, head_node.max_network_interface_count):
head_lt_nw_interfaces.append(
ec2.CfnLaunchTemplate.NetworkInterfaceProperty(
device_index=device_index,
network_card_index=device_index,
groups=head_lt_security_groups,
subnet_id=head_node.networking.subnet_id,
)
)
# Head node Launch Template
head_node_launch_template = ec2.CfnLaunchTemplate(
self,
"HeadNodeLaunchTemplate",
launch_template_data=ec2.CfnLaunchTemplate.LaunchTemplateDataProperty(
instance_type=head_node.instance_type,
cpu_options=ec2.CfnLaunchTemplate.CpuOptionsProperty(core_count=head_node.vcpus, threads_per_core=1)
if head_node.pass_cpu_options_in_launch_template
else None,
block_device_mappings=get_block_device_mappings(head_node.local_storage, self.config.image.os),
key_name=head_node.ssh.key_name,
network_interfaces=head_lt_nw_interfaces,
image_id=self.config.head_node_ami,
ebs_optimized=head_node.is_ebs_optimized,
iam_instance_profile=ec2.CfnLaunchTemplate.IamInstanceProfileProperty(
name=self._head_node_instance_profile
),
user_data=Fn.base64(
Fn.sub(
get_user_data_content("../resources/head_node/user_data.sh"),
{**get_common_user_data_env(head_node, self.config)},
)
),
tag_specifications=[
ec2.CfnLaunchTemplate.TagSpecificationProperty(
resource_type="instance",
tags=get_default_instance_tags(
self._stack_name, self.config, head_node, "HeadNode", self.shared_storage_mappings
)
+ get_custom_tags(self.config),
),
ec2.CfnLaunchTemplate.TagSpecificationProperty(
resource_type="volume",
tags=get_default_volume_tags(self._stack_name, "HeadNode") + get_custom_tags(self.config),
),
],
),
)
# Metadata
head_node_launch_template.add_metadata("Comment", "AWS ParallelCluster Head Node")
# CloudFormation::Init metadata
pre_install_action, post_install_action = (None, None)
if head_node.custom_actions:
pre_install_action = head_node.custom_actions.on_node_start
post_install_action = head_node.custom_actions.on_node_configured
dna_json = json.dumps(
{
"cluster": {
"stack_name": self._stack_name,
"stack_arn": self.stack_id,
"scheduler_plugin_substack_arn": self.scheduler_plugin_stack.ref
if self.scheduler_plugin_stack
else "",
"raid_vol_ids": get_shared_storage_ids_by_type(
self.shared_storage_mappings, SharedStorageType.RAID
),
"raid_parameters": get_shared_storage_options_by_type(
self.shared_storage_options, SharedStorageType.RAID
),
"disable_hyperthreading_manually": "true"
if head_node.disable_simultaneous_multithreading_manually
else "false",
"base_os": self.config.image.os,
"preinstall": pre_install_action.script if pre_install_action else "NONE",
"preinstall_args": join_shell_args(pre_install_action.args)
if pre_install_action and pre_install_action.args
else "NONE",
"postinstall": post_install_action.script if post_install_action else "NONE",
"postinstall_args": join_shell_args(post_install_action.args)
if post_install_action and post_install_action.args
else "NONE",
"region": self.region,
"efs_fs_id": get_shared_storage_ids_by_type(self.shared_storage_mappings, SharedStorageType.EFS),
"efs_shared_dir": get_shared_storage_options_by_type(
self.shared_storage_options, SharedStorageType.EFS
), # FIXME
"fsx_fs_id": get_shared_storage_ids_by_type(self.shared_storage_mappings, SharedStorageType.FSX),
"fsx_mount_name": self.shared_storage_attributes[SharedStorageType.FSX].get("MountName", ""),
"fsx_dns_name": self.shared_storage_attributes[SharedStorageType.FSX].get("DNSName", ""),
"fsx_options": get_shared_storage_options_by_type(
self.shared_storage_options, SharedStorageType.FSX
),
"volume": get_shared_storage_ids_by_type(self.shared_storage_mappings, SharedStorageType.EBS),
"scheduler": self.config.scheduling.scheduler,
"ephemeral_dir": head_node.local_storage.ephemeral_volume.mount_dir
if head_node.local_storage and head_node.local_storage.ephemeral_volume
else "/scratch",
"ebs_shared_dirs": get_shared_storage_options_by_type(
self.shared_storage_options, SharedStorageType.EBS
),
"proxy": head_node.networking.proxy.http_proxy_address if head_node.networking.proxy else "NONE",
"node_type": "HeadNode",
"cluster_user": OS_MAPPING[self.config.image.os]["user"],
"log_group_name": self.log_group.log_group_name
if self.config.monitoring.logs.cloud_watch.enabled
else "NONE",
"dcv_enabled": "head_node" if self.config.is_dcv_enabled else "false",
"dcv_port": head_node.dcv.port if head_node.dcv else "NONE",
"enable_intel_hpc_platform": "true" if self.config.is_intel_hpc_platform_enabled else "false",
"cw_logging_enabled": "true" if self.config.is_cw_logging_enabled else "false",
"cluster_s3_bucket": self.bucket.name,
"cluster_config_s3_key": "{0}/configs/{1}".format(
self.bucket.artifact_directory, PCLUSTER_S3_ARTIFACTS_DICT.get("config_name")
),
"cluster_config_version": self.config.config_version,
"instance_types_data_s3_key": f"{self.bucket.artifact_directory}/configs/instance-types-data.json",
"custom_node_package": self.config.custom_node_package or "",
"custom_awsbatchcli_package": self.config.custom_aws_batch_cli_package or "",
"head_node_imds_secured": str(self.config.head_node.imds.secured).lower(),
**(
get_slurm_specific_dna_json_for_head_node(self.config, self.scheduler_resources)
if self._condition_is_slurm()
else {}
),
**get_directory_service_dna_json_for_head_node(self.config),
},
},
indent=4,
)
cfn_init = {
"configSets": {
"deployFiles": ["deployConfigFiles"],
"default": [
"cfnHupConfig",
"chefPrepEnv",
"shellRunPreInstall",
"chefConfig",
"shellRunPostInstall",
"chefFinalize",
],
"update": ["deployConfigFiles", "chefUpdate"],
},
"deployConfigFiles": {
"files": {
"/tmp/dna.json": { # nosec
"content": dna_json,
"mode": "000644",
"owner": "root",
"group": "root",
"encoding": "plain",
},
"/etc/chef/client.rb": {
"mode": "000644",
"owner": "root",
"group": "root",
"content": "cookbook_path ['/etc/chef/cookbooks']",
},
"/tmp/extra.json": { # nosec
"mode": "000644",
"owner": "root",
"group": "root",
"content": self.config.extra_chef_attributes,
},
"/tmp/wait_condition_handle.txt": { # nosec
"mode": "000644",
"owner": "root",
"group": "root",
"content": self.wait_condition_handle.ref,
},
},
"commands": {
"mkdir": {"command": "mkdir -p /etc/chef/ohai/hints"},
"touch": {"command": "touch /etc/chef/ohai/hints/ec2.json"},
"jq": {
"command": (
"jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json -n '$f1 * $f2' "
"> /etc/chef/dna.json "
'|| ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json )'
)
},
},
},
"cfnHupConfig": {
"files": {
"/etc/cfn/hooks.d/parallelcluster-update.conf": {
"content": Fn.sub(
(
"[parallelcluster-update]\n"
"triggers=post.update\n"
"path=Resources.HeadNodeLaunchTemplate.Metadata.AWS::CloudFormation::Init\n"
"action=PATH=/usr/local/bin:/bin:/usr/bin:/opt/aws/bin; "
"cfn-init -v --stack ${StackName} "
"--resource HeadNodeLaunchTemplate --configsets update --region ${Region}\n"
"runas=root\n"
),
{"StackName": self._stack_name, "Region": self.region},
),
"mode": "000400",
"owner": "root",
"group": "root",
},
"/etc/cfn/cfn-hup.conf": {
"content": Fn.sub(
"[main]\nstack=${StackId}\nregion=${Region}\ninterval=2",
{"StackId": self.stack_id, "Region": self.region},
),
"mode": "000400",
"owner": "root",
"group": "root",
},
}
},
"chefPrepEnv": {
"commands": {
"chef": {
"command": (
"cinc-client --local-mode --config /etc/chef/client.rb --log_level info "
"--logfile /var/log/chef-client.log --force-formatter --no-color "
"--chef-zero-port 8889 --json-attributes /etc/chef/dna.json "
"--override-runlist aws-parallelcluster::init"
),
"cwd": "/etc/chef",
}
}
},
"shellRunPreInstall": {
"commands": {"runpreinstall": {"command": "/opt/parallelcluster/scripts/fetch_and_run -preinstall"}}
},
"chefConfig": {
"commands": {
"chef": {
"command": (
"cinc-client --local-mode --config /etc/chef/client.rb --log_level info "
"--logfile /var/log/chef-client.log --force-formatter --no-color "
"--chef-zero-port 8889 --json-attributes /etc/chef/dna.json "
"--override-runlist aws-parallelcluster::config"
),
"cwd": "/etc/chef",
}
}
},
"shellRunPostInstall": {
"commands": {"runpostinstall": {"command": "/opt/parallelcluster/scripts/fetch_and_run -postinstall"}}
},
"chefFinalize": {
"commands": {
"chef": {
"command": (
"cinc-client --local-mode --config /etc/chef/client.rb --log_level info "
"--logfile /var/log/chef-client.log --force-formatter --no-color "
"--chef-zero-port 8889 --json-attributes /etc/chef/dna.json "
"--override-runlist aws-parallelcluster::finalize"
),
"cwd": "/etc/chef",
},
"bootstrap": {
"command": (
"[ ! -f /opt/parallelcluster/.bootstrapped ] && echo ${cookbook_version} "
"| tee /opt/parallelcluster/.bootstrapped || exit 0"
) # TODO check
},
}
},
"chefUpdate": {
"commands": {
"chef": {
"command": (
"cinc-client --local-mode --config /etc/chef/client.rb --log_level info"
" --logfile /var/log/chef-client.log --force-formatter --no-color"
" --chef-zero-port 8889 --json-attributes /etc/chef/dna.json"
" --override-runlist aws-parallelcluster::update &&"
" cfn-signal --exit-code=0 --reason='Update complete'"
f" '{self.wait_condition_handle.ref}' ||"
" cfn-signal --exit-code=1 --reason='Update failed'"
f" '{self.wait_condition_handle.ref}'"
),
"cwd": "/etc/chef",
}
}
},
}
if not self._condition_is_batch():
cfn_init["deployConfigFiles"]["files"]["/opt/parallelcluster/shared/launch-templates-config.json"] = {
"mode": "000644",
"owner": "root",
"group": "root",
"content": self._get_launch_templates_config(),
}
head_node_launch_template.add_metadata("AWS::CloudFormation::Init", cfn_init)
head_node_instance = ec2.CfnInstance(
self,
"HeadNode",
launch_template=ec2.CfnInstance.LaunchTemplateSpecificationProperty(
launch_template_id=head_node_launch_template.ref,
version=head_node_launch_template.attr_latest_version_number,
),
)
if not self._condition_is_batch():
head_node_instance.node.add_dependency(self.compute_fleet_resources)
if self._condition_is_scheduler_plugin() and self.scheduler_plugin_stack:
head_node_instance.add_depends_on(self.scheduler_plugin_stack)
return head_node_instance