in services/jenkins-autoscaling/lambda_mxnet_ci/autoscaling/handler.py [0:0]
def _unconnected_instances(nodes: list, instance_uptime: Dict[str, int], ec2_resource) \
-> Dict[str, List[str]]:
"""
Returns instances which are currently starting up but not connected yet
:return: Dict, mapping label to List of slave names that are still starting up
"""
dict_starting_nodes: Dict[str, List[str]] = defaultdict(list)
instances_filter = ec2_resource.instances.filter(
Filters=[
{'Name': 'instance-state-name', 'Values': ['pending', 'running']},
{'Name': 'tag:AutoScaledSlave', 'Values': ['True']} # Ensure only listing instances managed by auto scaling
])
for instance in instances_filter:
tags = _ec2Instance_tag_dict(instance)
target_node = _find_node_by_name(nodes=nodes, name=tags['Name'])
if target_node and target_node['offline'] and (not target_node['temporarilyOffline']):
logging.debug('Instance %s starting up but not connected yet', target_node['displayName'])
if 'label' in tags:
label = tags['label']
uptime_seconds = instance_uptime[target_node['displayName']]
logging.info('Instance %s - %s of type %s is starting up for %d seconds.',
instance.id, tags['Name'], label, uptime_seconds)
dict_starting_nodes[label].append(tags['Name'])
else: # pragma: no cover
logging.error("Managed slave instance %s does not have tag label", instance.id)
elif not target_node:
logging.error("Found orphaned / zombie instance: '%s'", instance.id)
if 'label' in tags:
label = tags['label']
dict_starting_nodes[label].append(tags['Name'])
else: # pragma: no cover
logging.error("Managed slave instance %s does not have tag label", instance.id)
return dict_starting_nodes