in source/Tools Integration/CE-Integration/CE-automation-scripts/3-Verify-instance-status.py [0:0]
def verify_instance_status(InstanceList, serverlist, token, access_key_id, secret_access_key, region_ids):
print("")
auth = {"Authorization": token}
found_in_region = None
instanceIds = []
for instance in InstanceList:
instanceIds.append(instance['InstanceId'])
for region_id in region_ids:
ec2_client = boto3.client('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, region_name=region_id)
try:
resp = ec2_client.describe_instance_status(InstanceIds=instanceIds, IncludeAllInstances=True)
found_in_region = region_id
print("INFO: Instance(s) found in region: " + found_in_region)
break
except:
pass
if found_in_region is not None:
ec2_client = boto3.client('ec2', aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, region_name=found_in_region)
else:
print("ERROR: Instances not found in Replication Regions.")
print("")
instance_not_ready = True
count = 0
while instance_not_ready:
instance_not_ready = False
instance_stopped = True
while instance_stopped:
resp = ec2_client.describe_instance_status(InstanceIds=instanceIds, IncludeAllInstances=True)
instance_stopped_list = []
instance_stopped = False
for instance in InstanceList:
for status in resp['InstanceStatuses']:
if status['InstanceState']['Name'] == "running":
if instance['InstanceId'] == status['InstanceId']:
if status['InstanceStatus']['Status'].lower() == "ok" and status['SystemStatus']['Status'].lower() == "ok":
instance['Status'] = "ok"
else:
instance['Status'] = "failed"
else:
instance_stopped = True
if instance_stopped:
instance_stopped_list.append(instance['InstanceName'])
if instance_stopped:
print("-------------------------------------------------------------")
print("- WARNING: the following instances are not in running state -")
print("- Please wait for a few minutes -")
print("-------------------------------------------------------------")
for instance in instance_stopped_list:
print(" - " + instance)
print("")
print("*** Retry after 1 minute ***")
print("")
time.sleep(60)
# Print out result
server_passed = {}
server_failed = {}
for instance in InstanceList:
if instance['Status'] == "ok":
server_passed[instance['InstanceName']] = "ok"
elif instance['Status'] == "failed":
server_failed[instance['InstanceName']] = "failed"
if len(server_passed) > 0:
print("----------------------------------------------------")
print("- The following instances PASSED 2/2 status checks -")
print("----------------------------------------------------")
for passed in server_passed.keys():
print(passed)
print("")
if len(server_failed) > 0:
instance_not_ready = True
print("-----------------------------------------------------------------")
print("- WARNING: the following instances FAILED 2/2 status checks -----")
print("-----------------------------------------------------------------")
for failed in server_failed.keys():
print(failed)
# Updating migration factory migration_status attributes
for instance in InstanceList:
lifeCycle = ""
if 'lastCutoverDateTime' in instance['lifeCycle']:
if 'lastTestLaunchDateTime' in instance['lifeCycle']:
if instance['lifeCycle']['lastCutoverDateTime'] > instance['lifeCycle']['lastTestLaunchDateTime']:
lifeCycle = "Cutover Launch - "
else:
lifeCycle = "Test Launch - "
else:
lifeCycle = "Cutover Launch - "
elif 'lastTestLaunchDateTime' in instance['lifeCycle']:
lifeCycle = "Test Launch - "
if instance['Status'] == "ok":
serverattr = {"migration_status": lifeCycle + "2/2 status checks : Passed"}
elif instance['Status'] == "failed":
serverattr = {"migration_status": lifeCycle + "2/2 status checks : Failed"}
for s in serverlist:
if s['server_name'].lower() == instance['InstanceName'].lower():
updateserver = requests.put(UserHOST + serverendpoint + '/' + s['server_id'], headers=auth, data=json.dumps(serverattr))
if updateserver.status_code == 401:
print("Error: Access to migration_status attribute is denied")
sys.exit(9)
elif updateserver.status_code != 200:
print("Error: Update migration_status attribute failed")
sys.exit(10)
if instance_not_ready:
count = count + 1
if count > 12:
print("")
print("******************************* ERROR **********************************")
print("* Instances has FAILED 2/2 check for more than 1 hour, please contact support *")
print("*******************************************************************************")
sys.exit(14)
print("")
print("***********************************************")
print("* Instance booting up - retry after 5 minutes *")
print("***********************************************")
print("")
time.sleep(300)