in global-clusters-automation/failover_and_delete_global_cluster.py [0:0]
def failover(global_cluster_id, secondary_cluster_arn, primary_cluster_cname, hosted_zone_id, is_delete_global_cluster):
try:
start_time = time.time()
print('Retrieving secondary clusters for global cluster ', global_cluster_id)
secondary_clusters = get_secondary_clusters(global_cluster_id)
is_secondary_cluster_promoted = False
# To minimize failover time, identify and promote the secondary cluster provided as input.
for each_secondary_cluster in secondary_clusters:
if each_secondary_cluster == secondary_cluster_arn:
print('Found secondary cluster to be promoted.')
print('Begin STEP 1 of 3 in failover process: Failover secondary cluster ', secondary_cluster_arn)
# Cluster should be in available status before removing from global cluster
cluster_status = ""
while cluster_status != 'available':
print('Checking for cluster and instance status before promotion...')
cluster_status = get_cluster_status(each_secondary_cluster)
time.sleep(1)
print('Cluster and instances are in available status. Promoting secondary cluster ***',
each_secondary_cluster, '*** to a standalone cluster.')
remove_from_global_cluster(each_secondary_cluster, global_cluster_id)
print('Initiated process to promote and remove secondary cluster ', secondary_cluster_arn,
' from global cluster ', global_cluster_id)
print('Waiting for secondary cluster ,', secondary_cluster_arn, 'to be removed from global cluster ',
global_cluster_id)
wait_for_promotion_to_complete(global_cluster_id, secondary_cluster_arn)
current_time = time.time()
is_secondary_cluster_promoted = True
print('Completed STEP 1 of 3 in failover process in ', current_time - start_time, ' seconds')
# Call Describe cluster on each_secondary_cluster to get cluster endpoint
print('Retrieving cluster endpoint for the recently promoted cluster ', each_secondary_cluster)
cluster_end_point = get_cluster_endpoint(each_secondary_cluster)
print('Begin STEP 2 of 3 in failover process : Updating route 53 CNAME ', primary_cluster_cname,
' to route to new cluster endpoint ', cluster_end_point)
# TODO : Check for explicit event for failover completion
route53_endpoint_management.manage_application_endpoint(hosted_zone_id, cluster_end_point,
primary_cluster_cname)
current_time = time.time()
print('Successfully Updated CNAME ', primary_cluster_cname, ' with record value ', cluster_end_point)
print('Completed STEP 2 of 3 in failover process in ', current_time - start_time, ' seconds')
break
if not is_secondary_cluster_promoted:
print("No matching secondary cluster found for *** ", secondary_cluster_arn, "***.Please check provided "
"input.")
raise RuntimeError
# Proceed to delete the global cluster
if is_delete_global_cluster:
print('Begin STEP 3 of 3 in failover process : '
'Process to delete other secondary clusters and the global cluster ', global_cluster_id)
delete_global_cluster(global_cluster_id, secondary_cluster_arn, secondary_clusters,
is_delete_global_cluster)
current_time = time.time()
print('Completed STEP 3 of 3 in failover process in ', current_time - start_time, ' seconds')
else:
print('Optional STEP 3 to delete cluster not chosen per provided input. '
'Process will complete now.')
print('********* SUCCESS : Secondary cluster ', secondary_cluster_arn,
'has been promoted to standalone '
'primary. *********')
except ClientError as e:
print('ERROR OCCURRED WHILE PROCESSING: ', e)
print('PROCESSING WILL STOP')
raise ClientError