in common/sagemaker_rl/orchestrator/resource_manager.py [0:0]
def _usable_shared_cf_stack_exists(self):
"""Check if the shared cf stack exist and is usable
Returns:
bool: Whether the shared cf stack is usable
"""
# we can remove this logic, and have checks only on CF stack exists,
# CF stack in one of [CREATE|UPDATE|ROLLBACK]_COMPLETE state
try:
stack_name = self.shared_resource_stack_name
response = self.cf_client.describe_stacks(StackName=stack_name)["Stacks"]
if len(response) == 0:
return False
except Exception as e:
if "UnauthorizedOperation" in str(e):
raise Exception(
"You are unauthorized to describe a CloudFormation Stack. Please update your Role with "
" appropriate permissions."
)
elif "ValidationError" in str(e):
# stack doesn't exists
return False
else:
raise e
stack_details = response[0]
stack_status = stack_details["StackStatus"]
if stack_status in ["UPDATE_COMPLETE", "CREATE_COMPLETE"]:
return True
elif stack_status in ["DELETE_COMPLETE"]:
return False
elif stack_status in ["ROLLBACK_COMPLETE"]:
logger.error(
f"Stack with name {stack_name} is in {stack_status} state! Please delete/ stabilize/ or "
"or update Config.yaml to create a new stack"
)
raise Exception(
f"A Cloudformation Stack with name {stack_name}, already exists in {stack_status} State. "
f"Please debug/ or delete the stack here: {self._get_cf_stack_events_link()}"
)
elif "FAILED" in stack_status:
logger.error(
f"Stack with name {stack_name} in {stack_status} state! Please delete the stack"
" or update Config.yaml to create a new stack"
)
raise Exception(
f"A Cloudformation Stack with name {stack_name}, already exists in {stack_status} State. "
f"Please debug/ or delete the stack here: {self._get_cf_stack_events_link()}"
)
elif "DELETE" in stack_status:
# already checked DELETE_COMPLETE above
logger.error(
"Stack with name {} is in {} state! Cannot continue further!"
" Please wait for the delete to complete".format(stack_name, stack_status)
)
raise Exception(
f"A Cloudformation Stack with name {stack_name}, already exists in {stack_status} State. "
f"Please retry after the stack gets Deleted/or debug the stack here: {self._get_cf_stack_events_link()}"
)
elif "CREATE" in stack_status:
# one of the create statuses!
logger.info("Stack with name {} exists in {} state".format(stack_name, stack_status))
logger.warn("Waiting for stack to get to CREATE_COMPLETE state....")
self._wait_for_cf_stack_create_to_complete()
return True
else:
# assume stack in modifying. wait for it to goto
logger.info(
"Stack in {} state. Waiting for it's to end in successful state...".format(
stack_status
)
)
self._wait_for_cf_stack_update_to_complete()
return True