in robot_ws/src/rl_agent/markov/s3_boto_data_store.py [0:0]
def _wait_for_ip_upload(self, timeout_in_second=600):
start_time = time.time()
s3_client = self._get_client()
while True:
response = s3_client.list_objects(Bucket=self.params.bucket, Prefix=self.ip_done_key)
if "Contents" not in response:
time.sleep(SLEEP_TIME_WHILE_WAITING_FOR_DATA_FROM_TRAINER_IN_SECOND)
time_elapsed_in_second = time.time() - start_time
if time_elapsed_in_second % 5 == 0:
print("Waiting for SageMaker Redis server IP... Time elapsed: %s seconds" % time_elapsed_in_second)
if time_elapsed_in_second % 300 == 0:
# Recreate the client with new credential every 5 minutes
s3_client = self._get_client()
if time_elapsed_in_second >= timeout_in_second:
raise RuntimeError("Cannot retrieve IP of redis server running in SageMaker")
else:
break