infrastructure-provisioning/src/general/scripts/aws/tensor-jupyterlab_configure.py [168:320]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to configure TensorFlow.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[INSTALLING USERs KEY]') additional_config = {"user_keyname": notebook_config['user_keyname'], "user_keydir": os.environ['conf_key_dir']} params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['datalab_ssh_user']) try: subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed installing users key.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[SETUP USER GIT CREDENTIALS]') params = '--os_user {} --notebook_ip {} --keyfile "{}"' \ .format(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) try: subprocess.run("~/scripts/{}.py {}".format('manage_git_creds', params), shell=True, check=True) except: datalab.fab.append_result("Failed setup git credentials") raise Exception except Exception as err: datalab.fab.append_result("Failed to setup git credentials.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[POST CONFIGURING PROCESS]') if notebook_config['notebook_image_name'] not in [notebook_config['expected_image_name'], 'None', '']: params = "--hostname {} --keyfile {} --os_user {} --nb_tag_name {} --nb_tag_value {}" \ .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], notebook_config['tag_name'], notebook_config['instance_name']) try: subprocess.run("~/scripts/{}.py {}".format('common_remove_remote_kernels', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to post configuring instance.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') additional_info = { 'instance_hostname': instance_hostname, 'tensor': True } params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'jupyter', notebook_config['exploratory_name'], json.dumps(additional_info)) try: subprocess.run("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params), shell=True, check=True) except: datalab.fab.append_result("Failed edge reverse proxy template") raise Exception except Exception as err: datalab.fab.append_result("Failed to set edge reverse proxy template.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) if notebook_config['image_enabled'] == 'true': try: logging.info('[CREATING AMI]') ami_id = datalab.meta_lib.get_ami_id_by_name(notebook_config['expected_image_name']) if ami_id == '' and notebook_config['shared_image_enabled'] == 'false': logging.info("Looks like it's first time we configure notebook server. Creating image.") try: os.environ['conf_additional_tags'] = '{2};project_tag:{0};endpoint_tag:{1};'.format( os.environ['project_name'], os.environ['endpoint_name'], os.environ['conf_additional_tags']) except KeyError: os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format( os.environ['project_name'], os.environ['endpoint_name']) image_id = datalab.actions_lib.create_image_from_instance( tag_name=notebook_config['tag_name'], instance_name=notebook_config['instance_name'], image_name=notebook_config['expected_image_name']) if image_id != '': logging.info("Image was successfully created. It's ID is {}".format(image_id)) else: logging.info("Looks like it's first time we configure notebook server. Creating image.") try: os.environ['conf_additional_tags'] = '{};ami:shared;endpoint_tag:{};'.format( os.environ['conf_additional_tags'], os.environ['endpoint_name']) except KeyError: os.environ['conf_additional_tags'] = 'ami:shared;endpoint_tag:{}'.format( os.environ['endpoint_name']) image_id = datalab.actions_lib.create_image_from_instance( tag_name=notebook_config['tag_name'], instance_name=notebook_config['instance_name'], image_name=notebook_config['expected_image_name']) if image_id != '': logging.info("Image was successfully created. It's ID is {}".format(image_id)) except Exception as err: datalab.fab.append_result("Failed creating image.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: # generating output information ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private') dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name']) tensorboard_url = "http://" + ip_address + ":6006/" jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name']) jupyter_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) tensorboard_access_url = "https://{}/{}-tensor/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) jupyter_ungit_access_url = "https://{}/{}-ungit/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) logging.info('[SUMMARY]') logging.info("Instance name: {}".format(notebook_config['instance_name'])) logging.info("Private DNS: {}".format(dns_name)) logging.info("Private IP: {}".format(ip_address)) logging.info("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name']))) logging.info("Instance type: {}".format(notebook_config['instance_type'])) logging.info("Key name: {}".format(notebook_config['key_name'])) logging.info("User key name: {}".format(notebook_config['user_keyname'])) logging.info("AMI name: {}".format(notebook_config['notebook_image_name'])) logging.info("Profile name: {}".format(notebook_config['role_profile_name'])) logging.info("SG name: {}".format(notebook_config['security_group_name'])) logging.info("TensorBoard URL: {}".format(tensorboard_url)) logging.info("TensorBoard log dir: /var/log/tensorboard") logging.info("Jupyter URL: {}".format(jupyter_ip_url)) logging.info("Ungit URL: {}".format(ungit_ip_url)) logging.info('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format( notebook_config['key_name'], notebook_config['datalab_ssh_user'], ip_address)) logging.info('SSH access (from Edge node, via FQDN): ssh -i {0}.pem {1}@{2}'.format( notebook_config['key_name'], notebook_config['datalab_ssh_user'], dns_name)) with open("/root/result.json", 'w') as result: res = {"hostname": dns_name, "ip": ip_address, "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name']), "master_keyname": os.environ['conf_key_name'], "tensorboard_log_dir": "/var/log/tensorboard", "notebook_name": notebook_config['instance_name'], "notebook_image_name": notebook_config['notebook_image_name'], "Action": "Create new notebook server", "exploratory_url": [ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - infrastructure-provisioning/src/general/scripts/aws/tensor_configure.py [184:336]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to configure TensorFlow.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[INSTALLING USERs KEY]') additional_config = {"user_keyname": notebook_config['user_keyname'], "user_keydir": os.environ['conf_key_dir']} params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( instance_hostname, keyfile_name, json.dumps(additional_config), notebook_config['datalab_ssh_user']) try: subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed installing users key.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[SETUP USER GIT CREDENTIALS]') params = '--os_user {} --notebook_ip {} --keyfile "{}"' \ .format(notebook_config['datalab_ssh_user'], instance_hostname, keyfile_name) try: subprocess.run("~/scripts/{}.py {}".format('manage_git_creds', params), shell=True, check=True) except: datalab.fab.append_result("Failed setup git credentials") raise Exception except Exception as err: datalab.fab.append_result("Failed to setup git credentials.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[POST CONFIGURING PROCESS]') if notebook_config['notebook_image_name'] not in [notebook_config['expected_image_name'], 'None', '']: params = "--hostname {} --keyfile {} --os_user {} --nb_tag_name {} --nb_tag_value {}" \ .format(instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], notebook_config['tag_name'], notebook_config['instance_name']) try: subprocess.run("~/scripts/{}.py {}".format('common_remove_remote_kernels', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to post configuring instance.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') additional_info = { 'instance_hostname': instance_hostname, 'tensor': True } params = "--edge_hostname {} --keyfile {} --os_user {} --type {} --exploratory_name {} --additional_info '{}'" \ .format(edge_instance_hostname, keyfile_name, notebook_config['datalab_ssh_user'], 'jupyter', notebook_config['exploratory_name'], json.dumps(additional_info)) try: subprocess.run("~/scripts/{}.py {}".format('common_configure_reverse_proxy', params), shell=True, check=True) except: datalab.fab.append_result("Failed edge reverse proxy template") raise Exception except Exception as err: datalab.fab.append_result("Failed to set edge reverse proxy template.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) if notebook_config['image_enabled'] == 'true': try: logging.info('[CREATING AMI]') ami_id = datalab.meta_lib.get_ami_id_by_name(notebook_config['expected_image_name']) if ami_id == '' and notebook_config['shared_image_enabled'] == 'false': logging.info("Looks like it's first time we configure notebook server. Creating image.") try: os.environ['conf_additional_tags'] = '{2};project_tag:{0};endpoint_tag:{1};'.format( os.environ['project_name'], os.environ['endpoint_name'], os.environ['conf_additional_tags']) except KeyError: os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format( os.environ['project_name'], os.environ['endpoint_name']) image_id = datalab.actions_lib.create_image_from_instance( tag_name=notebook_config['tag_name'], instance_name=notebook_config['instance_name'], image_name=notebook_config['expected_image_name']) if image_id != '': logging.info("Image was successfully created. It's ID is {}".format(image_id)) else: logging.info("Looks like it's first time we configure notebook server. Creating image.") try: os.environ['conf_additional_tags'] = '{};ami:shared;endpoint_tag:{};'.format( os.environ['conf_additional_tags'], os.environ['endpoint_name']) except KeyError: os.environ['conf_additional_tags'] = 'ami:shared;endpoint_tag:{}'.format( os.environ['endpoint_name']) image_id = datalab.actions_lib.create_image_from_instance( tag_name=notebook_config['tag_name'], instance_name=notebook_config['instance_name'], image_name=notebook_config['expected_image_name']) if image_id != '': logging.info("Image was successfully created. It's ID is {}".format(image_id)) except Exception as err: datalab.fab.append_result("Failed creating image.", str(err)) datalab.actions_lib.remove_ec2(notebook_config['tag_name'], notebook_config['instance_name']) sys.exit(1) try: # generating output information ip_address = datalab.meta_lib.get_instance_ip_address(notebook_config['tag_name'], notebook_config['instance_name']).get('Private') dns_name = datalab.meta_lib.get_instance_hostname(notebook_config['tag_name'], notebook_config['instance_name']) tensorboard_url = "http://" + ip_address + ":6006/" jupyter_ip_url = "http://" + ip_address + ":8888/{}/".format(notebook_config['exploratory_name']) jupyter_notebook_access_url = "https://{}/{}/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) tensorboard_access_url = "https://{}/{}-tensor/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) jupyter_ungit_access_url = "https://{}/{}-ungit/".format(notebook_config['edge_instance_hostname'], notebook_config['exploratory_name']) ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(notebook_config['exploratory_name']) logging.info('[SUMMARY]') logging.info("Instance name: {}".format(notebook_config['instance_name'])) logging.info("Private DNS: {}".format(dns_name)) logging.info("Private IP: {}".format(ip_address)) logging.info("Instance ID: {}".format(datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name']))) logging.info("Instance type: {}".format(notebook_config['instance_type'])) logging.info("Key name: {}".format(notebook_config['key_name'])) logging.info("User key name: {}".format(notebook_config['user_keyname'])) logging.info("AMI name: {}".format(notebook_config['notebook_image_name'])) logging.info("Profile name: {}".format(notebook_config['role_profile_name'])) logging.info("SG name: {}".format(notebook_config['security_group_name'])) logging.info("TensorBoard URL: {}".format(tensorboard_url)) logging.info("TensorBoard log dir: /var/log/tensorboard") logging.info("Jupyter URL: {}".format(jupyter_ip_url)) logging.info("Ungit URL: {}".format(ungit_ip_url)) logging.info('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format( notebook_config['key_name'], notebook_config['datalab_ssh_user'], ip_address)) logging.info('SSH access (from Edge node, via FQDN): ssh -i {0}.pem {1}@{2}'.format( notebook_config['key_name'], notebook_config['datalab_ssh_user'], dns_name)) with open("/root/result.json", 'w') as result: res = {"hostname": dns_name, "ip": ip_address, "instance_id": datalab.meta_lib.get_instance_by_name(notebook_config['tag_name'], notebook_config['instance_name']), "master_keyname": os.environ['conf_key_name'], "tensorboard_log_dir": "/var/log/tensorboard", "notebook_name": notebook_config['instance_name'], "notebook_image_name": notebook_config['notebook_image_name'], "Action": "Create new notebook server", "exploratory_url": [ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -