in infrastructure-provisioning/src/general/lib/aws/actions_lib.py [0:0]
def remove_kernels(emr_name, tag_name, nb_tag_value, ssh_user, key_path, emr_version, computational_name=''):
try:
ec2 = boto3.resource('ec2')
inst = ec2.instances.filter(
Filters=[{'Name': 'instance-state-name', 'Values': ['running']},
{'Name': 'tag:{}'.format(tag_name), 'Values': ['{}'.format(nb_tag_value)]}])
instances = list(inst)
if instances:
for instance in instances:
private = getattr(instance, 'private_dns_name')
global con
con = datalab.fab.init_datalab_connection(private, ssh_user, key_path)
con.sudo('rm -rf /home/{}/.local/share/jupyter/kernels/*_{}'.format(ssh_user, emr_name))
if exists(con, '/home/{}/.ensure_dir/dataengine-service_{}_interpreter_ensured'.format(ssh_user, emr_name)):
if os.environ['notebook_multiple_clusters'] == 'true':
try:
livy_port = con.sudo("cat /opt/" + emr_version + "/" + emr_name +
"/livy/conf/livy.conf | grep livy.server.port | tail -n 1 | "
"awk '{printf $3}'").stdout.replace('\n','')
process_number = con.sudo("netstat -natp 2>/dev/null | grep ':" + livy_port +
"' | awk '{print $7}' | sed 's|/.*||g'").stdout.replace('\n','')
con.sudo('kill -9 ' + process_number)
con.sudo('systemctl disable livy-server-' + livy_port)
except:
print("Wasn't able to find Livy server for this EMR!")
con.sudo('sed -i \"s/^export SPARK_HOME.*/export SPARK_HOME=\/opt\/spark/\" '
'/opt/zeppelin/conf/zeppelin-env.sh')
con.sudo("rm -rf /home/{}/.ensure_dir/dataengine-service_interpreter_ensure".format(ssh_user))
zeppelin_url = 'http://' + private + ':8080/api/interpreter/setting/'
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
req = opener.open(urllib.request.Request(zeppelin_url))
r_text = req.read()
interpreter_json = json.loads(r_text)
interpreter_prefix = emr_name
for interpreter in interpreter_json['body']:
if interpreter_prefix in interpreter['name']:
print("Interpreter with ID: {0} and name: {1} will be removed from zeppelin!".
format(interpreter['id'], interpreter['name']))
request = urllib.request.Request(zeppelin_url + interpreter['id'], data=''.encode())
request.get_method = lambda: 'DELETE'
url = opener.open(request)
print(url.read())
con.sudo('chown ' + ssh_user + ':' + ssh_user + ' -R /opt/zeppelin/')
con.sudo('systemctl daemon-reload')
con.sudo("service zeppelin-notebook stop")
con.sudo("service zeppelin-notebook start")
zeppelin_restarted = False
while not zeppelin_restarted:
con.sudo('sleep 5')
result = con.sudo('nmap -p 8080 localhost | grep "closed" > /dev/null; echo $?').stdout
result = result[:1]
if result == '1':
zeppelin_restarted = True
con.sudo('sleep 5')
con.sudo('rm -rf /home/{}/.ensure_dir/dataengine-service_{}_interpreter_ensured'.format(ssh_user,
emr_name))
if exists(con, '/home/{}/.ensure_dir/rstudio_dataengine-service_ensured'.format(ssh_user)):
datalab.fab.remove_rstudio_dataengines_kernel(computational_name, ssh_user)
con.sudo('rm -rf /opt/' + emr_version + '/' + emr_name + '/')
print("Notebook's {} kernels were removed".format(private))
con.close()
else:
print("There are no notebooks to clean kernels.")
except Exception as err:
logging.info("Unable to remove kernels on Notebook: " + str(err) + "\n Traceback: " + traceback.print_exc(
file=sys.stdout))
append_result(str({"error": "Unable to remove kernels on Notebook",
"error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)