in infrastructure-provisioning/src/general/lib/os/fab.py [0:0]
def install_pip_pkg(requisites, pip_version, lib_group, dataengine_service=False):
status = list()
error_parser = "Could not|No matching|ImportError:|failed|EnvironmentError:|requires|FileNotFoundError:|" \
"RuntimeError:|error:"
try:
if dataengine_service:
install_command = pip_version
elif os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
install_command = 'conda activate && {}'.format(pip_version)
else:
install_command = 'source /opt/python/python{0}/bin/activate && /opt/python/python{0}/bin/pip{1}'.format(
os.environ['notebook_python_venv_version'], os.environ['notebook_python_venv_version'][:3])
for pip_pkg in requisites:
name, vers = pip_pkg
if pip_pkg[1] == '' or pip_pkg[1] == 'N/A':
pip_pkg = pip_pkg[0]
version = 'N/A'
else:
version = pip_pkg[1]
pip_pkg = "{}=={}".format(pip_pkg[0], pip_pkg[1])
conn.sudo(
'''bash -l -c '{0} install -U {1} --use-deprecated=legacy-resolver --no-cache-dir 2>&1 | '''
'''tee /tmp/{4}_install_{3}.tmp; if ! grep -w -i -E "({2})" /tmp/{4}_install_{3}.tmp > /tmp/{4}_install_{3}.log; '''
'''then echo "" > /tmp/{4}_install_{3}.log;fi' '''.format(
install_command, pip_pkg, error_parser, name, pip_version))
err = conn.sudo('cat /tmp/{0}_install_{1}.log'.format(pip_version, pip_pkg.split("==")[0])).stdout.replace(
'"', "'").replace('\n', ' ')
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; '''
'''then echo "not_found" > /tmp/{2}_install_{1}.list;fi' '''.format(
install_command, name, pip_version))
res = conn.sudo('''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace(
'\n', '')
conn.sudo(
'''bash -l -c 'cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "(Successfully installed|up-to-date)" > '''
'''/tmp/{0}_install_{1}.list; then echo "not_installed" > /tmp/{0}_install_{1}.list;fi' '''.format(
pip_version, name))
installed_out = conn.sudo(
'''bash -l -c 'cat /tmp/{0}_install_{1}.list' '''.format(pip_version, name)).stdout.replace('\n', '')
changed_pip_pkg = False
if 'not_found' in res:
changed_pip_pkg = pip_pkg.split("==")[0].replace("_", "-").split('-')
changed_pip_pkg = changed_pip_pkg[0]
conn.sudo(
'''bash -l -c '{0} freeze --all | if ! grep -w -i {1} > /tmp/{2}_install_{1}.list; then echo "" > '''
'''/tmp/{2}_install_{1}.list;fi' '''.format(
install_command, changed_pip_pkg, pip_version))
res = conn.sudo('cat /tmp/{0}_install_{1}.list'.format(pip_version, changed_pip_pkg)).stdout.replace(
'\n', '')
if err and name not in installed_out:
status_msg = 'installation_error'
if 'ERROR: No matching distribution found for {}'.format(name) in err:
status_msg = 'invalid_name'
elif res:
res = res.lower()
ansi_escape = re.compile(r'\x1b[^m]*m')
ver = ansi_escape.sub('', res).split("\r\n")
if changed_pip_pkg:
version = [i for i in ver if changed_pip_pkg.lower() in i][0].split('==')[1]
else:
version = \
[i for i in ver if pip_pkg.split("==")[0].lower() in i][0].split('==')[1]
status_msg = "installed"
versions = []
if 'Could not find a version that satisfies the requirement' in err \
and 'ERROR: No matching distribution found for {}=='.format(name) in err:
versions = err[err.find("(from versions: ") + 16: err.find(") ")]
if versions != '' and versions != 'none':
versions = versions.split(', ')
version = vers
status_msg = 'invalid_version'
else:
versions = []
conn.sudo('cat /tmp/{0}_install_{1}.tmp | if ! grep -w -i -E "Installing collected packages:" > '
'/tmp/{0}_install_{1}.dep; then echo "" > /tmp/{0}_install_{1}.dep;fi'.format(pip_version, name))
dep = conn.sudo('cat /tmp/{0}_install_{1}.dep'.format(pip_version, name)).stdout.replace('\n', '').strip()[
31:]
if dep == '':
dep = []
else:
dep = dep.split(', ')
for n, i in enumerate(dep):
if i == name:
dep[n] = ''
else:
conn.sudo('{0} show {1} 2>&1 | if ! grep Version: > '
'/tmp/{0}_install_{1}.log; then echo "" > /tmp/{0}_install_{1}.log;fi'.format(
pip_version, i))
dep[n] = conn.sudo('cat /tmp/{0}_install_{1}.log'.format(
pip_version, i)).stdout.replace('\n', '').replace('Version: ', '{} v.'.format(i))
dep = [i for i in dep if i]
status.append({"group": lib_group, "name": name, "version": version, "status": status_msg,
"error_message": err, "available_versions": versions, "add_pkgs": dep})
conn.sudo('rm -rf /tmp/*{}*'.format(name))
return status
except Exception as err:
for pip_pkg in requisites:
name, vers = pip_pkg
status.append({"group": lib_group, "name": name, "version": vers, "status": 'installation_error',
"error_message": err})
logging.error("Failed to install {} packages: {}".format(pip_version, err))
traceback.print_exc()
return status