in infrastructure-provisioning/scripts/deploy_repository/deploy_repository.py [0:0]
def download_packages():
try:
if not exists(conn,'/home/{}/.ensure_dir/packages_downloaded'.format(configuration['conf_os_user'])):
packages_urls = [
'https://pkg.jenkins.io/debian/jenkins-ci.org.key',
'http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'.format(
maven_version.split('.')[0], maven_version),
'https://nodejs.org/dist/v8.15.0/node-v8.15.0.tar.gz',
'https://github.com/sass/node-sass/releases/download/v4.11.0/linux-x64-57_binding.node',
'http://nginx.org/download/nginx-{}.tar.gz'.format(configuration['reverse_proxy_nginx_version']),
'https://www.scala-lang.org/files/archive/scala-{}.deb'.format(configuration['notebook_scala_version']),
'https://archive.apache.org/dist/spark/spark-{0}/spark-{0}-bin-hadoop{1}.tgz'.format(
configuration['notebook_spark_version'], configuration['notebook_hadoop_version']),
'https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/{0}/hadoop-aws-{0}.jar'.format('2.7.4'),
'https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/{0}/aws-java-sdk-{0}.jar'.format('1.7.4'),
# 'https://maven.twttr.com/com/hadoop/gplcompression/hadoop-lzo/{0}/hadoop-lzo-{0}.jar'.format('0.4.20'),
'https://repo1.maven.org/maven2/org/scalanlp/breeze_{0}/{1}/breeze_{0}-{1}.jar'.format('2.11', '0.12'),
'https://repo1.maven.org/maven2/org/scalanlp/breeze-natives_{0}/{1}/breeze-natives_{0}-{1}.jar'.format(
'2.11', '0.12'),
'https://repo1.maven.org/maven2/org/scalanlp/breeze-viz_{0}/{1}/breeze-viz_{0}-{1}.jar'.format(
'2.11', '0.12'),
'https://repo1.maven.org/maven2/org/scalanlp/breeze-macros_{0}/{1}/breeze-macros_{0}-{1}.jar'.format(
'2.11', '0.12'),
'https://repo1.maven.org/maven2/org/scalanlp/breeze-parent_{0}/{1}/breeze-parent_{0}-{1}.jar'.format(
'2.11', '0.12'),
'https://repo1.maven.org/maven2/org/jfree/jfreechart/{0}/jfreechart-{0}.jar'.format('1.0.19'),
'https://repo1.maven.org/maven2/org/jfree/jcommon/{0}/jcommon-{0}.jar'.format('1.0.24'),
'--no-check-certificate https://brunelvis.org/jar/spark-kernel-brunel-all-{0}.jar'.format('2.3'),
'http://archive.apache.org/dist/incubator/toree/0.3.0-incubating/toree-pip/toree-0.3.0.tar.gz',
'https://download2.rstudio.org/server/trusty/amd64/rstudio-server-{}-amd64.deb'.format(
configuration['notebook_rstudio_version']),
'http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run'.format(
configuration['notebook_nvidia_version']),
'https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'.format(
cuda_version_deeplearning, cuda_deeplearingn_file_name),
'https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'.format(
configuration['notebook_cuda_version'], configuration['notebook_cuda_file_name']),
'https://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1}'.format(
cudnn_version_deeplearning, cudnn_file_name_deeplearning),
'https://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1}'.format(
configuration['notebook_cudnn_version'], configuration['notebook_cudnn_file_name']),
'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-'
'linux_x86_64.whl'.format(tensorflow_version_deeplearning),
'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-'
'linux_x86_64.whl'.format(tensorflow_version_deeplearning),
'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-'
'linux_x86_64.whl'.format(configuration['notebook_tensorflow_version']),
'https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-'
'linux_x86_64.whl'.format(configuration['notebook_tensorflow_version']),
'https://cmake.org/files/v{1}/cmake-{0}.tar.gz'.format(
configuration['notebook_cmake_version'],
configuration['notebook_cmake_version'].split('.')[0] +
"." + configuration['notebook_cmake_version'].split('.')[1]),
'https://cntk.ai/PythonWheel/GPU/cntk-{}-cp27-cp27mu-linux_x86_64.whl'.format(
configuration['notebook_cntk_version']),
'https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl'.format(
configuration['notebook_cntk_version']),
'https://www.python.org/ftp/python/{0}/Python-{0}.tgz'.format(python3_version),
'https://nexus.develop.dlabanalytics.com/repository/packages-public/zeppelin-{}-prebuilt.tar.gz'.format(
configuration['notebook_zeppelin_version']),
'http://archive.cloudera.com/beta/livy/livy-server-{}.zip'.format(
configuration['notebook_livy_version']),
'https://repos.spark-packages.org/tapanalyticstoolkit/spark-tensorflow-connector/'
'1.0.0-s_2.11/spark-tensorflow-connector-1.0.0-s_2.11.jar',
'https://archive.apache.org/dist/incubator/toree/0.3.0-incubating/toree/'
'toree-0.3.0-incubating-bin.tar.gz',
'https://repo1.maven.org/maven2/org/apache/toree/toree-assembly/0.3.0-incubating/'
'toree-assembly-0.3.0-incubating.jar',
'https://cran.r-project.org/src/contrib/Archive/keras/keras_{}.tar.gz'.format(
configuration['notebook_keras_version'])
]
packages_list = list()
for package in packages_urls:
package_name = package.split('/')[-1]
packages_list.append({'url': package, 'name': package_name})
conn.run('mkdir packages')
for package in packages_list:
conn.run('cd packages && wget {0}'.format(package['url']))
conn.run('curl -v -u admin:{2} -F "raw.directory=/" -F '
'"raw.asset1=@/home/{0}/packages/{1}" '
'-F "raw.asset1.filename={1}" '
'"http://localhost:8081/service/rest/v1/components?repository=packages"'.format(
configuration['conf_os_user'], package['name'], args.nexus_admin_password))
conn.sudo('touch /home/{}/.ensure_dir/packages_downloaded'.format(configuration['conf_os_user']))
except Exception as err:
traceback.print_exc()
print('Failed to download packages: ', str(err))
sys.exit(1)