nvidia-efa-ami_base/nvidia-efa-ml-ubuntu1804.yml [102:165]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      "sudo apt update",
      "echo ****INSTALL CUDA NCCL****",
      "sudo apt install -o Dpkg::Options::='--force-overwrite' {{user `nvidia_version`}} {{user `cuda_version`}} {{user `cudnn_version`}} {{user `cudnn_version`}} -y",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64\nexport PATH=$PATH:/usr/local/cuda/bin' | sudo tee /etc/profile.d/cuda.sh && sudo chmod +x /etc/profile.d/cuda.sh",
      "echo ****SET ENVIRONMENT****",
      "sudo systemctl enable nvidia-fabricmanager.service",
      "sudo sed -i '/Unattended/s/1/0/g' /etc/apt/apt.conf.d/20auto-upgrades"],
    "pause_before":"20s"
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "wget -O /tmp/aws-efa-installer.tar.gz https://efa-installer.amazonaws.com/{{user `efa_pkg`}}",
      "tar -xf /tmp/aws-efa-installer.tar.gz -C /tmp",
      "cd /tmp/aws-efa-installer",
      "sudo ./efa_installer.sh -y -g",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/amazon/openmpi/lib:/opt/amazon/efa/lib\nexport PATH=$PATH:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin' | sudo tee /etc/profile.d/amazon_efa.sh && sudo chmod +x /etc/profile.d/amazon_efa.sh"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "cd /opt",
      "sudo git clone https://github.com/NVIDIA/nccl.git",
      "cd nccl",
      "sudo git checkout {{user `nccl_version`}}",
      "sudo make -j src.build CUDA_HOME=/usr/local/cuda NVCC_GENCODE='-gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=sm_80'",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/nccl/build/lib\nexport NCCL_PROTO=simple' | sudo tee /etc/profile.d/nccl.sh && sudo chmod +x /etc/profile.d/nccl.sh"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "git clone https://github.com/aws/aws-ofi-nccl.git -b aws",
      "cd aws-ofi-nccl",
      "export PATH=$PATH:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin",
      "./autogen.sh",
      "sudo mkdir -p /opt/aws-ofi-nccl",
      "./configure --prefix=/opt/aws-ofi-nccl --with-mpi=/opt/amazon/openmpi --with-libfabric=/opt/amazon/efa --with-nccl=/opt/nccl/build --with-cuda=/usr/local/cuda",
      "make && sudo make install",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/aws-ofi-nccl/lib' | sudo tee /etc/profile.d/aws-ofi-nccl.sh && sudo chmod +x /etc/profile.d/aws-ofi-nccl.sh" ]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "git clone https://github.com/NVIDIA/nccl-tests.git",
      "cd nccl-tests",
      "make MPI=1 MPI_HOME=/opt/amazon/openmpi CUDA_HOME=/usr/local/cuda NCCL_HOME=/opt/nccl/build"]
    },    
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "echo '    StrictHostKeyChecking no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    HostbasedAuthentication no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    CheckHostIP no' | sudo tee -a /etc/ssh/ssh_config"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -",
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



nvidia-efa-ami_base/nvidia-efa-ml-ubuntu2004.yml [99:162]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      "sudo apt update",
      "echo ****INSTALL CUDA NCCL****",
      "sudo apt install -o Dpkg::Options::='--force-overwrite' {{user `nvidia_version`}} {{user `cuda_version`}} {{user `cudnn_version`}} {{user `cudnn_version`}} -y",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64\nexport PATH=$PATH:/usr/local/cuda/bin' | sudo tee /etc/profile.d/cuda.sh && sudo chmod +x /etc/profile.d/cuda.sh",
      "echo ****SET ENVIRONMENT****",
      "sudo systemctl enable nvidia-fabricmanager.service",
      "sudo sed -i '/Unattended/s/1/0/g' /etc/apt/apt.conf.d/20auto-upgrades"],
    "pause_before":"20s"
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "wget -O /tmp/aws-efa-installer.tar.gz https://efa-installer.amazonaws.com/{{user `efa_pkg`}}",
      "tar -xf /tmp/aws-efa-installer.tar.gz -C /tmp",
      "cd /tmp/aws-efa-installer",
      "sudo ./efa_installer.sh -y -g",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/amazon/openmpi/lib:/opt/amazon/efa/lib\nexport PATH=$PATH:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin' | sudo tee /etc/profile.d/amazon_efa.sh && sudo chmod +x /etc/profile.d/amazon_efa.sh"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "cd /opt",
      "sudo git clone https://github.com/NVIDIA/nccl.git",
      "cd nccl",
      "sudo git checkout {{user `nccl_version`}}",
      "sudo make -j src.build CUDA_HOME=/usr/local/cuda NVCC_GENCODE='-gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=sm_80'",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/nccl/build/lib\nexport NCCL_PROTO=simple' | sudo tee /etc/profile.d/nccl.sh && sudo chmod +x /etc/profile.d/nccl.sh"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "git clone https://github.com/aws/aws-ofi-nccl.git -b aws",
      "cd aws-ofi-nccl",
      "export PATH=$PATH:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin",
      "./autogen.sh",
      "sudo mkdir -p /opt/aws-ofi-nccl",
      "./configure --prefix=/opt/aws-ofi-nccl --with-mpi=/opt/amazon/openmpi --with-libfabric=/opt/amazon/efa --with-nccl=/opt/nccl/build --with-cuda=/usr/local/cuda",
      "make && sudo make install",
      "echo -e '#!/bin/sh\nexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/aws-ofi-nccl/lib' | sudo tee /etc/profile.d/aws-ofi-nccl.sh && sudo chmod +x /etc/profile.d/aws-ofi-nccl.sh" ]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "git clone https://github.com/NVIDIA/nccl-tests.git",
      "cd nccl-tests",
      "make MPI=1 MPI_HOME=/opt/amazon/openmpi CUDA_HOME=/usr/local/cuda NCCL_HOME=/opt/nccl/build"]
    },    
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "echo '    StrictHostKeyChecking no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    HostbasedAuthentication no' | sudo tee -a /etc/ssh/ssh_config",
      "echo '    CheckHostIP no' | sudo tee -a /etc/ssh/ssh_config"]
    },
    {
    "type": "shell",
    "inline_shebang": "/bin/bash -xe",
    "inline": [
      "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -",
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



