workshops/eda-workshop-soca/templates/nfs_server_ebs_zfs_soca.yaml [445:878]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      IamInstanceProfile: !Ref 'RootInstanceProfile'
      ImageId: !FindInMap
        - AWSRegionAMI
        - !Ref 'AWS::Region'
        - !Ref 'OperatingSystem'
      InstanceType: !Ref 'NFSInstanceType'
      KeyName: !Ref 'AdminKeyPair'
      NetworkInterfaces:
        - AssociatePublicIpAddress: !Ref 'UsePublicIp'
          DeleteOnTermination: true
          DeviceIndex: '0'
          GroupSet:
            - !Ref 'ExistingSecurityGroup'
          PrivateIpAddress: !If
            - Has_Static_Private_IP
            - !Ref 'StaticPrivateIpAddress'
            - !Ref 'AWS::NoValue'
          SubnetId: !Ref Subnet
      PlacementGroupName: !If
        - no_placement_group
        - !Ref 'AWS::NoValue'
        - !Ref 'ExistingPlacementGroup'
      Tags:
        - 
          Key: "Name"
          Value: "NFS Server"

      UserData: !Base64
        Fn::Sub: |
          #!/usr/bin/env bash

          set -x
          exec > >(tee /var/log/user-data.log|logger -t user-data ) 2>&1

          echo "*** BEGIN NFS SERVER BOOTSTRAP ***"

          nfs_instance_type="${NFSInstanceType}"
          nfs_os="${OperatingSystem}"
          nfs_cidr_block="${NFSCidr}"
          nfs_opts="${NFSOpts}"
          my_wait_handle="${NFSInstanceWaitHandle}"
          volume_type="${VolumeType}"
          hypervisor_type="nitro"
          zfs_or_mdadm="${ZfsOrMdadm}"

          zfs_pool_name="${ZfsPool}"
          zfs_mount_point="${ZfsMountPoint}"
          use_l2arc="${UseL2arc}"
          use_zil="${UseZil}"
          #@IgnoreInspection BashAddShebang

          #
          # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
          #
          # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file
          # except in compliance with the License. A copy of the License is located at
          #
          #  http://aws.amazon.com/apache2.0/
          #
          # or in the "license" file accompanying this file. This file is distributed on an "AS IS"
          # BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
          # License for the specific language governing permissions and limitations under the License.
          #

          function inst_pkgs {
            echo "Installing packages and updates"
            sudo yum install nfs-utils rpcbind vim nvme-cli mdadm -y
            #sudo yum update -y
            echo "Updates complete"
          }

          function inst_ssm {
            echo "Installing and starting AWS Systems Manager (SSM) agent"
            sudo yum install -q -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
            systemctl enable amazon-ssm-agent
            systemctl start amazon-ssm-agent
          }

          function inst_cwagent {
            echo "Installing CloudWatch agent"
            sudo yum install -q -y https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
            sudo su -c 'echo -e "{
                    \"agent\": {
                            \"metrics_collection_interval\": 60,
                            \"run_as_user\": \"root\"
                    },
                    \"metrics\": {
                            \"append_dimensions\": {
                                    \"InstanceId\": \"$\{aws:InstanceId\}\",
                                    \"InstanceType\": \"$\{aws:InstanceType\}\"
                            },
                            \"metrics_collected\": {
                                    \"cpu\": {
                                            \"measurement\": [
                                                    \"cpu_usage_idle\",
                                                    \"cpu_usage_iowait\",
                                                    \"cpu_usage_user\",
                                                    \"cpu_usage_system\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"totalcpu\": true
                                    },
                                    \"disk\": {
                                            \"measurement\": [
                                                    \"used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"resources\": [
                                                    \"nvme1n1\",
                                                    \"nvme2n1\",
                                                    \"nvme3n1\",
                                                    \"nvme4n1\",
                                                    \"nvme5n1\",
                                                    \"nvme6n1\",
                                                    \"nvme7n1\",
                                                    \"nvme8n1\"
                                            ]
                                    },
                                    \"diskio\": {
                                            \"measurement\": [
                                                    \"io_time\",
                                                    \"iops_in_progress\",
                                                    \"write_bytes\",
                                                    \"write_time\",
                                                    \"read_bytes\",
                                                    \"read_time\",
                                                    \"writes\",
                                                    \"reads\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"resources\": [
                                                    \"nvme1n1\",
                                                    \"nvme2n1\",
                                                    \"nvme3n1\",
                                                    \"nvme4n1\",
                                                    \"nvme5n1\",
                                                    \"nvme6n1\",
                                                    \"nvme7n1\",
                                                    \"nvme8n1\"
                                            ]
                                    },
                                    \"mem\": {
                                            \"measurement\": [
                                                    \"mem_used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    },
                                    \"netstat\": {
                                            \"measurement\": [
                                                    \"tcp_established\",
                                                    \"tcp_time_wait\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    },
                                    \"swap\": {
                                            \"measurement\": [
                                                    \"swap_used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    }
                            }
                    }
            }" >> /opt/aws/amazon-cloudwatch-agent/bin/config.json'
            sudo sed -i 's/\(Instance.*\)\\{\(.*\)\\}/\1{\2}/g' /opt/aws/amazon-cloudwatch-agent/bin/config.json
            echo "Starting CloudWatch agent"
            sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json -s
          }

          function sys_limits {
            echo "Updating sys limits"
            sudo su -c 'echo -e "
            fs.file-max = 2097152
            vm.swappiness = 1
            vm.dirty_ratio = 60
            vm.dirty_background_ratio = 2
            vm.dirty_bytes = 4294967296
            vm.dirty_background_bytes = 1073741824
            vm.min_free_kbytes = 1048576
            net.core.rmem_default = 31457280
            net.core.rmem_max = 33554432
            net.core.wmem_default = 31457280
            net.core.wmem_max = 33554432
            net.core.somaxconn = 65535
            net.core.netdev_max_backlog = 250000
            net.core.optmem_max = 25165824
            net.ipv4.tcp_mem = 786432 1048576 26777216
            net.ipv4.udp_mem = 65536 131072 262144
            net.ipv4.tcp_rmem = 8192 87380 33554432
            net.ipv4.udp_rmem_min = 16384
            net.ipv4.tcp_wmem = 8192 65536 33554432
            net.ipv4.udp_wmem_min = 16384
            net.ipv4.tcp_sack = 1
            net.ipv4.tcp_dsack = 1
            net.ipv4.tcp_fack = 1
            net.ipv4.tcp_fin_timeout = 30
            " >> /etc/sysctl.conf'
            sudo sysctl -p
            sudo su -c 'echo -e "
            *   hard nofile 1048576
            *   soft nofile 1048576
            " >> /etc/security/limits.conf'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/rmem_default'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/rmem_max'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/wmem_default'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/wmem_max'
          }

          function ck_devs_count_build_list {

            if [[ "$volume_type" == "EBS" ]]; then
            echo "Checking the total number of EBS volumes, currently ${!devs_count_ready} devices ready."
            devs_count_ready=0
            let dev_count=$(sudo nvme list | grep "${!nvme_grep_str}" | grep -v nvme0n1 | wc -l)
            if [[ $dev_count -eq ${!total_ebs_vols} ]]; then
              if [[ ! ${!dev_list} ]]; then
                for d in $(sudo nvme list | grep "${!nvme_grep_str}" | grep -v nvme0n1 | awk {'print $1'} | cut -f3 -d"/")
                  do echo -n "$d "
                  dev_list="${!dev_list} ${!d}"
                done
              fi
              devs_count_ready=1
            else
              devs_count_ready=0
            fi
            elif [[ "$volume_type" == "InstanceStore" ]]; then
            echo "Checking the total number of Instance Store NVMe volumes, currently ${!devs_count_ready} devices ready."
            devs_count_ready=0
            let dev_count=$(sudo nvme list | grep "${!nvme_grep_str}" | wc -l)
            if [[ $dev_count -eq ${!total_i_store_vols} ]]; then
              if [[ ! ${!dev_list} ]]; then
                for d in $(sudo nvme list | grep "${!nvme_grep_str}" | awk {'print $1'} | cut -f3 -d"/")
                  do echo -n "$d "
                  dev_list="${!dev_list} ${!d}"
                done
              fi
              devs_count_ready=1
            else
              devs_count_ready=0
            fi
            fi

          }

          timeout_count=0
          function ck_devs {
            devs_ready=0

            if [[ "$volume_type" == "EBS" ]]; then
            nvme_grep_str="Amazon Elastic Block Store"
            total_ebs_vols=8
            total_vols=${!total_ebs_vols}
            elif [[ "$volume_type" == "InstanceStore" ]]; then
            nvme_grep_str="Amazon EC2 NVMe Instance Storage"
            total_i_store_vols=${!instStore_devs["${!nfs_instance_type}"]}
            total_vols=${!total_i_store_vols}
            fi

            while [[ ! $(which nvme) ]]; do
            sleep 10
            done

            while [[ ! ${!devs_count_ready} ]]; do
            ck_devs_count_build_list
            sleep 10
            done

            echo "Using device list ${!dev_list}"

            while [[ ! ${!devs_ready} ]]; do
            for l in ${!dev_list}; do
              if [[ ! -b /dev/${!l} ]]; then
                devs_ready=0
                ((timeout_count = $timeout_count + 1))
                if [[ $timeout_count -gt 30 ]]; then
                  echo "All storage volumes are ready, exiting..."
                  exit
                else
                  sleep 10
                fi
              else
                devs_ready=1
                ck_devs
              fi
            done
            done
            echo "Devices ready."
          }

          function mk_raid {
            raid_level="0"
            full_path_dev_list=""
            for d in ${!dev_list}; do
            temp_d="/dev/${!d}"
            full_path_dev_list="${!full_path_dev_list} ${!temp_d}"
            done
            yes | sudo mdadm --create --verbose /dev/md0 --level=${!raid_level} --name=raid-vol1 --raid-devices=${!total_vols} ${!full_path_dev_list}
          }

          function mk_ext4_fs {
            mdadm_fs_dev="/dev/md0"
            mdadm_fs_name=${!mdadm_raid_fs_name}
            mdadm_fs_label=${!mdadm_raid_fs_name}
            sudo mkfs.ext4 -L ${!mdadm_fs_label} ${!mdadm_fs_dev} -E nodiscard
            sudo mkdir /${!mdadm_fs_name}
            sudo mount LABEL=${!mdadm_fs_label} /${!mdadm_fs_name}
            sudo su -c "echo 'LABEL=${!mdadm_fs_label}     /${!mdadm_fs_name}   ext4    defaults,nofail     0      2' >> /etc/fstab"
          }

          function fix_zfs_repo {
            echo "Fixing ZFS repo..."
            sudo cat /etc/yum.repos.d/zfs.repo  | while read l
            do
            if [[ "$l" == "[zfs]" ]]; then
              let on_zfs=1
            elif [[ "$l" == "[zfs-kmod]" ]]; then
              let on_zfs_kmod=1
            fi
            if [[ "$on_zfs" -eq 1 ]] && [[ "$l" == "enabled=1" ]]; then
              l="enabled=0"
              let on_zfs=0
            elif [[ "$on_zfs_kmod" -eq 1 ]] &&  [[ "$l" == "enabled=0" ]]; then
              l="enabled=1"
              let on_zfs_kmod=0
            fi
            echo $l
            done > /tmp/new_zfs.repo
            sudo cp /etc/yum.repos.d/zfs.repo /etc/yum.repos.d/zfs.repo.dist
            sudo mv /tmp/new_zfs.repo /etc/yum.repos.d/zfs.repo
            echo "Done fixing ZFS repo"

          }

          function zfs_install_centos7 {
            echo -n "Fixing, removing, and re-installing ZFS..."
            known_zfs_key="C93AFFFD9F3F7B03C310CEB6A9D5A1C0F14AB620"
            sudo yum -y remove zfs zfs-kmod spl spl-kmod libzfs2 libnvpair1 libuutil1 libzpool2 zfs-release
            sudo yum -y install http://download.zfsonlinux.org/epel/zfs-release.el7_7.noarch.rpm
            actual_zfs_key=$(gpg --quiet --with-fingerprint /etc/pki/rpm-gpg/RPM-GPG-KEY-zfsonlinux | grep "Key fingerprint" | cut -d"=" -f2 | tr -d ' ')
            echo "Checking keys..."
            if [[ "$known_zfs_key" != "$actual_zfs_key" ]]; then
            echo "ERROR: ZFS installation keys not valid!!!"
            echo "Exiting..."
            exit
            fi
            fix_zfs_repo
            sudo yum -y autoremove
            sudo yum -y clean metadata
            sudo yum -y install zfs
            echo "Done installing ZFS"
          }

          function zfs_install_amzn2 {
            echo -n "Fixing, removing, and re-installing ZFS..."
            known_zfs_key="C93AFFFD9F3F7B03C310CEB6A9D5A1C0F14AB620"
            sudo yum -y remove zfs zfs-kmod spl spl-kmod libzfs2 libnvpair1 libuutil1 libzpool2 zfs-release
            sudo yum -y install http://download.zfsonlinux.org/epel/zfs-release.el7_7.noarch.rpm
            actual_zfs_key=$(gpg --quiet --with-fingerprint /etc/pki/rpm-gpg/RPM-GPG-KEY-zfsonlinux | grep "Key fingerprint" | cut -d"=" -f2 | tr -d ' ')
            echo "Checking keys..."
            if [[ "$known_zfs_key" != "$actual_zfs_key" ]]; then
            echo "ERROR: ZFS installation keys not valid!!!"
            echo "Exiting..."
            exit
            fi
            sudo yum -y autoremove
            sudo yum -y clean metadata
            sudo yum -y install dkms "kernel-devel-uname-r == $(uname -r)" zfs
            echo "Done installing ZFS"
          }

          function zfs_create {

            echo "Using these devices for ZFS:"
            sudo nvme list | grep "Amazon Elastic Block Store" | grep -v nvme0n1

            sudo /sbin/modprobe zfs
            modinfo zfs
            sudo zpool create -O compression=lz4 -O atime=off -O sync=disabled -f ${!zfs_pool_name} -o ashift=12 ${!dev_list}
            sudo zpool status -v
            sudo zfs create ${!zfs_pool_name}/${!zfs_mount_point}
          }

          function zfs_startup {
            sudo systemctl enable zfs-import-cache
            sudo systemctl enable zfs-mount.service
            sudo systemctl enable zfs.target
          }

          function nfs_server_settings {
            sudo su -c 'echo -e "STATD_PORT=\"32765\"\nSTATD_OUTGOING_PORT=\"32766\"\nSTATDARG=\"-p 32765 -o 32766\"\nMOUNTD_PORT=\"32767\"\nRPCMOUNTDOPTS=\"-p 32767\"
          LOCKD_UDPPORT=\"32768\"\nLOCKD_TCPPORT=\"32768\"\nRQUOTAD_PORT=\"32769\"\nRQUOTAD=\"no\"\nRPCNFSDCOUNT=\"128\""' > /etc/sysconfig/nfs
          }

          function nfs_config {
            nfs_server_settings

            if [[ "${!zfs_or_mdadm}" == "ZFS" ]]; then
            sudo su -c 'echo "'"/${!zfs_pool_name}/${!zfs_mount_point} ${!nfs_cidr_block}${!nfs_opts}"'" > /etc/exports'
            elif [[ "${!zfs_or_mdadm}" == "mdadm" ]]; then
            sudo su -c 'echo "'"/${!mdadm_raid_fs_name} ${!nfs_cidr_block}${!nfs_opts}"'" > /etc/exports'
            fi
            sudo systemctl enable rpcbind
            sudo systemctl enable nfs-server
            sudo systemctl start rpcbind
            sudo systemctl start nfs-server
            sudo systemctl start rpc-statd
            showmount -e localhost
          }

          function ck_status {

            if [[ "${!zfs_or_mdadm}" == "ZFS" ]]; then
            zfs_status=$(sudo zpool status | grep state | awk {'print $2'})
            sudo mount | grep ${!zfs_pool_name}
            let zfs_mount_rc=$?
            sudo showmount -e localhost | grep -v Export | grep ${!zfs_mount_point}
            let nfs_status_rc=$?
            if [[ "${!zfs_status}" == "ONLINE" ]] && [[ "${!nfs_status_rc}" -eq 0 ]] && [[ "${!zfs_mount_rc}" -eq 0 ]]; then
              echo "ZFS and NFS installed and configured"
              curl -X PUT -H 'Content-Type:' --data-binary '{ "Status" : "SUCCESS",  "Reason" : "ZFS and NFS installed and configured",  "UniqueId" : "ZFS001",  "Data" : "ZFS and NFS installed and configured."}' "${!my_wait_handle}"
            fi
            elif [[ "${!zfs_or_mdadm}" == "mdadm" ]]; then
            mdadm_status=$(sudo mdadm --detail /dev/md0 | grep "State :" | awk {'print $NF'})
            sudo mount | grep ${!mdadm_fs_name}
            let mdadm_mount_rc=$?
            sudo showmount -e localhost | grep -v Export | grep ${!mdadm_fs_name}
            let nfs_status_rc=$?
            if [[ "${!mdadm_status}" == "clean" ]] && [[ "${!nfs_status_rc}" -eq 0 ]] && [[ "${!mdadm_mount_rc}" -eq 0 ]]; then
              echo "mdadm raid and NFS installed and configured"
              curl -X PUT -H 'Content-Type:' --data-binary '{ "Status" : "SUCCESS",  "Reason" : "mdadm raid and NFS installed and configured",  "UniqueId" : "ZFS001",  "Data" : "mdadm raid and NFS installed and configured."}'   "${!my_wait_handle}"
            fi
            fi
          }

          declare -A instStore_devs
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



workshops/eda-workshop-soca/templates/nfs_server_instanceStore_zfs_soca.yaml [323:756]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      IamInstanceProfile: !Ref 'RootInstanceProfile'
      ImageId: !FindInMap
        - AWSRegionAMI
        - !Ref 'AWS::Region'
        - !Ref 'OperatingSystem'
      InstanceType: !Ref 'NFSInstanceType'
      KeyName: !Ref 'AdminKeyPair'
      NetworkInterfaces:
        - AssociatePublicIpAddress: !Ref 'UsePublicIp'
          DeleteOnTermination: true
          DeviceIndex: '0'
          GroupSet:
            - !Ref 'ExistingSecurityGroup'
          PrivateIpAddress: !If
            - Has_Static_Private_IP
            - !Ref 'StaticPrivateIpAddress'
            - !Ref 'AWS::NoValue'
          SubnetId: !Ref Subnet
      PlacementGroupName: !If
        - no_placement_group
        - !Ref 'AWS::NoValue'
        - !Ref 'ExistingPlacementGroup'
      Tags:
        - 
          Key: "Name"
          Value: "NFS Server"

      UserData: !Base64
        Fn::Sub: |
          #!/usr/bin/env bash

          set -x
          exec > >(tee /var/log/user-data.log|logger -t user-data ) 2>&1

          echo "*** BEGIN NFS SERVER BOOTSTRAP ***"

          nfs_instance_type="${NFSInstanceType}"
          nfs_os="${OperatingSystem}"
          nfs_cidr_block="${NFSCidr}"
          nfs_opts="${NFSOpts}"
          my_wait_handle="${NFSInstanceWaitHandle}"
          volume_type="${VolumeType}"
          hypervisor_type="nitro"
          zfs_or_mdadm="${ZfsOrMdadm}"

          zfs_pool_name="${ZfsPool}"
          zfs_mount_point="${ZfsMountPoint}"
          use_l2arc="${UseL2arc}"
          use_zil="${UseZil}"
          #@IgnoreInspection BashAddShebang

          #
          # Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
          #
          # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file
          # except in compliance with the License. A copy of the License is located at
          #
          #  http://aws.amazon.com/apache2.0/
          #
          # or in the "license" file accompanying this file. This file is distributed on an "AS IS"
          # BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
          # License for the specific language governing permissions and limitations under the License.
          #

          function inst_pkgs {
            echo "Installing packages and updates"
            sudo yum install nfs-utils rpcbind vim nvme-cli mdadm -y
            #sudo yum update -y
            echo "Updates complete"
          }

          function inst_ssm {
            echo "Installing and starting AWS Systems Manager (SSM) agent"
            sudo yum install -q -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
            systemctl enable amazon-ssm-agent
            systemctl start amazon-ssm-agent
          }

          function inst_cwagent {
            echo "Installing CloudWatch agent"
            sudo yum install -q -y https://s3.amazonaws.com/amazoncloudwatch-agent/centos/amd64/latest/amazon-cloudwatch-agent.rpm
            sudo su -c 'echo -e "{
                    \"agent\": {
                            \"metrics_collection_interval\": 60,
                            \"run_as_user\": \"root\"
                    },
                    \"metrics\": {
                            \"append_dimensions\": {
                                    \"InstanceId\": \"$\{aws:InstanceId\}\",
                                    \"InstanceType\": \"$\{aws:InstanceType\}\"
                            },
                            \"metrics_collected\": {
                                    \"cpu\": {
                                            \"measurement\": [
                                                    \"cpu_usage_idle\",
                                                    \"cpu_usage_iowait\",
                                                    \"cpu_usage_user\",
                                                    \"cpu_usage_system\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"totalcpu\": true
                                    },
                                    \"disk\": {
                                            \"measurement\": [
                                                    \"used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"resources\": [
                                                    \"nvme1n1\",
                                                    \"nvme2n1\",
                                                    \"nvme3n1\",
                                                    \"nvme4n1\",
                                                    \"nvme5n1\",
                                                    \"nvme6n1\",
                                                    \"nvme7n1\",
                                                    \"nvme8n1\"
                                            ]
                                    },
                                    \"diskio\": {
                                            \"measurement\": [
                                                    \"io_time\",
                                                    \"iops_in_progress\",
                                                    \"write_bytes\",
                                                    \"write_time\",
                                                    \"read_bytes\",
                                                    \"read_time\",
                                                    \"writes\",
                                                    \"reads\"
                                            ],
                                            \"metrics_collection_interval\": 60,
                                            \"resources\": [
                                                    \"nvme1n1\",
                                                    \"nvme2n1\",
                                                    \"nvme3n1\",
                                                    \"nvme4n1\",
                                                    \"nvme5n1\",
                                                    \"nvme6n1\",
                                                    \"nvme7n1\",
                                                    \"nvme8n1\"
                                            ]
                                    },
                                    \"mem\": {
                                            \"measurement\": [
                                                    \"mem_used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    },
                                    \"netstat\": {
                                            \"measurement\": [
                                                    \"tcp_established\",
                                                    \"tcp_time_wait\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    },
                                    \"swap\": {
                                            \"measurement\": [
                                                    \"swap_used_percent\"
                                            ],
                                            \"metrics_collection_interval\": 60
                                    }
                            }
                    }
            }" >> /opt/aws/amazon-cloudwatch-agent/bin/config.json'
            sudo sed -i 's/\(Instance.*\)\\{\(.*\)\\}/\1{\2}/g' /opt/aws/amazon-cloudwatch-agent/bin/config.json
            echo "Starting CloudWatch agent"
            sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json -s
          }

          function sys_limits {
            echo "Updating sys limits"
            sudo su -c 'echo -e "
            fs.file-max = 2097152
            vm.swappiness = 1
            vm.dirty_ratio = 60
            vm.dirty_background_ratio = 2
            vm.dirty_bytes = 4294967296
            vm.dirty_background_bytes = 1073741824
            vm.min_free_kbytes = 1048576
            net.core.rmem_default = 31457280
            net.core.rmem_max = 33554432
            net.core.wmem_default = 31457280
            net.core.wmem_max = 33554432
            net.core.somaxconn = 65535
            net.core.netdev_max_backlog = 250000
            net.core.optmem_max = 25165824
            net.ipv4.tcp_mem = 786432 1048576 26777216
            net.ipv4.udp_mem = 65536 131072 262144
            net.ipv4.tcp_rmem = 8192 87380 33554432
            net.ipv4.udp_rmem_min = 16384
            net.ipv4.tcp_wmem = 8192 65536 33554432
            net.ipv4.udp_wmem_min = 16384
            net.ipv4.tcp_sack = 1
            net.ipv4.tcp_dsack = 1
            net.ipv4.tcp_fack = 1
            net.ipv4.tcp_fin_timeout = 30
            " >> /etc/sysctl.conf'
            sudo sysctl -p
            sudo su -c 'echo -e "
            *   hard nofile 1048576
            *   soft nofile 1048576
            " >> /etc/security/limits.conf'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/rmem_default'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/rmem_max'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/wmem_default'
            sudo su -c 'echo 10485760 > /proc/sys/net/core/wmem_max'
          }

          function ck_devs_count_build_list {

            if [[ "$volume_type" == "EBS" ]]; then
            echo "Checking the total number of EBS volumes, currently ${!devs_count_ready} devices ready."
            devs_count_ready=0
            let dev_count=$(sudo nvme list | grep "${!nvme_grep_str}" | grep -v nvme0n1 | wc -l)
            if [[ $dev_count -eq ${!total_ebs_vols} ]]; then
              if [[ ! ${!dev_list} ]]; then
                for d in $(sudo nvme list | grep "${!nvme_grep_str}" | grep -v nvme0n1 | awk {'print $1'} | cut -f3 -d"/")
                  do echo -n "$d "
                  dev_list="${!dev_list} ${!d}"
                done
              fi
              devs_count_ready=1
            else
              devs_count_ready=0
            fi
            elif [[ "$volume_type" == "InstanceStore" ]]; then
            echo "Checking the total number of Instance Store NVMe volumes, currently ${!devs_count_ready} devices ready."
            devs_count_ready=0
            let dev_count=$(sudo nvme list | grep "${!nvme_grep_str}" | wc -l)
            if [[ $dev_count -eq ${!total_i_store_vols} ]]; then
              if [[ ! ${!dev_list} ]]; then
                for d in $(sudo nvme list | grep "${!nvme_grep_str}" | awk {'print $1'} | cut -f3 -d"/")
                  do echo -n "$d "
                  dev_list="${!dev_list} ${!d}"
                done
              fi
              devs_count_ready=1
            else
              devs_count_ready=0
            fi
            fi

          }

          timeout_count=0
          function ck_devs {
            devs_ready=0

            if [[ "$volume_type" == "EBS" ]]; then
            nvme_grep_str="Amazon Elastic Block Store"
            total_ebs_vols=8
            total_vols=${!total_ebs_vols}
            elif [[ "$volume_type" == "InstanceStore" ]]; then
            nvme_grep_str="Amazon EC2 NVMe Instance Storage"
            total_i_store_vols=${!instStore_devs["${!nfs_instance_type}"]}
            total_vols=${!total_i_store_vols}
            fi

            while [[ ! $(which nvme) ]]; do
            sleep 10
            done

            while [[ ! ${!devs_count_ready} ]]; do
            ck_devs_count_build_list
            sleep 10
            done

            echo "Using device list ${!dev_list}"

            while [[ ! ${!devs_ready} ]]; do
            for l in ${!dev_list}; do
              if [[ ! -b /dev/${!l} ]]; then
                devs_ready=0
                ((timeout_count = $timeout_count + 1))
                if [[ $timeout_count -gt 30 ]]; then
                  echo "All storage volumes are ready, exiting..."
                  exit
                else
                  sleep 10
                fi
              else
                devs_ready=1
                ck_devs
              fi
            done
            done
            echo "Devices ready."
          }

          function mk_raid {
            raid_level="0"
            full_path_dev_list=""
            for d in ${!dev_list}; do
            temp_d="/dev/${!d}"
            full_path_dev_list="${!full_path_dev_list} ${!temp_d}"
            done
            yes | sudo mdadm --create --verbose /dev/md0 --level=${!raid_level} --name=raid-vol1 --raid-devices=${!total_vols} ${!full_path_dev_list}
          }

          function mk_ext4_fs {
            mdadm_fs_dev="/dev/md0"
            mdadm_fs_name=${!mdadm_raid_fs_name}
            mdadm_fs_label=${!mdadm_raid_fs_name}
            sudo mkfs.ext4 -L ${!mdadm_fs_label} ${!mdadm_fs_dev} -E nodiscard
            sudo mkdir /${!mdadm_fs_name}
            sudo mount LABEL=${!mdadm_fs_label} /${!mdadm_fs_name}
            sudo su -c "echo 'LABEL=${!mdadm_fs_label}     /${!mdadm_fs_name}   ext4    defaults,nofail     0      2' >> /etc/fstab"
          }

          function fix_zfs_repo {
            echo "Fixing ZFS repo..."
            sudo cat /etc/yum.repos.d/zfs.repo  | while read l
            do
            if [[ "$l" == "[zfs]" ]]; then
              let on_zfs=1
            elif [[ "$l" == "[zfs-kmod]" ]]; then
              let on_zfs_kmod=1
            fi
            if [[ "$on_zfs" -eq 1 ]] && [[ "$l" == "enabled=1" ]]; then
              l="enabled=0"
              let on_zfs=0
            elif [[ "$on_zfs_kmod" -eq 1 ]] &&  [[ "$l" == "enabled=0" ]]; then
              l="enabled=1"
              let on_zfs_kmod=0
            fi
            echo $l
            done > /tmp/new_zfs.repo
            sudo cp /etc/yum.repos.d/zfs.repo /etc/yum.repos.d/zfs.repo.dist
            sudo mv /tmp/new_zfs.repo /etc/yum.repos.d/zfs.repo
            echo "Done fixing ZFS repo"

          }

          function zfs_install_centos7 {
            echo -n "Fixing, removing, and re-installing ZFS..."
            known_zfs_key="C93AFFFD9F3F7B03C310CEB6A9D5A1C0F14AB620"
            sudo yum -y remove zfs zfs-kmod spl spl-kmod libzfs2 libnvpair1 libuutil1 libzpool2 zfs-release
            sudo yum -y install http://download.zfsonlinux.org/epel/zfs-release.el7_7.noarch.rpm
            actual_zfs_key=$(gpg --quiet --with-fingerprint /etc/pki/rpm-gpg/RPM-GPG-KEY-zfsonlinux | grep "Key fingerprint" | cut -d"=" -f2 | tr -d ' ')
            echo "Checking keys..."
            if [[ "$known_zfs_key" != "$actual_zfs_key" ]]; then
            echo "ERROR: ZFS installation keys not valid!!!"
            echo "Exiting..."
            exit
            fi
            fix_zfs_repo
            sudo yum -y autoremove
            sudo yum -y clean metadata
            sudo yum -y install zfs
            echo "Done installing ZFS"
          }

          function zfs_install_amzn2 {
            echo -n "Fixing, removing, and re-installing ZFS..."
            known_zfs_key="C93AFFFD9F3F7B03C310CEB6A9D5A1C0F14AB620"
            sudo yum -y remove zfs zfs-kmod spl spl-kmod libzfs2 libnvpair1 libuutil1 libzpool2 zfs-release
            sudo yum -y install http://download.zfsonlinux.org/epel/zfs-release.el7_7.noarch.rpm
            actual_zfs_key=$(gpg --quiet --with-fingerprint /etc/pki/rpm-gpg/RPM-GPG-KEY-zfsonlinux | grep "Key fingerprint" | cut -d"=" -f2 | tr -d ' ')
            echo "Checking keys..."
            if [[ "$known_zfs_key" != "$actual_zfs_key" ]]; then
            echo "ERROR: ZFS installation keys not valid!!!"
            echo "Exiting..."
            exit
            fi
            sudo yum -y autoremove
            sudo yum -y clean metadata
            sudo yum -y install dkms "kernel-devel-uname-r == $(uname -r)" zfs
            echo "Done installing ZFS"
          }

          function zfs_create {

            echo "Using these devices for ZFS:"
            sudo nvme list | grep "Amazon Elastic Block Store" | grep -v nvme0n1

            sudo /sbin/modprobe zfs
            modinfo zfs
            sudo zpool create -O compression=lz4 -O atime=off -O sync=disabled -f ${!zfs_pool_name} -o ashift=12 ${!dev_list}
            sudo zpool status -v
            sudo zfs create ${!zfs_pool_name}/${!zfs_mount_point}
          }

          function zfs_startup {
            sudo systemctl enable zfs-import-cache
            sudo systemctl enable zfs-mount.service
            sudo systemctl enable zfs.target
          }

          function nfs_server_settings {
            sudo su -c 'echo -e "STATD_PORT=\"32765\"\nSTATD_OUTGOING_PORT=\"32766\"\nSTATDARG=\"-p 32765 -o 32766\"\nMOUNTD_PORT=\"32767\"\nRPCMOUNTDOPTS=\"-p 32767\"
          LOCKD_UDPPORT=\"32768\"\nLOCKD_TCPPORT=\"32768\"\nRQUOTAD_PORT=\"32769\"\nRQUOTAD=\"no\"\nRPCNFSDCOUNT=\"128\""' > /etc/sysconfig/nfs
          }

          function nfs_config {
            nfs_server_settings

            if [[ "${!zfs_or_mdadm}" == "ZFS" ]]; then
            sudo su -c 'echo "'"/${!zfs_pool_name}/${!zfs_mount_point} ${!nfs_cidr_block}${!nfs_opts}"'" > /etc/exports'
            elif [[ "${!zfs_or_mdadm}" == "mdadm" ]]; then
            sudo su -c 'echo "'"/${!mdadm_raid_fs_name} ${!nfs_cidr_block}${!nfs_opts}"'" > /etc/exports'
            fi
            sudo systemctl enable rpcbind
            sudo systemctl enable nfs-server
            sudo systemctl start rpcbind
            sudo systemctl start nfs-server
            sudo systemctl start rpc-statd
            showmount -e localhost
          }

          function ck_status {

            if [[ "${!zfs_or_mdadm}" == "ZFS" ]]; then
            zfs_status=$(sudo zpool status | grep state | awk {'print $2'})
            sudo mount | grep ${!zfs_pool_name}
            let zfs_mount_rc=$?
            sudo showmount -e localhost | grep -v Export | grep ${!zfs_mount_point}
            let nfs_status_rc=$?
            if [[ "${!zfs_status}" == "ONLINE" ]] && [[ "${!nfs_status_rc}" -eq 0 ]] && [[ "${!zfs_mount_rc}" -eq 0 ]]; then
              echo "ZFS and NFS installed and configured"
              curl -X PUT -H 'Content-Type:' --data-binary '{ "Status" : "SUCCESS",  "Reason" : "ZFS and NFS installed and configured",  "UniqueId" : "ZFS001",  "Data" : "ZFS and NFS installed and configured."}' "${!my_wait_handle}"
            fi
            elif [[ "${!zfs_or_mdadm}" == "mdadm" ]]; then
            mdadm_status=$(sudo mdadm --detail /dev/md0 | grep "State :" | awk {'print $NF'})
            sudo mount | grep ${!mdadm_fs_name}
            let mdadm_mount_rc=$?
            sudo showmount -e localhost | grep -v Export | grep ${!mdadm_fs_name}
            let nfs_status_rc=$?
            if [[ "${!mdadm_status}" == "clean" ]] && [[ "${!nfs_status_rc}" -eq 0 ]] && [[ "${!mdadm_mount_rc}" -eq 0 ]]; then
              echo "mdadm raid and NFS installed and configured"
              curl -X PUT -H 'Content-Type:' --data-binary '{ "Status" : "SUCCESS",  "Reason" : "mdadm raid and NFS installed and configured",  "UniqueId" : "ZFS001",  "Data" : "mdadm raid and NFS installed and configured."}'   "${!my_wait_handle}"
            fi
            fi
          }

          declare -A instStore_devs
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



