CFTemplates/MSKBastionKafkaClientCluster.yml [101:206]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      UserData: 
        Fn::Base64: 
          !Sub |
            #!/bin/bash
            yum update -y
            yum install python3.7 -y
            yum install java-1.8.0-openjdk-devel -y
            yum install nmap-ncat -y
            yum install git -y
            yum erase awscli -y
            yum install jq -y
            yum install maven -y
            amazon-linux-extras install docker -y
            service docker start
            usermod -a -G docker ec2-user

            cd /home/ec2-user
            wget https://bootstrap.pypa.io/get-pip.py
            su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install awscli --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install kafka-python --user" -s /bin/sh ec2-user

            # install AWS CLI 2 - access with aws2
            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            ./aws/install -b /usr/local/bin/aws2
            su -c "ln -s /usr/local/bin/aws2/aws ~/.local/bin/aws2" -s /bin/sh ec2-user

            # Create dirs, get Apache Kafka 2.3.1, 2.4.0 and unpack it
            su -c "mkdir -p kafka231 kafka241 confluent" -s /bin/sh ec2-user
            cd kafka231
            su -c "wget https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz" -s /bin/sh ec2-user
            su -c "tar -xzf kafka_2.12-2.3.1.tgz --strip 1" -s /bin/sh ec2-user

            cd /home/ec2-user
            ln -s /home/ec2-user/kafka241 /home/ec2-user/kafka
            cd kafka241
            su -c "wget http://mirrors.gigenet.com/apache/kafka/2.4.1/kafka_2.12-2.4.1.tgz" -s /bin/sh ec2-user
            su -c "tar -xzf kafka_2.12-2.4.1.tgz --strip 1" -s /bin/sh ec2-user

            # Get Confluent Community and unpack it
            cd /home/ec2-user
            cd confluent
            su -c "wget http://packages.confluent.io/archive/5.4/confluent-community-5.4.1-2.12.tar.gz" -s /bin/sh ec2-user
            su -c "tar -xzf confluent-community-5.4.1-2.12.tar.gz --strip 1" -s /bin/sh ec2-user
            
            # Initialize the Kafka cert trust store
            su -c 'find /usr/lib/jvm/ -name "cacerts" -exec cp {} /tmp/kafka.client.truststore.jks \;' -s /bin/sh ec2-user

            # Setup prometheus agent
            cd /home/ec2-user
            su -c "mkdir prometheus" -s /bin/sh ec2-user
            cd prometheus
            su -c "wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar" -s /bin/sh ec2-user

            # Copy files from S3
            cd /tmp
            su -c "mkdir -p kafka" -s /bin/sh ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/producer.properties_msk /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/consumer.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/confluent/share/java/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/MM2GroupOffsetSync-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/MSKLabs/schema-registry-ssl/schema-registry.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/generatePropertiesFiles.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/generateStartupFile.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/connect-distributed.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-consumer-python.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/setup-env.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/GlobalSeqNo.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-msc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-hbc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-cpc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-cpc-cust-repl-policy.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-msc-cust-repl-policy.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-connect.yml /home/ec2-user/prometheus" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-producer-consumer.yml /home/ec2-user/prometheus" -l ec2-user

            # Setup unit in systemd for Schema Registry
            echo -n "
            [Unit]
            Description=Confluent Schema Registry
            After=network.target

            [Service]
            Type=simple
            User=ec2-user
            ExecStart=/bin/sh -c '/home/ec2-user/confluent/bin/schema-registry-start /tmp/kafka/schema-registry.properties > /tmp/kafka/schema-registry.log 2>&1'
            ExecStop=/home/ec2-user/confluent/bin/schema-registry-stop
            Restart=on-abnormal

            [Install]
            WantedBy=multi-user.target" > /etc/systemd/system/confluent-schema-registry.service

            # Setup unit in systemd for Kafka Connect
            echo -n "
            [Unit]
            Description=Kafka Connect
            After=network.target

            [Service]
            Type=simple
            User=ec2-user
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



CFTemplates/MSKBastionKafkaClientCluster.yml [234:340]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      UserData: 
        Fn::Base64: 
          !Sub | 
            #!/bin/bash
            yum update -y
            yum install python3.7 -y
            yum install java-1.8.0-openjdk-devel -y
            yum install nmap-ncat -y
            yum install git -y
            yum erase awscli -y
            yum install jq -y
            yum install maven -y
            amazon-linux-extras install docker -y
            service docker start
            usermod -a -G docker ec2-user

            cd /home/ec2-user
            wget https://bootstrap.pypa.io/get-pip.py
            su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install awscli --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install kafka-python --user" -s /bin/sh ec2-user

            # install AWS CLI 2 - access with aws2
            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            ./aws/install -b /usr/local/bin/aws2
            su -c "ln -s /usr/local/bin/aws2/aws ~/.local/bin/aws2" -s /bin/sh ec2-user

            # Create dirs, get Apache Kafka 2.3.1, 2.4.0 and unpack it
            su -c "mkdir -p kafka231 kafka241 confluent" -s /bin/sh ec2-user
            cd kafka231
            su -c "wget https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz" -s /bin/sh ec2-user
            su -c "tar -xzf kafka_2.12-2.3.1.tgz --strip 1" -s /bin/sh ec2-user

            cd /home/ec2-user
            ln -s /home/ec2-user/kafka241 /home/ec2-user/kafka
            cd kafka241
            su -c "wget http://mirrors.gigenet.com/apache/kafka/2.4.1/kafka_2.12-2.4.1.tgz" -s /bin/sh ec2-user
            su -c "tar -xzf kafka_2.12-2.4.1.tgz --strip 1" -s /bin/sh ec2-user

            # Get Confluent Community and unpack it
            cd /home/ec2-user
            cd confluent
            su -c "wget http://packages.confluent.io/archive/5.4/confluent-community-5.4.1-2.12.tar.gz" -s /bin/sh ec2-user
            su -c "tar -xzf confluent-community-5.4.1-2.12.tar.gz --strip 1" -s /bin/sh ec2-user
            
            # Initialize the Kafka cert trust store
            su -c 'find /usr/lib/jvm/ -name "cacerts" -exec cp {} /tmp/kafka.client.truststore.jks \;' -s /bin/sh ec2-user

            # Setup prometheus agent
            cd /home/ec2-user
            su -c "mkdir prometheus" -s /bin/sh ec2-user
            cd prometheus
            su -c "wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar" -s /bin/sh ec2-user


            # Copy files from S3
            cd /tmp
            su -c "mkdir -p kafka" -s /bin/sh ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/producer.properties_msk /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/consumer.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/confluent/share/java/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/jars/MM2GroupOffsetSync-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/MSKLabs/schema-registry-ssl/schema-registry.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/generatePropertiesFiles.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/generateStartupFile.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/connect-distributed.properties /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-consumer-python.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/setup-env.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/GlobalSeqNo.py /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-msc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-hbc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-cpc.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-cpc-cust-repl-policy.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/mm2-msc-cust-repl-policy.json /tmp/kafka" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-connect.yml /home/ec2-user/prometheus" -l ec2-user
            su -c "aws s3 cp s3://reinvent2019-msk-liftandshift/kafka-producer-consumer.yml /home/ec2-user/prometheus" -l ec2-user

            # Setup unit in systemd for Schema Registry
            echo -n "
            [Unit]
            Description=Confluent Schema Registry
            After=network.target

            [Service]
            Type=simple
            User=ec2-user
            ExecStart=/bin/sh -c '/home/ec2-user/confluent/bin/schema-registry-start /tmp/kafka/schema-registry.properties > /tmp/kafka/schema-registry.log 2>&1'
            ExecStop=/home/ec2-user/confluent/bin/schema-registry-stop
            Restart=on-abnormal

            [Install]
            WantedBy=multi-user.target" > /etc/systemd/system/confluent-schema-registry.service

            # Setup unit in systemd for Kafka Connect
            echo -n "
            [Unit]
            Description=Kafka Connect
            After=network.target

            [Service]
            Type=simple
            User=ec2-user
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



