AWSTemplateFormatVersion: '2010-09-09'
Description: >
  **WARNING** This template creates EC2
  instances, Cloud9 environment and related
  resources. You will be billed for the AWS
  resources used if you create a stack from
  this template.
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "VPC and subnets configuration"
        Parameters:
          - VPCCidr
          - PublicSubnetCidr
          - PrivateSubnetFirstCidr
          - PrivateSubnetSecondCidr
          - PrivateSubnetThirdCidr
      - Label:
          default: "EC2 instances configuration"
        Parameters:
          - EC2KeyPairName
          - LatestAmiId
      - Label:
          default: "Cross Account IAM Role Arn"
        Parameters:
          - CrossAccountGlueSchemaRegistryRoleArn
    ParameterLabels:
      VPCCidr:
        default: "VPC CIDR address"
      PublicSubnetCidr:
        default: "Public subnet CIDR address"
      PrivateSubnetFirstCidr:
        default: "First private subnet CIDR address"
      PrivateSubnetSecondCidr:
        default: "Second private subnet CIDR address"
      PrivateSubnetThirdCidr:
        default: "Third private subnet CIDR address"
      EC2KeyPairName:
        default: "EC2 Keypair to enable SSH access to the instances created through this template"
      LatestAmiId:
        default: "Latest EC2 AMI id"
      CrossAccountGlueSchemaRegistryRoleArn:
        default: >
          IAM Role Arn from schema registry AWS Account. Role created by the CloudFormation template in a separate AWS Account.
        Type: String
Parameters:
  VPCCidr:
    Type: String
    Default: '10.0.0.0/16'
  PublicSubnetCidr:
    Type: String
    Default: '10.0.0.0/24'
  PrivateSubnetFirstCidr:
    Type: String
    Default: '10.0.1.0/24'
  PrivateSubnetSecondCidr:
    Type: String
    Default: '10.0.2.0/24'
  PrivateSubnetThirdCidr:
    Type: String
    Default: '10.0.3.0/24'
  EC2KeyPairName:
    Type: AWS::EC2::KeyPair::KeyName
    ConstraintDescription: Can contain only ASCII characters.
  LatestAmiId:
    Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2'
  CrossAccountGlueSchemaRegistryRoleArn:
    Type: String
Resources:
  MSKVPCStack:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: "https://aws-bigdata-blog.s3.amazonaws.com/artifacts/bdb-1801-cloudformation/cfn-vpc.yaml"
      Parameters:
        VPCCidr: !Ref VPCCidr
        PublicSubnetCidr: !Ref PublicSubnetCidr
        PrivateSubnetFirstCidr: !Ref PrivateSubnetFirstCidr
        PrivateSubnetSecondCidr: !Ref PrivateSubnetSecondCidr
        PrivateSubnetThirdCidr: !Ref PrivateSubnetThirdCidr
  KafkaProducerConsumerRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: "/"
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonMSKReadOnlyAccess
        - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess
      Policies:
        - PolicyName: allow-cross-account-gsr-access-policy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: "sts:AssumeRole"
                Resource: !Ref 'CrossAccountGlueSchemaRegistryRoleArn'
  EC2InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref 'KafkaProducerConsumerRole'
  KafkaClientInstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Enable SSH access via port 22 from BastionHostSecurityGroup
      VpcId: !GetAtt MSKVPCStack.Outputs.VPCId
      SecurityGroupIngress:
      - Description: ssh access from Cloud9 environment in VPC public subnet
        IpProtocol: tcp
        FromPort: 22
        ToPort: 22
        CidrIp: !Ref PublicSubnetCidr
      SecurityGroupEgress:
      - Description: all egress allowed
        IpProtocol: -1
        CidrIp: 0.0.0.0/0
  Cloud9EC2Bastion:
    Type: AWS::Cloud9::EnvironmentEC2
    Properties: 
      AutomaticStopTimeMinutes: 600
      Description: "Cloud9 EC2 environment"
      InstanceType: m5.large
      SubnetId: !GetAtt MSKVPCStack.Outputs.PublicSubnetOne
      Tags:
        - Key: 'Purpose'
          Value: 'Cloud9EC2BastionHostInstance'
  KafkaProducerEC2Instance:
    Type: AWS::EC2::Instance
    Properties:
      InstanceType: m5.large
      KeyName: !Ref 'EC2KeyPairName'
      IamInstanceProfile: !Ref EC2InstanceProfile
      AvailabilityZone: 
        Fn::Select:
         - 0
         - Fn::GetAZs: {Ref: 'AWS::Region'}
      SubnetId: !GetAtt MSKVPCStack.Outputs.PrivateSubnetMSKOne
      SecurityGroupIds: [!GetAtt KafkaClientInstanceSecurityGroup.GroupId]
      ImageId: !Ref LatestAmiId
      Tags:
        - Key: 'Name'
          Value: 'KafkaProducerInstance'
      UserData: 
        Fn::Base64: 
          !Sub |
            #!/bin/bash
            yum update -y
            yum install python3.7 -y
            yum install java-1.8.0-openjdk-devel -y
            yum install nmap-ncat -y
            yum install git -y
            yum erase awscli -y
            yum install jq -y
            amazon-linux-extras install docker -y
            service docker start
            usermod -a -G docker ec2-user

            cd /home/ec2-user
            wget https://bootstrap.pypa.io/get-pip.py
            su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user
            su -c "/home/ec2-user/.local/bin/pip3 install awscli --user" -s /bin/sh ec2-user

            # install AWS CLI 2 - access with aws2
            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            ./aws/install -b /usr/local/bin/aws2
            su -c "ln -s /usr/local/bin/aws2/aws ~/.local/bin/aws2" -s /bin/sh ec2-user

            # Create dirs, get Apache Kafka 2.7.1 and unpack it
            su -c "mkdir -p kafka271" -s /bin/sh ec2-user
            cd /home/ec2-user
            ln -s /home/ec2-user/kafka271 /home/ec2-user/kafka
            cd kafka271
            su -c "wget https://archive.apache.org/dist/kafka/2.7.1/kafka_2.13-2.7.1.tgz" -s /bin/sh ec2-user
            su -c "tar -xzf kafka_2.13-2.7.1.tgz --strip 1" -s /bin/sh ec2-user

            # Download Kafka Producer code on this EC2 instance
            su -c "aws s3 cp s3://aws-bigdata-blog/artifacts/bdb-1801-cloudformation/kafka-cross-account-gsr-producer.jar /home/ec2-user" -l ec2-user
  KafkaConsumerEC2Instance:
    Type: AWS::EC2::Instance
    Properties:
      InstanceType: m5.large
      KeyName: !Ref 'EC2KeyPairName'
      IamInstanceProfile: !Ref EC2InstanceProfile
      AvailabilityZone:
        Fn::Select:
          - 1
          - Fn::GetAZs: { Ref: 'AWS::Region' }
      SubnetId: !GetAtt MSKVPCStack.Outputs.PrivateSubnetMSKTwo
      SecurityGroupIds: [ !GetAtt KafkaClientInstanceSecurityGroup.GroupId ]
      ImageId: !Ref LatestAmiId
      Tags:
        - Key: 'Name'
          Value: 'KafkaConsumerInstance'
      UserData:
        Fn::Base64:
          !Sub |
          #!/bin/bash
          yum update -y
          yum install python3.7 -y
          yum install java-1.8.0-openjdk-devel -y
          yum install nmap-ncat -y
          yum install git -y
          yum erase awscli -y
          yum install jq -y
          amazon-linux-extras install docker -y
          service docker start
          usermod -a -G docker ec2-user

          cd /home/ec2-user
          wget https://bootstrap.pypa.io/get-pip.py
          su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user
          su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user
          su -c "/home/ec2-user/.local/bin/pip3 install awscli --user" -s /bin/sh ec2-user

          # install AWS CLI 2 - access with aws2
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          ./aws/install -b /usr/local/bin/aws2
          su -c "ln -s /usr/local/bin/aws2/aws ~/.local/bin/aws2" -s /bin/sh ec2-user

          # Create dirs, get Apache Kafka 2.7.1 and unpack it
          su -c "mkdir -p kafka271" -s /bin/sh ec2-user
          cd /home/ec2-user
          ln -s /home/ec2-user/kafka271 /home/ec2-user/kafka
          cd kafka271
          su -c "wget https://archive.apache.org/dist/kafka/2.7.1/kafka_2.13-2.7.1.tgz" -s /bin/sh ec2-user
          su -c "tar -xzf kafka_2.13-2.7.1.tgz --strip 1" -s /bin/sh ec2-user

          # Download Kafka Consumer code on this EC2 instance
          su -c "aws s3 cp s3://aws-bigdata-blog/artifacts/bdb-1801-cloudformation/kafka-cross-account-gsr-consumer.jar /home/ec2-user" -l ec2-user
Outputs:
  VPCId:
    Description: The ID of the VPC created
    Value: !GetAtt MSKVPCStack.Outputs.VPCId
    Export:
      Name: !Sub "${AWS::StackName}-VPCId"
  PublicSubnetCidr:
    Description: Public subnet CIDR address
    Value: !GetAtt MSKVPCStack.Outputs.PublicSubnetCidr
    Export:
      Name: !Sub "${AWS::StackName}-PublicSubnetCidr"
  PublicSubnetOne:
    Description: The ID of the public subnet created
    Value: !GetAtt MSKVPCStack.Outputs.PublicSubnetOne
    Export:
      Name: !Sub "${AWS::StackName}-PublicSubnetOne"
  PrivateSubnetMSKOne:
    Description: The ID of private subnet one created
    Value: !GetAtt MSKVPCStack.Outputs.PrivateSubnetMSKOne
    Export:
      Name: !Sub "${AWS::StackName}-PrivateSubnetMSKOne"
  PrivateSubnetMSKTwo:
    Description: The ID of private subnet two created
    Value: !GetAtt MSKVPCStack.Outputs.PrivateSubnetMSKTwo
    Export:
      Name: !Sub "${AWS::StackName}-PrivateSubnetMSKTwo"
  PrivateSubnetMSKThree:
    Description: The ID of private subnet three created
    Value: !GetAtt MSKVPCStack.Outputs.PrivateSubnetMSKThree
    Export:
      Name: !Sub "${AWS::StackName}-PrivateSubnetMSKThree"
  KafkaClientInstanceSecurityGroup:
    Description: Security Group associated with Kafka Client Instance
    Value: !GetAtt KafkaClientInstanceSecurityGroup.GroupId
    Export:
      Name: !Sub "${AWS::StackName}-KafkaClientEC2InstanceSecurityGroupId"