AWSTemplateFormatVersion: 2010-09-09 Parameters: KeyName: Description: Name of an existing EC2 KeyPair to enable SSH access to the instance Type: 'AWS::EC2::KeyPair::KeyName' ConstraintDescription: Can contain only ASCII characters. SSHLocation: Description: The IP address range that can be used to SSH to the EC2 instances Type: String MinLength: '9' MaxLength: '18' Default: 0.0.0.0/0 AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x Mappings: SubnetConfig: VPC: CIDR: 10.0.0.0/16 PublicOne: CIDR: 10.0.0.0/24 PrivateOne: CIDR: 10.0.1.0/24 PrivateTwo: CIDR: 10.0.2.0/24 PrivateThree: CIDR: 10.0.3.0/24 RegionAMI: us-east-1: HVM64: ami-0c6b1d09930fac512 us-west-2: HVM64: ami-0cb72367e98845d43 Resources: VPC: Type: 'AWS::EC2::VPC' Properties: EnableDnsSupport: true EnableDnsHostnames: true CidrBlock: !FindInMap - SubnetConfig - VPC - CIDR Tags: - Key: Name Value: MMVPC Metadata: 'AWS::CloudFormation::Designer': id: e191108b-e241-404a-bed2-b2863bcd5235 PublicSubnetOne: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: !Select - 0 - !GetAZs Ref: 'AWS::Region' VpcId: !Ref VPC CidrBlock: !FindInMap - SubnetConfig - PublicOne - CIDR MapPublicIpOnLaunch: true Tags: - Key: Name Value: MMPublicSubnet Metadata: 'AWS::CloudFormation::Designer': id: 90cff42d-8429-4a50-b584-5a2ba16651ea PrivateSubnetOne: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: !Select - 0 - !GetAZs Ref: 'AWS::Region' VpcId: !Ref VPC CidrBlock: !FindInMap - SubnetConfig - PrivateOne - CIDR Tags: - Key: Name Value: MMPrivateSubnetOne Metadata: 'AWS::CloudFormation::Designer': id: be121547-ef38-4d55-a9a9-f67eb35c9cd0 PrivateSubnetTwo: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: !Select - 1 - !GetAZs Ref: 'AWS::Region' VpcId: !Ref VPC CidrBlock: !FindInMap - SubnetConfig - PrivateTwo - CIDR Tags: - Key: Name Value: MMPrivateSubnetTwo Metadata: 'AWS::CloudFormation::Designer': id: 1a1c22e5-745b-443d-93d6-78c3572de3c4 PrivateSubnetThree: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: !Select - 2 - !GetAZs Ref: 'AWS::Region' VpcId: !Ref VPC CidrBlock: !FindInMap - SubnetConfig - PrivateThree - CIDR Tags: - Key: Name Value: MMPrivateSubnetThree Metadata: 'AWS::CloudFormation::Designer': id: 69ad49b5-ffb5-4019-82dd-1b5433531511 InternetGateway: Type: 'AWS::EC2::InternetGateway' Metadata: 'AWS::CloudFormation::Designer': id: d2a821e3-5c45-46a4-92fd-09268f72f679 GatewayAttachement: Type: 'AWS::EC2::VPCGatewayAttachment' Properties: VpcId: !Ref VPC InternetGatewayId: !Ref InternetGateway Metadata: 'AWS::CloudFormation::Designer': id: 988bff04-9cdd-4ce1-8b5d-59fc3291eae1 PublicRouteTable: Type: 'AWS::EC2::RouteTable' Properties: VpcId: !Ref VPC Metadata: 'AWS::CloudFormation::Designer': id: 93747341-451f-4c6e-8265-6873b110ae9e PublicRoute: Type: 'AWS::EC2::Route' DependsOn: GatewayAttachement Properties: RouteTableId: !Ref PublicRouteTable DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway Metadata: 'AWS::CloudFormation::Designer': id: 412c8444-e8b8-43eb-8c07-3f822c921e34 PublicSubnetOneRouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: SubnetId: !Ref PublicSubnetOne RouteTableId: !Ref PublicRouteTable Metadata: 'AWS::CloudFormation::Designer': id: c2c64928-4fe4-4640-a449-08cdc9e50d84 PrivateRouteTable: Type: 'AWS::EC2::RouteTable' Properties: VpcId: !Ref VPC Metadata: 'AWS::CloudFormation::Designer': id: 7a45b627-d06a-4956-9f4b-a79b5e65bc7d PrivateSubnetOneRouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PrivateRouteTable SubnetId: !Ref PrivateSubnetOne Metadata: 'AWS::CloudFormation::Designer': id: 912ef118-eda5-437f-afd7-3f92d83fd70b PrivateSubnetTwoRouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PrivateRouteTable SubnetId: !Ref PrivateSubnetTwo Metadata: 'AWS::CloudFormation::Designer': id: d345d815-288f-4446-add0-dd637cc6c57f PrivateSubnetThreeRouteTableAssociation: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: RouteTableId: !Ref PrivateRouteTable SubnetId: !Ref PrivateSubnetThree Metadata: 'AWS::CloudFormation::Designer': id: 4f6ec5cd-f063-4163-8494-31bd55bcb27f KafkaClientInstanceSecurityGroup: Type: 'AWS::EC2::SecurityGroup' Properties: GroupDescription: Enable SSH access via port 22 VpcId: !Ref VPC SecurityGroupIngress: - IpProtocol: tcp FromPort: 22 ToPort: 22 CidrIp: !Ref SSHLocation Metadata: 'AWS::CloudFormation::Designer': id: 758d2702-7bfd-470e-84a0-5101471dd251 MSKSecurityGroup: Type: 'AWS::EC2::SecurityGroup' Properties: GroupDescription: Enable SSH access via port 22 VpcId: !Ref VPC SecurityGroupIngress: - IpProtocol: tcp FromPort: 2181 ToPort: 2181 SourceSecurityGroupId: !GetAtt - KafkaClientInstanceSecurityGroup - GroupId - IpProtocol: tcp FromPort: 9094 ToPort: 9094 SourceSecurityGroupId: !GetAtt - KafkaClientInstanceSecurityGroup - GroupId - IpProtocol: tcp FromPort: 9092 ToPort: 9092 SourceSecurityGroupId: !GetAtt - KafkaClientInstanceSecurityGroup - GroupId Metadata: 'AWS::CloudFormation::Designer': id: 20d8d324-9da6-49f9-9e37-4bd433625c47 KafkaClientEC2Instance: Type: 'AWS::EC2::Instance' Properties: InstanceType: m5.2xlarge KeyName: !Ref KeyName IamInstanceProfile: !Ref EC2InstanceProfile AvailabilityZone: !Select - 0 - !GetAZs Ref: 'AWS::Region' SubnetId: !Ref PublicSubnetOne SecurityGroupIds: - !GetAtt - KafkaClientInstanceSecurityGroup - GroupId ImageId: !FindInMap - RegionAMI - !Ref 'AWS::Region' - HVM64 Tags: - Key: Name Value: KafkaClientInstance UserData: !Base64 > #!/bin/bash yum update -y yum install python3.7 -y yum install java-1.8.0-openjdk-devel -y yum erase awscli -y cd /home/ec2-user echo "export PATH=.local/bin:$PATH" >> .bash_profile mkdir kafka mkdir mm cd kafka wget https://archive.apache.org/dist/kafka/2.4.0/kafka_2.12-2.4.0.tgz tar -xzf kafka_2.12-2.4.0.tgz cd /home/ec2-user wget https://bootstrap.pypa.io/get-pip.py su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install awscli --user" -s /bin/sh ec2-user chown -R ec2-user ./kafka chgrp -R ec2-user ./kafka chown -R ec2-user ./mm chgrp -R ec2-user ./mm Metadata: 'AWS::CloudFormation::Designer': id: 25e7995f-31fc-45b0-b790-42bcb519905a EC2Role: Type: 'AWS::IAM::Role' Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Sid: '' Effect: Allow Principal: Service: ec2.amazonaws.com Action: 'sts:AssumeRole' Path: / ManagedPolicyArns: - 'arn:aws:iam::aws:policy/AmazonMSKFullAccess' - 'arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess' Metadata: 'AWS::CloudFormation::Designer': id: 268a951d-285d-438a-a365-d135a51bd01c EC2InstanceProfile: Type: 'AWS::IAM::InstanceProfile' Properties: InstanceProfileName: EC2MSKCFProfile Roles: - !Ref EC2Role Metadata: 'AWS::CloudFormation::Designer': id: 01de5e28-2175-45f1-a37f-65e5bd7d9356 MSKCluster: Type: 'AWS::MSK::Cluster' Properties: BrokerNodeGroupInfo: ClientSubnets: - !Ref PrivateSubnetOne - !Ref PrivateSubnetTwo - !Ref PrivateSubnetThree InstanceType: kafka.m5.large SecurityGroups: - !GetAtt - MSKSecurityGroup - GroupId StorageInfo: EBSStorageInfo: VolumeSize: 2000 ClusterName: MSKCluster EncryptionInfo: EncryptionInTransit: ClientBroker: TLS_PLAINTEXT InCluster: true EnhancedMonitoring: PER_TOPIC_PER_BROKER KafkaVersion: 2.2.1 NumberOfBrokerNodes: 3 Metadata: 'AWS::CloudFormation::Designer': id: 56f5417a-ddff-49c4-9f30-8041824a0601 Outputs: VPCId: Description: The ID of the VPC created Value: !Ref VPC PublicSubnetOne: Description: The name of the public subnet created Value: !Ref PublicSubnetOne PrivateSubnetOne: Description: The ID of private subnet one created Value: !Ref PrivateSubnetOne PrivateSubnetTwo: Description: The ID of private subnet two created Value: !Ref PrivateSubnetTwo PrivateSubnetThree: Description: The ID of private subnet three created Value: !Ref PrivateSubnetThree MSKSecurityGroupID: Description: The ID of the security group created for the MSK clusters Value: !GetAtt - MSKSecurityGroup - GroupId KafkaClientEC2InstancePublicDNS: Description: The Public DNS for the MirrorMaker EC2 instance Value: !GetAtt - KafkaClientEC2Instance - PublicDnsName MSKClusterArn: Description: The Arn for the MSKMMCluster1 MSK cluster Value: !Ref MSKCluster Metadata: 'AWS::CloudFormation::Designer': 268a951d-285d-438a-a365-d135a51bd01c: size: width: 60 height: 60 position: x: 60 'y': 930 z: 1 embeds: [] 01de5e28-2175-45f1-a37f-65e5bd7d9356: size: width: 60 height: 60 position: x: 180 'y': 930 z: 1 embeds: [] isassociatedwith: - 268a951d-285d-438a-a365-d135a51bd01c d2a821e3-5c45-46a4-92fd-09268f72f679: size: width: 60 height: 60 position: x: 300 'y': 930 z: 1 embeds: [] e191108b-e241-404a-bed2-b2863bcd5235: size: width: 870 height: 780 position: x: 320 'y': 40 z: 1 embeds: - 758d2702-7bfd-470e-84a0-5101471dd251 - 20d8d324-9da6-49f9-9e37-4bd433625c47 - 7a45b627-d06a-4956-9f4b-a79b5e65bc7d - 93747341-451f-4c6e-8265-6873b110ae9e - 69ad49b5-ffb5-4019-82dd-1b5433531511 - 1a1c22e5-745b-443d-93d6-78c3572de3c4 - be121547-ef38-4d55-a9a9-f67eb35c9cd0 - 90cff42d-8429-4a50-b584-5a2ba16651ea 758d2702-7bfd-470e-84a0-5101471dd251: size: width: 60 height: 60 position: x: 980 'y': 310 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 20d8d324-9da6-49f9-9e37-4bd433625c47: size: width: 60 height: 60 position: x: 980 'y': 430 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 7a45b627-d06a-4956-9f4b-a79b5e65bc7d: size: width: 150 height: 150 position: x: 950 'y': 100 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 93747341-451f-4c6e-8265-6873b110ae9e: size: width: 240 height: 240 position: x: 650 'y': 100 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: - 412c8444-e8b8-43eb-8c07-3f822c921e34 iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 988bff04-9cdd-4ce1-8b5d-59fc3291eae1: source: id: e191108b-e241-404a-bed2-b2863bcd5235 target: id: d2a821e3-5c45-46a4-92fd-09268f72f679 412c8444-e8b8-43eb-8c07-3f822c921e34: size: width: 60 height: 60 position: x: 680 'y': 160 z: 3 parent: 93747341-451f-4c6e-8265-6873b110ae9e embeds: [] isassociatedwith: - d2a821e3-5c45-46a4-92fd-09268f72f679 iscontainedinside: - 93747341-451f-4c6e-8265-6873b110ae9e dependson: - 988bff04-9cdd-4ce1-8b5d-59fc3291eae1 69ad49b5-ffb5-4019-82dd-1b5433531511: size: width: 150 height: 150 position: x: 770 'y': 400 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 4f6ec5cd-f063-4163-8494-31bd55bcb27f: source: id: 7a45b627-d06a-4956-9f4b-a79b5e65bc7d target: id: 69ad49b5-ffb5-4019-82dd-1b5433531511 1a1c22e5-745b-443d-93d6-78c3572de3c4: size: width: 150 height: 150 position: x: 560 'y': 400 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 d345d815-288f-4446-add0-dd637cc6c57f: source: id: 7a45b627-d06a-4956-9f4b-a79b5e65bc7d target: id: 1a1c22e5-745b-443d-93d6-78c3572de3c4 be121547-ef38-4d55-a9a9-f67eb35c9cd0: size: width: 150 height: 150 position: x: 350 'y': 400 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: [] iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 56f5417a-ddff-49c4-9f30-8041824a0601: size: width: 60 height: 60 position: x: 420 'y': 930 z: 1 embeds: [] 912ef118-eda5-437f-afd7-3f92d83fd70b: source: id: 7a45b627-d06a-4956-9f4b-a79b5e65bc7d target: id: be121547-ef38-4d55-a9a9-f67eb35c9cd0 90cff42d-8429-4a50-b584-5a2ba16651ea: size: width: 240 height: 240 position: x: 350 'y': 100 z: 2 parent: e191108b-e241-404a-bed2-b2863bcd5235 embeds: - 25e7995f-31fc-45b0-b790-42bcb519905a iscontainedinside: - e191108b-e241-404a-bed2-b2863bcd5235 25e7995f-31fc-45b0-b790-42bcb519905a: size: width: 60 height: 60 position: x: 380 'y': 160 z: 3 parent: 90cff42d-8429-4a50-b584-5a2ba16651ea embeds: [] isassociatedwith: - 758d2702-7bfd-470e-84a0-5101471dd251 iscontainedinside: - 90cff42d-8429-4a50-b584-5a2ba16651ea c2c64928-4fe4-4640-a449-08cdc9e50d84: source: id: 93747341-451f-4c6e-8265-6873b110ae9e target: id: 90cff42d-8429-4a50-b584-5a2ba16651ea