--- AWSTemplateFormatVersion: '2010-09-09' Description: Amazon EC2 Edge Node with SSH Key setup Mappings: RegionMap: ap-northeast-1: ami: ami-0218d08a1f9dac831 ap-northeast-2: ami: ami-0eb14fe5735c13eb5 ap-northeast-3: ami: ami-0f1ffb565070e6947 ap-south-1: ami: ami-052cef05d01020f1d ap-southeast-1: ami: ami-0dc5785603ad4ff54 ap-southeast-2: ami: ami-0bd2230cfb28832f7 ca-central-1: ami: ami-0bae7412735610274 eu-central-1: ami: ami-05d34d340fb1d89e5 eu-north-1: ami: ami-06bfd6343550d4a29 eu-west-1: ami: ami-04dd4500af104442f eu-west-2: ami: ami-0d37e07bd4ff37148 eu-west-3: ami: ami-0d3c032f5934e1b41 sa-east-1: ami: ami-0056d4296b1120bc3 us-east-1: ami: ami-0ed9277fb7eb570c9 us-east-2: ami: ami-002068ed284fb165b us-west-1: ami: ami-03af6a70ccd8cb578 us-west-2: ami: ami-00f7e5c52c0f43726 Parameters: EMRClusterID: Description: "Enter EMR Cluster ID to setup Edge Node. ex. j-1UHP9DN383VKJ " Type: String Default: j-391YRU9L0GWO8 S3BucketForArtifacts: Description: "Enter S3 Bucket name in your AWS account to store EMR artifacts setup. ex. bucketname (do NOT include s3://) " Type: String Default: aws-emr-bda-public VpcID: Description: "Enter VPC ID where Edge Node needs to be setup. ex. vpc-999yyyzz (EMR should be reachable or use same VPC as EMR) " Type: String Default: vpc-97c98af1 SubnetID: Description: "Enter Subnet ID where Edge Node needs to be setup. ex. subnet-999yyy (EMR should be reachable or use same Subnet as EMR) " Type: String Default: subnet-09f11641 SecurityGroupID: Description: "Enter Security group for Edge Node needs to be setup. ex.sg-123rty (EMR should be reachable or use same SG as EMR master) " Type: String Default: sg-b27298cd EC2KeyPairName: Type: AWS::EC2::KeyPair::KeyName Description: "Choose EC2 KeyPair for the list, if none, create EC2 KeyPair and return to this CFN" Resources: CopyEMRConfigsStep: Type: AWS::EMR::Step Properties: Name: CopyEMRConfigsStep ActionOnFailure: CANCEL_AND_WAIT JobFlowId: !Ref EMRClusterID HadoopJarStep: Jar: "command-runner.jar" Args: - "bash" - "-c" - !Sub "sudo rm -rf /tmp/emr_edge_node && sudo yum install git -y && git clone --depth 1 https://github.com/aws-samples/aws-emr-utilities.git /tmp/emr_edge_node && cd /tmp/emr_edge_node && git filter-branch --prune-empty --subdirectory-filter utilities/emr-edge-node-creator HEAD && bash copy-emr-client-deps.sh --s3-folder s3://${S3BucketForArtifacts}/emr-client-deps/" EdgeNode: Type: AWS::EC2::Instance CreationPolicy: ResourceSignal: Count: '1' Timeout: PT15M Properties: ImageId: !FindInMap - RegionMap - !Ref 'AWS::Region' - ami KeyName: !Ref EC2KeyPairName InstanceType: 'r5.2xlarge' SourceDestCheck: false SubnetId: !Ref SubnetID SecurityGroupIds: - !Ref SecurityGroupID IamInstanceProfile: !Ref EdgeInstanceProfile Tags: - Key: "Name" Value: "EdgeNodeForEMR" BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: VolumeType: gp2 VolumeSize: 100 DeleteOnTermination: true Encrypted: true UserData: Fn::Base64: !Sub | Content-Type: multipart/mixed; boundary="//" MIME-Version: 1.0 --// Content-Type: text/cloud-config; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="cloud-config.txt" #cloud-config cloud_final_modules: - [scripts-user, always] --// Content-Type: text/x-shellscript; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="userdata.txt" #!/bin/bash export AWS_DEFAULT_REGION=${AWS::Region} yum update -y yum install -y git aws-cfn-bootstrap yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm while [ ! "$(aws s3 ls s3://${S3BucketForArtifacts}/emr-client-deps/emr-client-deps-${EMRClusterID}.tar.gz)" ]; do echo "File $FILE_NAME not found in bucket $BUCKET_NAME. Sleeping for 5 seconds..." sleep 5 done echo "File $FILE_NAME found in bucket $BUCKET_NAME." sleep 60; git clone --depth 1 https://github.com/aws-samples/aws-emr-utilities.git /tmp/emr_edge_node cd /tmp/emr_edge_node git filter-branch --prune-empty --subdirectory-filter utilities/emr-edge-node-creator HEAD sh setup-emr-edge-node-s3.sh --emr-client-deps s3://${S3BucketForArtifacts}/emr-client-deps/emr-client-deps-${EMRClusterID}.tar.gz /opt/aws/bin/cfn-signal -e 0 --stack ${AWS::StackName} --resource EdgeNode --region ${AWS::Region} DependsOn: CopyEMRConfigsStep EdgeInstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: / Roles: - !Ref EdgeRole EdgeNodeEIP: Type: 'AWS::EC2::EIP' Properties: Domain: standard InstanceId: !Ref EdgeNode EdgeRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Effect: Allow Principal: Service: - ec2.amazonaws.com Action: - sts:AssumeRole Path: / ManagedPolicyArns: - 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM' - 'arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role' Policies: - PolicyName: gitlab-runner PolicyDocument: Statement: - Effect: Allow Action: - ssm:GetParameter Resource: - !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/edge/default/*' EdgeNodeSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: 'Security group for the edge node host' VpcId: !Ref VpcID SecurityGroupIngress: - IpProtocol: '-1' CidrIp: '0.0.0.0/0' SecurityGroupEgress: - IpProtocol: '-1' CidrIp: '0.0.0.0/0' Outputs: EdgeNodePublicIp: Value: !GetAtt EdgeNode.PublicIp EdgeNodePublicDnsName: Value: !GetAtt EdgeNode.PublicDnsName