---
AWSTemplateFormatVersion: 2010-09-09
Description:  CloudFormation template for running an RDSLogs scheduled task retriever in ECS Cluster on EC2 Spot Instances.
Mappings:
  ecsOptimizedAmi:
    ap-northeast-1:
      AMI: ami-f63f6f91
    ap-southeast-1:
      AMI: ami-b4ae1dd7
    ap-southeast-2:
      AMI: ami-fbe9eb98
    ca-central-1:
      AMI: ami-ee58e58a
    eu-central-1:
      AMI: ami-085e8a67
    eu-west-1:
      AMI: ami-95f8d2f3
    eu-west-2:
      AMI: ami-bf9481db
    us-east-1:
      AMI: ami-275ffe31
    us-east-2:
      AMI: ami-62745007
    us-west-1:
      AMI: ami-689bc208
    us-west-2:
      AMI: ami-62d35c02
Metadata:
  Authors:
    Description: Madhuri Peri (mperi@amazon.com), Shawn OConnor (shawo@amazon.com), Chad Schmutzer (schmutze@amazon.com)
  License:
    Description: 'Copyright 2017 Amazon.com, Inc. and its affiliates. All Rights Reserved.

      Licensed under the Amazon Software License (the "License"). You may not use this file
      except in compliance with the License. A copy of the License is located at

      http://aws.amazon.com/asl/

      or in the "license" file accompanying this file. This file is distributed on an "AS IS"
      BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
      License for the specific language governing permissions and limitations under the License.'
  Update:
    Description: Modified the original tempate for Experian Consumer Services for RDS logs setup.
Outputs:
  ECRRepository:
    Description: ECR repository
    Value:
      !Join ['', [!Ref 'AWS::AccountId', '.dkr.ecr.', !Ref 'AWS::Region', '.amazonaws.com/', !Ref MySampleRepository]]
  RDSLogsBucket:
    Description: RDS logs bucket to store DB instance logs
    Value:
      Ref: RDSLogsBucket
  awsRegionName:
    Description: The name of the AWS Region your template was launched in
    Value:
      Ref: AWS::Region
  cloudWatchLogsGroupName:
    Description: Name of the CloudWatch Logs Group
    Value:
      Ref: cloudWatchLogsGroup
  ecsClusterName:
    Description: The name of the ECS cluster
    Value:
      Ref: ecsCluster
  snsTopic:
    Description: SNS Topic ARN
    Value:
      Ref: snsTopic
  spotFleetRequestId:
    Description: The Spot fleet Request Id
    Value:
      Ref: spotFleet
Parameters:
  DBSubnetA:
    Description: RDS-Aurora DB SubnetA
    Type: String
  DBSubnetB:
    Description: RDS-Aurora DB SubnetB
    Type: String
  DBSubnetC:
    Description: RDS-Aurora DB SubnetC
    Type: String
  ecsClusterTargetCapacity:
    Default: 1
    Description: Number of EC2 Spot instances to initially launch in the ECS cluster
    Type: Number
  instanceType:
    AllowedValues:
    - m3.large
    - m4.large
    - c3.large
    - c4.large
    - r3.large
    - r4.large
    Default: m3.large
    Description: EC2 instance type to use for ECS cluster
    Type: String
  keyName:
    Description: Name of an existing EC2 KeyPair to enable SSH access to the EC2 instances
    Type: AWS::EC2::KeyPair::KeyName
  sourceCidr:
    Description: Optional - CIDR/IP range for instance ssh access - defaults to VPC CIDR ??
    Type: String
  spotBidPrice:
    Default: 0.0246
    Description: Spot Instance bid price
    Type: String
  myVPC:
    Description: VPC-ID for the SecurityGroup
    Type: String
Resources:
  MySampleRepository:
    Type: "AWS::ECR::Repository"
    Properties:
      RepositoryPolicyText:
        Version: "2012-10-17"
        Statement:
          -
            Sid: AllowPushPull
            Effect: Allow
            Principal:
              AWS:
                - "*"
            Action:
              - "ecr:GetDownloadUrlForLayer"
              - "ecr:BatchGetImage"
              - "ecr:BatchCheckLayerAvailability"
              - "ecr:PutImage"
              - "ecr:InitiateLayerUpload"
              - "ecr:UploadLayerPart"
              - "ecr:CompleteLayerUpload"
  ECSSecurityGroup:
    Type: "AWS::EC2::SecurityGroup"
    Properties:
      GroupDescription: "Enable HTTP access on the configured port"
      VpcId: !Ref myVPC
      SecurityGroupIngress:
        -
          IpProtocol: "tcp"
          FromPort: "22"
          ToPort: "22"
          CidrIp: "0.0.0.0/0"
  RDSLogsBucket:
    Type: "AWS::S3::Bucket"
  cloudWatchLogsGroup:
    Properties:
      RetentionInDays: 7
    Type: AWS::Logs::LogGroup
  ecsCluster:
    Type: AWS::ECS::Cluster
  snsTopic:
    Type: AWS::SNS::Topic
  CloudWatchRDSToECSType:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          -
            Effect: "Allow"
            Principal:
              Service:
                - "events.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Path: "/"
      Policies:
        -
          PolicyName: "CloudwatchEventsServiceInvokeECSRunTask"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              -
                Effect: "Allow"
                Action: "ecs:RunTask"
                Resource:
                  Ref: RDSTaskDef
  RDSTaskDef:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Join ['', [!Ref 'AWS::StackName', "-taskdef"]]
      ContainerDefinitions:
        -
          Name: "rdslogsshipper"
          Image: !Join ['', [!Ref 'AWS::AccountId', '.dkr.ecr.', !Ref 'AWS::Region', '.amazonaws.com/', !Ref MySampleRepository]]
          Cpu: "20"
          Memory: "128"
          Essential: "true"
          WorkingDirectory: "/usr/src/app"
          LogConfiguration:
              LogDriver: "awslogs"
              Options:
                "awslogs-group": !Ref cloudWatchLogsGroup
                "awslogs-region": !Ref 'AWS::Region'
          Command:
             - "python"
             - "-u"
             - "./rdslogsshipper.py"
          Environment:
            -
              Name: RDSLOGSBUCKET
              Value: !Ref RDSLogsBucket
      Volumes:
        -
          Host:
            SourcePath: "/var/lib/docker/vfs/dir/"
          Name: "my-vol"
  spotFleet:
    DependsOn:
    - spotFleetRole
    - spotFleetInstanceProfile
    - ecsCluster
    Properties:
      SpotFleetRequestConfigData:
        AllocationStrategy: diversified
        IamFleetRole:
          Fn::GetAtt:
          - spotFleetRole
          - Arn
        LaunchSpecifications:
        - IamInstanceProfile:
            Arn:
              Fn::GetAtt:
              - spotFleetInstanceProfile
              - Arn
          ImageId:
            Fn::FindInMap:
            - ecsOptimizedAmi
            - Ref: AWS::Region
            - AMI
          InstanceType:
            Ref: instanceType
          KeyName:
            Ref: keyName
          Monitoring:
            Enabled: true
          SecurityGroups:
          - GroupId: !Ref ECSSecurityGroup
          SubnetId:
            Fn::Join:
            - ','
            - - !Ref DBSubnetA
              - !Ref DBSubnetB
              - !Ref DBSubnetC
          UserData:
            Fn::Base64:
              Fn::Sub: '#!/bin/bash -xe

                export PATH=/usr/local/bin:$PATH

                yum -y --security update

                yum -y install jq

                easy_install pip

                yum install aws-cli

                aws configure set default.region ${AWS::Region}

                echo ECS_CLUSTER=${ecsCluster} >> /etc/ecs/ecs.config



                cat <<EOF > /tmp/awslogs.conf

                [general]

                state_file = /var/awslogs/state/agent-state



                [/var/log/dmesg]

                file = /var/log/dmesg

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/dmesg

                initial_position = start_of_file



                [/var/log/messages]

                file = /var/log/messages

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/messages

                datetime_format = %b %d %H:%M:%S

                initial_position = start_of_file



                [/var/log/docker]

                file = /var/log/docker

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/docker

                datetime_format = %Y-%m-%dT%H:%M:%S.%f

                initial_position = start_of_file



                [/var/log/ecs/ecs-init.log]

                file = /var/log/ecs/ecs-init.log.*

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/ecs/ecs-init.log

                datetime_format = %Y-%m-%dT%H:%M:%SZ

                initial_position = start_of_file



                [/var/log/ecs/ecs-agent.log]

                file = /var/log/ecs/ecs-agent.log.*

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/ecs/ecs-agent.log

                datetime_format = %Y-%m-%dT%H:%M:%SZ

                initial_position = start_of_file



                [/var/log/ecs/audit.log]

                file = /var/log/ecs/audit.log.*

                log_group_name = ${cloudWatchLogsGroup}

                log_stream_name = %ECS_CLUSTER/%CONTAINER_INSTANCE/var/log/ecs/audit.log

                datetime_format = %Y-%m-%dT%H:%M:%SZ

                initial_position = start_of_file

                EOF



                cd /tmp && curl -sO https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py

                python /tmp/awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/awslogs.conf



                cat <<EOF > /etc/init/cloudwatch-logs-start.conf

                description "Configure and start CloudWatch Logs agent on Amazon ECS container
                instance"

                author "Amazon Web Services"

                start on started ecs

                script

                exec 2>>/var/log/cloudwatch-logs-start.log

                set -x

                until curl -s http://localhost:51678/v1/metadata; do sleep 1; done

                ECS_CLUSTER=\$(curl -s http://localhost:51678/v1/metadata | jq .Cluster | tr
                -d \")

                CONTAINER_INSTANCE=\$(curl -s http://localhost:51678/v1/metadata | jq .ContainerInstanceArn
                | tr -d \")

                sed -i "s|%ECS_CLUSTER|\$ECS_CLUSTER|g" /var/awslogs/etc/awslogs.conf

                sed -i "s|%CONTAINER_INSTANCE|\$CONTAINER_INSTANCE|g" /var/awslogs/etc/awslogs.conf

                chkconfig awslogs on

                service awslogs start

                end script

                EOF



                cat <<EOF > /etc/init/spot-instance-termination-notice-handler.conf

                description "Start spot instance termination handler monitoring script"

                author "Amazon Web Services"

                start on started ecs

                script

                echo \$\$ > /var/run/spot-instance-termination-notice-handler.pid

                exec /usr/local/bin/spot-instance-termination-notice-handler.sh

                end script

                pre-start script

                logger "[spot-instance-termination-notice-handler.sh]: spot instance termination
                notice handler started"

                end script

                EOF



                cat <<EOF > /usr/local/bin/spot-instance-termination-notice-handler.sh

                #!/bin/bash

                while sleep 5; do

                if [ -z \$(curl -Isf http://169.254.169.254/latest/meta-data/spot/termination-time)];
                then

                /bin/false

                else

                logger "[spot-instance-termination-notice-handler.sh]: spot instance termination
                notice detected"

                STATUS=DRAINING

                ECS_CLUSTER=\$(curl -s http://localhost:51678/v1/metadata | jq .Cluster | tr
                -d \")

                CONTAINER_INSTANCE=\$(curl -s http://localhost:51678/v1/metadata | jq .ContainerInstanceArn
                | tr -d \")

                logger "[spot-instance-termination-notice-handler.sh]: putting instance in state
                \$STATUS"

                logger "[spot-instance-termination-notice-handler.sh]: running: /usr/local/bin/aws
                ecs update-container-instances-state --cluster \$ECS_CLUSTER --container-instances
                \$CONTAINER_INSTANCE --status \$STATUS"

                /usr/local/bin/aws ecs update-container-instances-state --cluster \$ECS_CLUSTER
                --container-instances \$CONTAINER_INSTANCE --status \$STATUS

                logger "[spot-instance-termination-notice-handler.sh]: running: \"/usr/local/bin/aws
                sns publish --topic-arn ${snsTopic} --message \"Spot instance termination notice
                detected. Details: cluster: \$ECS_CLUSTER, container_instance: \$CONTAINER_INSTANCE.
                Putting instance in state \$STATUS.\""

                /usr/local/bin/aws sns publish --topic-arn ${snsTopic} --message "Spot instance
                termination notice detected. Details: cluster: \$ECS_CLUSTER, container_instance:
                \$CONTAINER_INSTANCE. Putting instance in state \$STATUS."

                logger "[spot-instance-termination-notice-handler.sh]: putting myself to sleep..."

                sleep 120

                fi

                done

                EOF



                chmod +x /usr/local/bin/spot-instance-termination-notice-handler.sh

                '
        SpotPrice:
          Ref: spotBidPrice
        TargetCapacity:
          Ref: ecsClusterTargetCapacity
        TerminateInstancesWithExpiration: true
    Type: AWS::EC2::SpotFleet
  spotFleetInstanceProfile:
    DependsOn:
    - spotFleetInstanceRole
    Properties:
      Path: /
      Roles:
      - Ref: spotFleetInstanceRole
    Type: AWS::IAM::InstanceProfile
  spotFleetInstanceRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role
      - arn:aws:iam::aws:policy/AmazonRDSFullAccess
      - arn:aws:iam::aws:policy/AmazonS3FullAccess
      - arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess
      Path: /
      Policies:
      - PolicyDocument:
          Statement:
          - Action:
            - ecs:UpdateContainerInstancesState
            - ecr:BatchGetImage
            Effect: Allow
            Resource: '*'
          Version: 2012-10-17
        PolicyName: ecsUpdateContainerInstancesStatePolicy
      - PolicyDocument:
          Statement:
          - Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Effect: Allow
            Resource: arn:aws:logs:*:*:*
          Version: 2012-10-17
        PolicyName: cloudWatchLogsPolicy
      - PolicyDocument:
          Statement:
          - Action:
            - sns:Publish
            Effect: Allow
            Resource:
              Ref: snsTopic
          Version: 2012-10-17
        PolicyName: snsPublishPolicy
    Type: AWS::IAM::Role
  spotFleetRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Effect: Allow
          Principal:
            Service:
            - spotfleet.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetRole
      Path: /
    Type: AWS::IAM::Role
...