AWSTemplateFormatVersion: "2010-09-09" Parameters: LogsLifeCycle: Type: Number Description: Number in days to retain logs Default: 30 ApplicationAccount: Type: String Description: Application account id Default: "" Conditions: HasApplicationAccount: !Not [!Equals [!Ref "ApplicationAccount", ""]] Resources: LogBucket: Type: AWS::S3::Bucket Properties: BucketName: !Sub "logs-${AWS::AccountId}" AccessControl: Private PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 LifecycleConfiguration: Rules: - Status: Enabled ExpirationInDays: !Ref LogsLifeCycle GlueRole: Type: AWS::IAM::Role Properties: RoleName: !Sub "GlueCrawler-to-S3-${AWS::AccountId}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - glue.amazonaws.com Action: - "sts:AssumeRole" Policies: - PolicyName: gluecrawler-s3-policy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - s3:* Resource: - !GetAtt LogBucket.Arn - !Sub "${LogBucket.Arn}/*" - Effect: Allow Action: - glue:* - logs:* Resource: - "*" KinesisRole: Type: AWS::IAM::Role Properties: RoleName: !Sub "KinesisFirehose-to-S3-${AWS::AccountId}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - firehose.amazonaws.com Action: - "sts:AssumeRole" Policies: - PolicyName: kinesisfirehose-s3-policy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - s3:AbortMultipartUpload - s3:GetBucketLocation - s3:GetObject - s3:ListBucket - s3:ListBucketMultipartUploads - s3:PutObject Resource: - !GetAtt LogBucket.Arn - !Sub "${LogBucket.Arn}/*" - Effect: Allow Action: - kinesis:DescribeStream - kinesis:GetShardIterator - kinesis:GetRecords - kinesis:ListShards - kms:Decrypt - kms:GenerateDataKey - logs:PutLogEvents - lambda:InvokeFunction - lambda:GetFunctionConfiguration Resource: - "*" GlueDatabase: Type: AWS::Glue::Database Properties: CatalogId: !Ref AWS::AccountId DatabaseInput: Name: !Sub "logdb-${AWS::AccountId}" GlueCrawler: Type: AWS::Glue::Crawler Properties: Name: !Sub "crawler-${AWS::AccountId}" Role: !GetAtt GlueRole.Arn Schedule: ScheduleExpression: cron(0 12 * * ? *) DatabaseName: !Ref GlueDatabase Targets: S3Targets: - Path: !Sub "${LogBucket}" KinesisFirehose: Type: AWS::KinesisFirehose::DeliveryStream Properties: DeliveryStreamName: !Sub "kinesis-firehose-${AWS::AccountId}" DeliveryStreamType: DirectPut S3DestinationConfiguration: BucketARN: !GetAtt LogBucket.Arn Prefix: "logging/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/" ErrorOutputPrefix: "ecs-error/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/" RoleARN: !GetAtt KinesisRole.Arn BufferingHints: IntervalInSeconds: 60 SizeInMBs: 5 ApplicationAccountRole: Condition: HasApplicationAccount Type: AWS::IAM::Role Properties: RoleName: !Sub "Application-Assume-${AWS::AccountId}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: AWS: - !Ref ApplicationAccount Action: - "sts:AssumeRole" Policies: - PolicyName: firehose-put-policy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "firehose:PutRecord" - "firehose:PutRecordBatch" Resource: - !GetAtt KinesisFirehose.Arn Outputs: BucketName: Value: !Ref LogBucket KinesisFirehoseECSName: Value: !Ref KinesisFirehose KinesisFirehoseECSArn: Value: !GetAtt KinesisFirehose.Arn ApplicationAccountRoleArn: Condition: HasApplicationAccount Value: !GetAtt ApplicationAccountRole.Arn