{ "Parameters": { "S3Bucket": { "Type": "String", "AllowedPattern": ".+", "Description": "The S3 bucket where the Amazon Kinesis Data Analytics application gets your application's JAR file" }, "S3StorageBucket": { "Type": "String", "AllowedPattern": ".+", "Description": "The S3 bucket name used to store the server-sent events data" }, "S3StorageBucketPrefix": { "Type": "String", "Default": "sse-data", "Description": "The prefix used when storing server-sent events data into the S3 bucket" }, "S3StorageBucketErrorPrefix": { "Type": "String", "Default": "sse-error", "Description": "The prefix used when storing error events into the S3 bucket" }, "FlinkApplication": { "Type": "String", "Default": "amazon-kinesis-data-analytics-apache-flink-server-sent-events-1.0.0.jar", "AllowedPattern": ".+", "Description": "The Apache Flink application jar filename located in the S3 bucket" }, "Subnets": { "Type": "List", "Description": "The subnet Ids used for the Amazon Kinesis Data Analytics application" }, "SecurityGroups": { "Type": "List", "Description": "The security group Ids used for the Amazon Kinesis Data Analytics application" } }, "Resources": { "KinesisServerSentEventsDataStreamA4B5CADC": { "Type": "AWS::Kinesis::Stream", "Properties": { "ShardCount": 1, "RetentionPeriodHours": 24 } }, "KinesisFirehoseS3DeliveryServiceRole161C0743": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com" } } ], "Version": "2012-10-17" } } }, "KinesisFirehoseS3DeliveryServiceRoleDefaultPolicy41331D93": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "kinesis:DescribeStreamSummary", "kinesis:GetRecords", "kinesis:GetShardIterator", "kinesis:ListShards", "kinesis:SubscribeToShard", "kinesis:DescribeStream" ], "Effect": "Allow", "Resource": { "Fn::GetAtt": [ "KinesisServerSentEventsDataStreamA4B5CADC", "Arn" ] } } ], "Version": "2012-10-17" }, "PolicyName": "KinesisFirehoseS3DeliveryServiceRoleDefaultPolicy41331D93", "Roles": [ { "Ref": "KinesisFirehoseS3DeliveryServiceRole161C0743" } ] } }, "KinesisFirehoseS3DeliveryS3DestinationRoleB687076B": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com" } } ], "Version": "2012-10-17" } } }, "KinesisFirehoseS3DeliveryS3DestinationRoleDefaultPolicyC2E014A6": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*" ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3StorageBucket" } ] ] }, { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3StorageBucket" }, "/*" ] ] } ] }, { "Action": [ "logs:CreateLogStream", "logs:PutLogEvents" ], "Effect": "Allow", "Resource": { "Fn::GetAtt": [ "KinesisFirehoseS3DeliveryLogGroupDA9D03DA", "Arn" ] } } ], "Version": "2012-10-17" }, "PolicyName": "KinesisFirehoseS3DeliveryS3DestinationRoleDefaultPolicyC2E014A6", "Roles": [ { "Ref": "KinesisFirehoseS3DeliveryS3DestinationRoleB687076B" } ] } }, "KinesisFirehoseS3DeliveryLogGroupDA9D03DA": { "Type": "AWS::Logs::LogGroup", "Properties": { "RetentionInDays": 731 }, "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, "KinesisFirehoseS3DeliveryLogGroupS3DestinationACEB009D": { "Type": "AWS::Logs::LogStream", "Properties": { "LogGroupName": { "Ref": "KinesisFirehoseS3DeliveryLogGroupDA9D03DA" } }, "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, "KinesisFirehoseS3DeliveryE807D61C": { "Type": "AWS::KinesisFirehose::DeliveryStream", "Properties": { "DeliveryStreamType": "KinesisStreamAsSource", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3StorageBucket" } ] ] }, "BufferingHints": { "IntervalInSeconds": 60, "SizeInMBs": 5 }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "KinesisFirehoseS3DeliveryLogGroupDA9D03DA" }, "LogStreamName": { "Ref": "KinesisFirehoseS3DeliveryLogGroupS3DestinationACEB009D" } }, "ErrorOutputPrefix": { "Ref": "S3StorageBucketErrorPrefix" }, "Prefix": { "Ref": "S3StorageBucketPrefix" }, "RoleARN": { "Fn::GetAtt": [ "KinesisFirehoseS3DeliveryS3DestinationRoleB687076B", "Arn" ] } }, "KinesisStreamSourceConfiguration": { "KinesisStreamARN": { "Fn::GetAtt": [ "KinesisServerSentEventsDataStreamA4B5CADC", "Arn" ] }, "RoleARN": { "Fn::GetAtt": [ "KinesisFirehoseS3DeliveryServiceRole161C0743", "Arn" ] } } }, "DependsOn": [ "KinesisFirehoseS3DeliveryS3DestinationRoleDefaultPolicyC2E014A6", "KinesisFirehoseS3DeliveryServiceRoleDefaultPolicy41331D93" ] }, "KinesisAnalyticsServerSentEventsApplicationRole37FA4990": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "kinesisanalytics.amazonaws.com" } } ], "Version": "2012-10-17" } } }, "KinesisAnalyticsServerSentEventsApplicationRoleDefaultPolicy35941F86": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { "Statement": [ { "Action": "cloudwatch:PutMetricData", "Effect": "Allow", "Resource": "*" }, { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*" ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3Bucket" } ] ] }, { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3Bucket" }, "/*" ] ] } ] }, { "Action": "logs:DescribeLogGroups", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":logs:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" }, ":log-group:*" ] ] } }, { "Action": "logs:DescribeLogStreams", "Effect": "Allow", "Resource": { "Fn::GetAtt": [ "KinesisAnalyticsServerSentEventsApplicationLogGroup1870CD29", "Arn" ] } }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":logs:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" }, ":log-group:", { "Ref": "KinesisAnalyticsServerSentEventsApplicationLogGroup1870CD29" }, ":log-stream:", { "Ref": "KinesisAnalyticsServerSentEventsApplicationLogStream9F10C1D5" } ] ] } }, { "Action": [ "ec2:DescribeVpcs", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeDhcpOptions" ], "Effect": "Allow", "Resource": "*" }, { "Action": [ "ec2:CreateNetworkInterface", "ec2:CreateNetworkInterfacePermission", "ec2:DescribeNetworkInterfaces", "ec2:DeleteNetworkInterface" ], "Effect": "Allow", "Resource": "*" }, { "Action": [ "kinesis:DescribeStreamSummary", "kinesis:GetRecords", "kinesis:GetShardIterator", "kinesis:ListShards", "kinesis:SubscribeToShard", "kinesis:PutRecord", "kinesis:PutRecords" ], "Effect": "Allow", "Resource": { "Fn::GetAtt": [ "KinesisServerSentEventsDataStreamA4B5CADC", "Arn" ] } } ], "Version": "2012-10-17" }, "PolicyName": "KinesisAnalyticsServerSentEventsApplicationRoleDefaultPolicy35941F86", "Roles": [ { "Ref": "KinesisAnalyticsServerSentEventsApplicationRole37FA4990" } ] } }, "KinesisAnalyticsServerSentEventsApplication821A3B97": { "Type": "AWS::KinesisAnalyticsV2::Application", "Properties": { "RuntimeEnvironment": "FLINK-1_11", "ServiceExecutionRole": { "Fn::GetAtt": [ "KinesisAnalyticsServerSentEventsApplicationRole37FA4990", "Arn" ] }, "ApplicationConfiguration": { "ApplicationCodeConfiguration": { "CodeContent": { "S3ContentLocation": { "BucketARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":s3:::", { "Ref": "S3Bucket" } ] ] }, "FileKey": { "Ref": "FlinkApplication" } } }, "CodeContentType": "ZIPFILE" }, "ApplicationSnapshotConfiguration": { "SnapshotsEnabled": true }, "FlinkApplicationConfiguration": { "ParallelismConfiguration": { "AutoScalingEnabled": false, "ConfigurationType": "CUSTOM", "Parallelism": 1, "ParallelismPerKPU": 1 } } } }, "DependsOn": [ "KinesisAnalyticsServerSentEventsApplicationRoleDefaultPolicy35941F86", "KinesisAnalyticsServerSentEventsApplicationRole37FA4990" ], "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" }, "KinesisAnalyticsServerSentEventsApplicationLogGroup1870CD29": { "Type": "AWS::Logs::LogGroup", "Properties": { "RetentionInDays": 731 }, "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, "KinesisAnalyticsServerSentEventsApplicationLogStream9F10C1D5": { "Type": "AWS::Logs::LogStream", "Properties": { "LogGroupName": { "Ref": "KinesisAnalyticsServerSentEventsApplicationLogGroup1870CD29" } }, "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, "KinesisAnalyticsServerSentEventsApplicationLoggingOption8ED90FE6": { "Type": "AWS::KinesisAnalyticsV2::ApplicationCloudWatchLoggingOption", "Properties": { "ApplicationName": { "Ref": "KinesisAnalyticsServerSentEventsApplication821A3B97" }, "CloudWatchLoggingOption": { "LogStreamARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":logs:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" }, ":log-group:", { "Ref": "KinesisAnalyticsServerSentEventsApplicationLogGroup1870CD29" }, ":log-stream:", { "Ref": "KinesisAnalyticsServerSentEventsApplicationLogStream9F10C1D5" } ] ] } } } }, "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRole51ACCDDF": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com" } } ], "Version": "2012-10-17" }, "ManagedPolicyArns": [ { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" ] ] } ] } }, "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRoleDefaultPolicy839A5D8E": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "kinesisanalytics:UpdateApplication", "kinesisanalytics:DescribeApplication", "kinesisanalytics:AddApplicationVpcConfiguration" ], "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition" }, ":kinesisanalytics:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" }, ":application/", { "Ref": "KinesisAnalyticsServerSentEventsApplication821A3B97" } ] ] } } ], "Version": "2012-10-17" }, "PolicyName": "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRoleDefaultPolicy839A5D8E", "Roles": [ { "Ref": "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRole51ACCDDF" } ] } }, "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a7651F875152": { "Type": "AWS::Lambda::Function", "Properties": { "Code": { "ZipFile": "// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.\n// SPDX-License-Identifier: MIT-0\n\nconst AWS = require('aws-sdk');\nconst response = require('cfn-response');\nvar kinesisanalyticsv2 = new AWS.KinesisAnalyticsV2();\nexports.handler = (event, context) => {\n console.info(\"Event: \" + JSON.stringify(event));\n if (event.RequestType == \"Create\") {\n try {\n kinesisanalyticsv2.describeApplication({\n ApplicationName: process.env.ApplicationName,\n IncludeAdditionalDetails: true\n }, function(err, app) {\n if (err) {\n response.send(event, context, response.FAILED, err);\n }\n else {\n kinesisanalyticsv2.updateApplication({\n \"ApplicationName\": process.env.ApplicationName,\n \"CurrentApplicationVersionId\": app.ApplicationDetail.ApplicationVersionId,\n \"ApplicationConfigurationUpdate\": {\n \"EnvironmentPropertyUpdates\": {\n \"PropertyGroups\": [\n {\n \"PropertyGroupId\": \"ProducerConfigProperties\",\n \"PropertyMap\": {\n \"AggregationEnabled\" : \"false\",\n \"aws.region\" : process.env.AWS_REGION\n }\n },\n {\n \"PropertyGroupId\": \"OutputStreamProperties\",\n \"PropertyMap\": {\n \"DefaultStream\" : process.env.OutputStream\n }\n },\n {\n \"PropertyGroupId\": \"SSESourceProperties\",\n \"PropertyMap\": {\n \"url\" : \"https://stream.wikimedia.org/v2/stream/recentchange\"\n }\n }\n ]\n }\n }\n }, function (err, update) {\n if (err) {\n response.send(event, context, response.FAILED, err);\n } else {\n kinesisanalyticsv2.addApplicationVpcConfiguration({\n \"ApplicationName\": process.env.ApplicationName,\n \"VpcConfiguration\": event.ResourceProperties.VpcConfiguration,\n \"CurrentApplicationVersionId\": app.ApplicationDetail.ApplicationVersionId + 1\n }, function (err, vpcConfig) {\n if (err) {\n response.send(event, context, response.FAILED, err);\n } else {\n response.send(event, context, response.SUCCESS, {});\n }\n });\n }\n });\n }\n });\n }\n catch (err) {\n response.send(event, context, response.FAILED, err);\n }\n }\n else {\n response.send(event, context, response.SUCCESS, {});\n }\n};\n" }, "Role": { "Fn::GetAtt": [ "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRole51ACCDDF", "Arn" ] }, "Description": "Initialize the Amazon Kinesis Data Analytics application", "Environment": { "Variables": { "ApplicationName": { "Ref": "KinesisAnalyticsServerSentEventsApplication821A3B97" }, "OutputStream": { "Ref": "KinesisServerSentEventsDataStreamA4B5CADC" } } }, "Handler": "index.handler", "Runtime": "nodejs12.x", "Timeout": 30 }, "DependsOn": [ "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRoleDefaultPolicy839A5D8E", "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a765ServiceRole51ACCDDF" ] }, "KinesisAnalyticsInitResourceVPC": { "Type": "AWS::CloudFormation::CustomResource", "Properties": { "ServiceToken": { "Fn::GetAtt": [ "SingletonLambda4a54b7c52cbf4ddd954cc516bc73a7651F875152", "Arn" ] }, "VpcConfiguration": { "SecurityGroupIds": { "Ref": "SecurityGroups" }, "SubnetIds": { "Ref": "Subnets" } } }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" } }, "Mappings": { "awscdkawskinesisfirehoseCidrBlocks": { "af-south-1": { "FirehoseCidrBlock": "13.244.121.224/27" }, "ap-east-1": { "FirehoseCidrBlock": "18.162.221.32/27" }, "ap-northeast-1": { "FirehoseCidrBlock": "13.113.196.224/27" }, "ap-northeast-2": { "FirehoseCidrBlock": "13.209.1.64/27" }, "ap-northeast-3": { "FirehoseCidrBlock": "13.208.177.192/27" }, "ap-south-1": { "FirehoseCidrBlock": "13.232.67.32/27" }, "ap-southeast-1": { "FirehoseCidrBlock": "13.228.64.192/27" }, "ap-southeast-2": { "FirehoseCidrBlock": "13.210.67.224/27" }, "ca-central-1": { "FirehoseCidrBlock": "35.183.92.128/27" }, "cn-north-1": { "FirehoseCidrBlock": "52.81.151.32/27" }, "cn-northwest-1": { "FirehoseCidrBlock": "161.189.23.64/27" }, "eu-central-1": { "FirehoseCidrBlock": "35.158.127.160/27" }, "eu-north-1": { "FirehoseCidrBlock": "13.53.63.224/27" }, "eu-south-1": { "FirehoseCidrBlock": "15.161.135.128/27" }, "eu-west-1": { "FirehoseCidrBlock": "52.19.239.192/27" }, "eu-west-2": { "FirehoseCidrBlock": "18.130.1.96/27" }, "eu-west-3": { "FirehoseCidrBlock": "35.180.1.96/27" }, "me-south-1": { "FirehoseCidrBlock": "15.185.91.0/27" }, "sa-east-1": { "FirehoseCidrBlock": "18.228.1.128/27" }, "us-east-1": { "FirehoseCidrBlock": "52.70.63.192/27" }, "us-east-2": { "FirehoseCidrBlock": "13.58.135.96/27" }, "us-gov-east-1": { "FirehoseCidrBlock": "18.253.138.96/27" }, "us-gov-west-1": { "FirehoseCidrBlock": "52.61.204.160/27" }, "us-west-1": { "FirehoseCidrBlock": "13.57.135.192/27" }, "us-west-2": { "FirehoseCidrBlock": "52.89.255.224/27" } } } }