// Jest Snapshot v1, https://goo.gl/fbAQLP exports[`test App Integration Construct 1`] = ` { "Parameters": { "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aArtifactHash88A33C87": { "Description": "Artifact hash for asset "09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16a"", "Type": "String", }, "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aS3BucketAB08A02E": { "Description": "S3 bucket for asset "09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16a"", "Type": "String", }, "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aS3VersionKey5B136550": { "Description": "S3 key for asset version "09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16a"", "Type": "String", }, "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464ArtifactHash36F616DD": { "Description": "Artifact hash for asset "8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464"", "Type": "String", }, "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464S3Bucket65363AF3": { "Description": "S3 bucket for asset "8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464"", "Type": "String", }, "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464S3VersionKey16E8A40A": { "Description": "S3 key for asset version "8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464"", "Type": "String", }, }, "Resources": { "AccessLogD598EBE9": { "DeletionPolicy": "Retain", "Properties": { "AccessControl": "LogDeliveryWrite", "BucketEncryption": { "ServerSideEncryptionConfiguration": [ { "ServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256", }, }, ], }, "PublicAccessBlockConfiguration": { "BlockPublicAcls": true, "BlockPublicPolicy": true, "IgnorePublicAcls": true, "RestrictPublicBuckets": true, }, }, "Type": "AWS::S3::Bucket", "UpdateReplacePolicy": "Retain", }, "AccessLogPolicy4B3DB856": { "Properties": { "Bucket": { "Ref": "AccessLogD598EBE9", }, "PolicyDocument": { "Statement": [ { "Action": "s3:*", "Condition": { "Bool": { "aws:SecureTransport": "false", }, }, "Effect": "Deny", "Principal": { "AWS": "*", }, "Resource": [ { "Fn::GetAtt": [ "AccessLogD598EBE9", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "AccessLogD598EBE9", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "s3:PutObject", "Condition": { "ArnLike": { "aws:SourceArn": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, ], }, "StringEquals": { "aws:SourceAccount": { "Ref": "AWS::AccountId", }, }, }, "Effect": "Allow", "Principal": { "Service": "logging.s3.amazonaws.com", }, "Resource": { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "AccessLogD598EBE9", "Arn", ], }, "/Integration*", ], ], }, "Sid": "IntegrationAccessLogsPolicy", }, ], "Version": "2012-10-17", }, }, "Type": "AWS::S3::BucketPolicy", }, "IntegrationCustomIngestionFirehoseGlueKmsE75E943F": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionFirehoseGlueKmsE75E943F", "Roles": [ { "Ref": "IntegrationCustomIngestionFirehoseRole740DD3F6", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionFirehoseGlueTablePolicy5BC6DFF0": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBCustomIngestionCfncustomingestionCF1D54E2", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionFirehoseGlueTablePolicy5BC6DFF0", "Roles": [ { "Ref": "IntegrationCustomIngestionFirehoseRole740DD3F6", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionFirehoseRole740DD3F6": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionFirehoseRoleDefaultPolicy26953B1A": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionFirehoseRoleDefaultPolicy26953B1A", "Roles": [ { "Ref": "IntegrationCustomIngestionFirehoseRole740DD3F6", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionItemFirehoseGlueKms8B9ECAD4": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionItemFirehoseGlueKms8B9ECAD4", "Roles": [ { "Ref": "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionItemFirehoseGlueTablePolicy1D9830B6": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBCustomIngestionItemCfncustomingestionitemC488BBC0", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionItemFirehoseGlueTablePolicy1D9830B6", "Roles": [ { "Ref": "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionItemFirehoseRoleB492CD4B": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionItemFirehoseRoleDefaultPolicy90F7728A": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionItemFirehoseRoleDefaultPolicy90F7728A", "Roles": [ { "Ref": "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionItemKinesisFirehose487658C0": { "DependsOn": [ "IntegrationCustomIngestionItemFirehoseGlueKms8B9ECAD4", "IntegrationCustomIngestionItemFirehoseGlueTablePolicy1D9830B6", "IntegrationCustomIngestionItemFirehoseRoleDefaultPolicy90F7728A", "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroup7B95EC98", }, "LogStreamName": { "Ref": "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroupfirehoselogstream6928D7C9", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBCustomIngestionItemCfncustomingestionitemC488BBC0", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/customingestionitem/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "customingestionitem/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionItemFirehoseRoleB492CD4B", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationCustomIngestionItemKinesisFirehoseKinesisFirehosePolicyEA67E9B6": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroup7B95EC98", }, ":log-stream:", { "Ref": "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroupfirehoselogstream6928D7C9", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionItemKinesisFirehoseKinesisFirehosePolicyEA67E9B6", "Roles": [ { "Ref": "IntegrationCustomIngestionItemKinesisFirehoseKinesisFirehoseRole8F63C867", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionItemKinesisFirehoseKinesisFirehoseRole8F63C867": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroup7B95EC98": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroupfirehoselogstream6928D7C9": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationCustomIngestionItemKinesisFirehosefirehoseloggroup7B95EC98", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationCustomIngestionKinesisFirehose9C19C395": { "DependsOn": [ "IntegrationCustomIngestionFirehoseGlueKmsE75E943F", "IntegrationCustomIngestionFirehoseGlueTablePolicy5BC6DFF0", "IntegrationCustomIngestionFirehoseRoleDefaultPolicy26953B1A", "IntegrationCustomIngestionFirehoseRole740DD3F6", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationCustomIngestionKinesisFirehosefirehoseloggroup95E8C3C0", }, "LogStreamName": { "Ref": "IntegrationCustomIngestionKinesisFirehosefirehoseloggroupfirehoselogstream9C10FF76", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionFirehoseRole740DD3F6", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBCustomIngestionCfncustomingestionCF1D54E2", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/customingestion/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "customingestion/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionFirehoseRole740DD3F6", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationCustomIngestionKinesisFirehoseKinesisFirehosePolicyB44E14D5": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationCustomIngestionKinesisFirehosefirehoseloggroup95E8C3C0", }, ":log-stream:", { "Ref": "IntegrationCustomIngestionKinesisFirehosefirehoseloggroupfirehoselogstream9C10FF76", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionKinesisFirehoseKinesisFirehosePolicyB44E14D5", "Roles": [ { "Ref": "IntegrationCustomIngestionKinesisFirehoseKinesisFirehoseRoleA89BEAFD", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionKinesisFirehoseKinesisFirehoseRoleA89BEAFD": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionKinesisFirehosefirehoseloggroup95E8C3C0": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationCustomIngestionKinesisFirehosefirehoseloggroupfirehoselogstream9C10FF76": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationCustomIngestionKinesisFirehosefirehoseloggroup95E8C3C0", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationCustomIngestionLoudnessFirehoseGlueKms336EC1C0": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionLoudnessFirehoseGlueKms336EC1C0", "Roles": [ { "Ref": "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionLoudnessFirehoseGlueTablePolicy8441ACDA": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBCustomIngestionLoudnessCfncustomingestionloudnessA5C73755", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionLoudnessFirehoseGlueTablePolicy8441ACDA", "Roles": [ { "Ref": "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionLoudnessFirehoseRoleDefaultPolicyE730D8F7": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionLoudnessFirehoseRoleDefaultPolicyE730D8F7", "Roles": [ { "Ref": "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionLoudnessKinesisFirehose5FB054D8": { "DependsOn": [ "IntegrationCustomIngestionLoudnessFirehoseGlueKms336EC1C0", "IntegrationCustomIngestionLoudnessFirehoseGlueTablePolicy8441ACDA", "IntegrationCustomIngestionLoudnessFirehoseRoleDefaultPolicyE730D8F7", "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupA42E9739", }, "LogStreamName": { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupfirehoselogstream68A3768F", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBCustomIngestionLoudnessCfncustomingestionloudnessA5C73755", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/customingestionloudness/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "customingestionloudness/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationCustomIngestionLoudnessFirehoseRoleB6FEE6DF", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationCustomIngestionLoudnessKinesisFirehoseKinesisFirehosePolicyFD03BF21": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupA42E9739", }, ":log-stream:", { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupfirehoselogstream68A3768F", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationCustomIngestionLoudnessKinesisFirehoseKinesisFirehosePolicyFD03BF21", "Roles": [ { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehoseKinesisFirehoseRoleCE598C48", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationCustomIngestionLoudnessKinesisFirehoseKinesisFirehoseRoleCE598C48": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupA42E9739": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupfirehoselogstream68A3768F": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehosefirehoseloggroupA42E9739", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationEntityFirehoseGlueKmsFB7BE402": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationEntityFirehoseGlueKmsFB7BE402", "Roles": [ { "Ref": "IntegrationEntityFirehoseRole6420D932", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationEntityFirehoseGlueTablePolicyA785738F": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBEntityCfnentity9A0483C9", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationEntityFirehoseGlueTablePolicyA785738F", "Roles": [ { "Ref": "IntegrationEntityFirehoseRole6420D932", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationEntityFirehoseRole6420D932": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationEntityFirehoseRoleDefaultPolicy9E2AF819": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationEntityFirehoseRoleDefaultPolicy9E2AF819", "Roles": [ { "Ref": "IntegrationEntityFirehoseRole6420D932", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationEntityKinesisFirehose22678A42": { "DependsOn": [ "IntegrationEntityFirehoseGlueKmsFB7BE402", "IntegrationEntityFirehoseGlueTablePolicyA785738F", "IntegrationEntityFirehoseRoleDefaultPolicy9E2AF819", "IntegrationEntityFirehoseRole6420D932", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationEntityKinesisFirehosefirehoseloggroup7C26E1E6", }, "LogStreamName": { "Ref": "IntegrationEntityKinesisFirehosefirehoseloggroupfirehoselogstreamD3C9FFEB", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationEntityFirehoseRole6420D932", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBEntityCfnentity9A0483C9", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/entity/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "entity/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationEntityFirehoseRole6420D932", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationEntityKinesisFirehoseKinesisFirehosePolicy2ED4D164": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationEntityKinesisFirehosefirehoseloggroup7C26E1E6", }, ":log-stream:", { "Ref": "IntegrationEntityKinesisFirehosefirehoseloggroupfirehoselogstreamD3C9FFEB", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationEntityKinesisFirehoseKinesisFirehosePolicy2ED4D164", "Roles": [ { "Ref": "IntegrationEntityKinesisFirehoseKinesisFirehoseRole5830F5A3", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationEntityKinesisFirehoseKinesisFirehoseRole5830F5A3": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationEntityKinesisFirehosefirehoseloggroup7C26E1E6": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationEntityKinesisFirehosefirehoseloggroupfirehoselogstreamD3C9FFEB": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationEntityKinesisFirehosefirehoseloggroup7C26E1E6", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationEventRuleEventRule0AllowEventRuletestStackIntegrationTextAnalysisLambdaFunction4852E2773B5AC7E8": { "Properties": { "Action": "lambda:InvokeFunction", "FunctionName": { "Fn::GetAtt": [ "IntegrationTextAnalysisLambdaFunctionAF9CFAB5", "Arn", ], }, "Principal": "events.amazonaws.com", "SourceArn": { "Fn::GetAtt": [ "IntegrationEventRuleEventRule0BCDBF110", "Arn", ], }, }, "Type": "AWS::Lambda::Permission", }, "IntegrationEventRuleEventRule0BCDBF110": { "Properties": { "EventBusName": { "Ref": "IntegrationEventRuleInferenceEvents6E8C129E", }, "EventPattern": { "account": [ { "Ref": "AWS::AccountId", }, ], "region": [ { "Ref": "AWS::Region", }, ], "source": [ "com.test", "metadata.call_analytics", ], }, "State": "ENABLED", "Targets": [ { "Arn": { "Fn::GetAtt": [ "IntegrationTextAnalysisLambdaFunctionAF9CFAB5", "Arn", ], }, "Id": "Target0", }, ], }, "Type": "AWS::Events::Rule", }, "IntegrationEventRuleEventRule1928958BC": { "Properties": { "EventBusName": { "Ref": "IntegrationEventRuleInferenceEvents6E8C129E", }, "EventPattern": { "account": [ { "Ref": "AWS::AccountId", }, ], "region": [ { "Ref": "AWS::Region", }, ], "source": [ "com.topic", "com.topic.mappings", ], }, "State": "ENABLED", "Targets": [ { "Arn": { "Fn::GetAtt": [ "IntegrationTopicAnalysisLambdaFunction60A34A39", "Arn", ], }, "Id": "Target0", }, ], }, "Type": "AWS::Events::Rule", }, "IntegrationEventRuleEventRule1AllowEventRuletestStackIntegrationTopicAnalysisLambdaFunctionD9660B2D94760C3B": { "Properties": { "Action": "lambda:InvokeFunction", "FunctionName": { "Fn::GetAtt": [ "IntegrationTopicAnalysisLambdaFunction60A34A39", "Arn", ], }, "Principal": "events.amazonaws.com", "SourceArn": { "Fn::GetAtt": [ "IntegrationEventRuleEventRule1928958BC", "Arn", ], }, }, "Type": "AWS::Lambda::Permission", }, "IntegrationEventRuleInferenceEvents6E8C129E": { "Properties": { "Name": "Integration-AppIntegration", }, "Type": "AWS::Events::EventBus", }, "IntegrationInfDBCustomIngestionCfncustomingestionCF1D54E2": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for customingestion", "Name": "customingestion", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "parent_id", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "lang", "Type": "string", }, { "Name": "_translated_text", "Type": "string", }, { "Name": "_cleansed_text", "Type": "string", }, { "Name": "source_file", "Type": "string", }, { "Name": "Id", "Type": "string", }, { "Name": "BeginOffsetMillis", "Type": "bigint", }, { "Name": "EndOffsetMillis", "Type": "bigint", }, { "Name": "Sentiment", "Type": "string", }, { "Name": "ParticipantRole", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/customingestion/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBCustomIngestionItemCfncustomingestionitemC488BBC0": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for customingestionitem", "Name": "customingestionitem", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "parent_id", "Type": "string", }, { "Name": "Type", "Type": "string", }, { "Name": "Confidence", "Type": "float", }, { "Name": "Content", "Type": "string", }, { "Name": "BeginOffsetMillis", "Type": "bigint", }, { "Name": "EndOffsetMillis", "Type": "bigint", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/customingestionitem/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBCustomIngestionLoudnessCfncustomingestionloudnessA5C73755": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for customingestionloudness", "Name": "customingestionloudness", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "score", "Type": "float", }, { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "parent_id", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/customingestionloudness/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBEntityCfnentity9A0483C9": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for entity", "Name": "entity", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "translated_text", "Type": "string", }, { "Name": "entity_text", "Type": "string", }, { "Name": "entity_type", "Type": "string", }, { "Name": "entity_score", "Type": "double", }, { "Name": "entity_begin_offset", "Type": "int", }, { "Name": "entity_end_offset", "Type": "int", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/entity/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBGlueSecConfigC5DEC89D": { "Properties": { "EncryptionConfiguration": { "CloudWatchEncryption": { "CloudWatchEncryptionMode": "SSE-KMS", "KmsKeyArn": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, "JobBookmarksEncryption": { "JobBookmarksEncryptionMode": "CSE-KMS", "KmsKeyArn": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, "S3Encryptions": [ { "S3EncryptionMode": "SSE-S3", }, ], }, "Name": "socialmediadb-sec-config", }, "Type": "AWS::Glue::SecurityConfiguration", }, "IntegrationInfDBKeyPhraseCfnkeyphraseF95A8695": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for keyphrase", "Name": "keyphrase", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "translated_text", "Type": "string", }, { "Name": "phrase", "Type": "string", }, { "Name": "phrase_score", "Type": "double", }, { "Name": "phrase_begin_offset", "Type": "int", }, { "Name": "phrase_end_offset", "Type": "int", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/keyphrase/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBMetadataCfnmetadata9488F83E": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for metadata", "Name": "metadata", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "parent_id", "Type": "string", }, { "Name": "Categories", "Type": "struct>", }, { "Name": "Channel", "Type": "string", }, { "Name": "JobName", "Type": "string", }, { "Name": "Participants", "Type": "array>", }, { "Name": "ConversationCharacteristics", "Type": "struct>,TotalTimeMillis:bigint>,Interruptions:struct>>>,TotalConversationDurationMillis:bigint,Sentiment:struct,SentimentByPeriod:struct>,CUSTOMER:array>>>>,TalkSpeed:struct,CUSTOMER:struct>>,TalkTime:struct,CUSTOMER:struct>,TotalTimeMillis:bigint>>", }, { "Name": "ContentMetadata", "Type": "struct", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/metadata/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBModerationLabelsCfnmoderationlabels2D8D0975": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for moderationlabels", "Name": "moderationlabels", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "image_url", "Type": "string", }, { "Name": "label_name", "Type": "string", }, { "Name": "confidence", "Type": "double", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/moderationlabels/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBNewsFeedCfnnewsfeedstorage1858269F": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for newsfeedstorage", "Name": "newsfeedstorage", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "entities", "Type": "struct>,media:array>>", }, { "Name": "metadata", "Type": "struct", }, { "Name": "lang", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "translated_text", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/newsfeedstorage/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBRedditCommentsCfnredditcomments9AF5B12F": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for redditcomments", "Name": "redditcomments", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "translated_text", "Type": "string", }, { "Name": "subreddit_id", "Type": "string", }, { "Name": "link_title", "Type": "string", }, { "Name": "ups", "Type": "int", }, { "Name": "total_awards_received", "Type": "int", }, { "Name": "subreddit", "Type": "string", }, { "Name": "link_author", "Type": "string", }, { "Name": "likes", "Type": "string", }, { "Name": "replies", "Type": "string", }, { "Name": "id", "Type": "string", }, { "Name": "author", "Type": "string", }, { "Name": "num_comments", "Type": "int", }, { "Name": "parent_id", "Type": "string", }, { "Name": "score", "Type": "float", }, { "Name": "author_fullname", "Type": "string", }, { "Name": "controversiality", "Type": "float", }, { "Name": "author_premium", "Type": "boolean", }, { "Name": "link_id", "Type": "string", }, { "Name": "permalink", "Type": "string", }, { "Name": "subreddit_type", "Type": "string", }, { "Name": "link_permalink", "Type": "string", }, { "Name": "name", "Type": "string", }, { "Name": "subreddit_name_prefixed", "Type": "string", }, { "Name": "created_utc", "Type": "bigint", }, { "Name": "link_url", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/redditcomments/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBSentimentCfnsentiment765321B6": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for sentiment", "Name": "sentiment", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "translated_text", "Type": "string", }, { "Name": "sentiment", "Type": "string", }, { "Name": "sentimentposscore", "Type": "double", }, { "Name": "sentimentnegscore", "Type": "double", }, { "Name": "sentimentneuscore", "Type": "double", }, { "Name": "sentimentmixscore", "Type": "double", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/sentiment/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTopicMappingsCfntopicmappingsA5BE2E21": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for topic-mappings", "Name": "topic-mappings", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "platform", "Type": "string", }, { "Name": "job_id", "Type": "string", }, { "Name": "job_timestamp", "Type": "timestamp", }, { "Name": "topic", "Type": "string", }, { "Name": "id_str", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/topic-mappings/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTopicsCfntopics92C7BC5A": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for topics", "Name": "topics", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "job_id", "Type": "string", }, { "Name": "job_timestamp", "Type": "timestamp", }, { "Name": "term", "Type": "string", }, { "Name": "weight", "Type": "double", }, { "Name": "topic", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/topics/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTweetDBEncryption7809FB74": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DataCatalogEncryptionSettings": { "EncryptionAtRest": { "CatalogEncryptionMode": "SSE-KMS", "SseAwsKmsKeyId": "alias/aws/glue", }, }, }, "Type": "AWS::Glue::DataCatalogEncryptionSettings", }, "IntegrationInfDBTweetDBF81EA9E5": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseInput": { "Name": "socialmediadb", }, }, "Type": "AWS::Glue::Database", }, "IntegrationInfDBTwitterCfntwfeedstorageDC850E99": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for twfeedstorage", "Name": "twfeedstorage", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "coordinates", "Type": "struct>", }, { "Name": "retweeted", "Type": "boolean", }, { "Name": "source", "Type": "string", }, { "Name": "entities", "Type": "struct>>,urls:array>>>", }, { "Name": "reply_count", "Type": "bigint", }, { "Name": "favorite_count", "Type": "bigint", }, { "Name": "geo", "Type": "struct>", }, { "Name": "id_str", "Type": "string", }, { "Name": "truncated", "Type": "boolean", }, { "Name": "text", "Type": "string", }, { "Name": "retweet_count", "Type": "bigint", }, { "Name": "possibly_sensitive", "Type": "boolean", }, { "Name": "filter_level", "Type": "string", }, { "Name": "place", "Type": "struct>>>>", }, { "Name": "favorited", "Type": "boolean", }, { "Name": "lang", "Type": "string", }, { "Name": "in_reply_to_screen_name", "Type": "string", }, { "Name": "is_quote_status", "Type": "boolean", }, { "Name": "in_reply_to_user_id_str", "Type": "string", }, { "Name": "user", "Type": "struct", }, { "Name": "quote_count", "Type": "bigint", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/twfeedstorage/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTxtInImgEntityCfntxtinimgentityB1EF7546": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for txtinimgentity", "Name": "txtinimgentity", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "entity_text", "Type": "string", }, { "Name": "entity_type", "Type": "string", }, { "Name": "entity_score", "Type": "double", }, { "Name": "entity_begin_offset", "Type": "int", }, { "Name": "entity_end_offset", "Type": "int", }, { "Name": "image_url", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/txtinimgentity/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTxtInImgKeyPhraseCfntxtinimgkeyphrase5C32F0E7": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for txtinimgkeyphrase", "Name": "txtinimgkeyphrase", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "phrase", "Type": "string", }, { "Name": "phrase_score", "Type": "double", }, { "Name": "phrase_begin_offset", "Type": "int", }, { "Name": "phrase_end_offset", "Type": "int", }, { "Name": "image_url", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/txtinimgkeyphrase/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBTxtInImgSentimentCfntxtinimgsentimentE52337E9": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for txtinimgsentiment", "Name": "txtinimgsentiment", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "search_query", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "sentiment", "Type": "string", }, { "Name": "sentimentposscore", "Type": "double", }, { "Name": "sentimentnegscore", "Type": "double", }, { "Name": "sentimentneuscore", "Type": "double", }, { "Name": "sentimentmixscore", "Type": "double", }, { "Name": "image_url", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/txtinimgsentiment/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationInfDBYoutubeCommentsCfnyoutubecommentsE837E65C": { "Properties": { "CatalogId": { "Ref": "AWS::AccountId", }, "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "TableInput": { "Description": "A table created with partition projection for youtubecomments", "Name": "youtubecomments", "Parameters": { "classification": "parquet", "has_encryped_data": false, "projection.created_at.format": "yyyy-MM-dd", "projection.created_at.range": "NOW-45DAYS,NOW", "projection.created_at.type": "date", "projection.enabled": "TRUE", }, "PartitionKeys": [ { "Name": "created_at", "Type": "timestamp", }, ], "StorageDescriptor": { "Columns": [ { "Name": "account_name", "Type": "string", }, { "Name": "platform", "Type": "string", }, { "Name": "id_str", "Type": "string", }, { "Name": "video_id", "Type": "string", }, { "Name": "title", "Type": "string", }, { "Name": "text", "Type": "string", }, { "Name": "parent_id", "Type": "string", }, { "Name": "viewer_rating", "Type": "string", }, { "Name": "like_count", "Type": "int", }, { "Name": "updated_at", "Type": "timestamp", }, { "Name": "search_query", "Type": "string", }, { "Name": "lang", "Type": "string", }, { "Name": "_translated_text", "Type": "string", }, { "Name": "_cleansed_text", "Type": "string", }, ], "Compressed": false, "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", "Location": { "Fn::Join": [ "", [ "s3://", { "Ref": "IntegrationS3Bucket22BFCA0D", }, "/youtubecomments/", ], ], }, "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", "SerdeInfo": { "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", }, "StoredAsSubDirectories": true, }, "TableType": "EXTERNAL_TABLE", }, }, "Type": "AWS::Glue::Table", }, "IntegrationKeyPhraseFirehoseGlueKms91D8A771": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationKeyPhraseFirehoseGlueKms91D8A771", "Roles": [ { "Ref": "IntegrationKeyPhraseFirehoseRoleFAE5F92E", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationKeyPhraseFirehoseGlueTablePolicy02BF04C1": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBKeyPhraseCfnkeyphraseF95A8695", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationKeyPhraseFirehoseGlueTablePolicy02BF04C1", "Roles": [ { "Ref": "IntegrationKeyPhraseFirehoseRoleFAE5F92E", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationKeyPhraseFirehoseRoleDefaultPolicy000C1FBB": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationKeyPhraseFirehoseRoleDefaultPolicy000C1FBB", "Roles": [ { "Ref": "IntegrationKeyPhraseFirehoseRoleFAE5F92E", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationKeyPhraseFirehoseRoleFAE5F92E": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationKeyPhraseKinesisFirehoseEE44A147": { "DependsOn": [ "IntegrationKeyPhraseFirehoseGlueKms91D8A771", "IntegrationKeyPhraseFirehoseGlueTablePolicy02BF04C1", "IntegrationKeyPhraseFirehoseRoleDefaultPolicy000C1FBB", "IntegrationKeyPhraseFirehoseRoleFAE5F92E", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationKeyPhraseKinesisFirehosefirehoseloggroup5FE853F3", }, "LogStreamName": { "Ref": "IntegrationKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream0B309A1B", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationKeyPhraseFirehoseRoleFAE5F92E", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBKeyPhraseCfnkeyphraseF95A8695", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/keyphrase/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "keyphrase/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationKeyPhraseFirehoseRoleFAE5F92E", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationKeyPhraseKinesisFirehoseKinesisFirehosePolicyA504EB8A": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationKeyPhraseKinesisFirehosefirehoseloggroup5FE853F3", }, ":log-stream:", { "Ref": "IntegrationKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream0B309A1B", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationKeyPhraseKinesisFirehoseKinesisFirehosePolicyA504EB8A", "Roles": [ { "Ref": "IntegrationKeyPhraseKinesisFirehoseKinesisFirehoseRole87C0DE63", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationKeyPhraseKinesisFirehoseKinesisFirehoseRole87C0DE63": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationKeyPhraseKinesisFirehosefirehoseloggroup5FE853F3": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream0B309A1B": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationKeyPhraseKinesisFirehosefirehoseloggroup5FE853F3", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationMetadataFirehoseGlueKmsCC580BBB": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationMetadataFirehoseGlueKmsCC580BBB", "Roles": [ { "Ref": "IntegrationMetadataFirehoseRole50F3295B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationMetadataFirehoseGlueTablePolicy47E8267D": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBMetadataCfnmetadata9488F83E", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationMetadataFirehoseGlueTablePolicy47E8267D", "Roles": [ { "Ref": "IntegrationMetadataFirehoseRole50F3295B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationMetadataFirehoseRole50F3295B": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationMetadataFirehoseRoleDefaultPolicy38D4871F": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationMetadataFirehoseRoleDefaultPolicy38D4871F", "Roles": [ { "Ref": "IntegrationMetadataFirehoseRole50F3295B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationMetadataKinesisFirehoseC6557973": { "DependsOn": [ "IntegrationMetadataFirehoseGlueKmsCC580BBB", "IntegrationMetadataFirehoseGlueTablePolicy47E8267D", "IntegrationMetadataFirehoseRoleDefaultPolicy38D4871F", "IntegrationMetadataFirehoseRole50F3295B", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationMetadataKinesisFirehosefirehoseloggroup08C5123C", }, "LogStreamName": { "Ref": "IntegrationMetadataKinesisFirehosefirehoseloggroupfirehoselogstreamAC7F954D", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationMetadataFirehoseRole50F3295B", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBMetadataCfnmetadata9488F83E", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/metadata/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "metadata/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationMetadataFirehoseRole50F3295B", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationMetadataKinesisFirehoseKinesisFirehosePolicyF8C3803C": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationMetadataKinesisFirehosefirehoseloggroup08C5123C", }, ":log-stream:", { "Ref": "IntegrationMetadataKinesisFirehosefirehoseloggroupfirehoselogstreamAC7F954D", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationMetadataKinesisFirehoseKinesisFirehosePolicyF8C3803C", "Roles": [ { "Ref": "IntegrationMetadataKinesisFirehoseKinesisFirehoseRoleEA0CD933", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationMetadataKinesisFirehoseKinesisFirehoseRoleEA0CD933": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationMetadataKinesisFirehosefirehoseloggroup08C5123C": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationMetadataKinesisFirehosefirehoseloggroupfirehoselogstreamAC7F954D": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationMetadataKinesisFirehosefirehoseloggroup08C5123C", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationModerationLabelsFirehoseGlueKmsF1A85007": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationModerationLabelsFirehoseGlueKmsF1A85007", "Roles": [ { "Ref": "IntegrationModerationLabelsFirehoseRoleCE8DFD78", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationModerationLabelsFirehoseGlueTablePolicy6A0A8539": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBModerationLabelsCfnmoderationlabels2D8D0975", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationModerationLabelsFirehoseGlueTablePolicy6A0A8539", "Roles": [ { "Ref": "IntegrationModerationLabelsFirehoseRoleCE8DFD78", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationModerationLabelsFirehoseRoleCE8DFD78": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationModerationLabelsFirehoseRoleDefaultPolicyCBAC0C86": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationModerationLabelsFirehoseRoleDefaultPolicyCBAC0C86", "Roles": [ { "Ref": "IntegrationModerationLabelsFirehoseRoleCE8DFD78", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationModerationLabelsKinesisFirehoseB9DCFAE5": { "DependsOn": [ "IntegrationModerationLabelsFirehoseGlueKmsF1A85007", "IntegrationModerationLabelsFirehoseGlueTablePolicy6A0A8539", "IntegrationModerationLabelsFirehoseRoleDefaultPolicyCBAC0C86", "IntegrationModerationLabelsFirehoseRoleCE8DFD78", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationModerationLabelsKinesisFirehosefirehoseloggroup82FAD98E", }, "LogStreamName": { "Ref": "IntegrationModerationLabelsKinesisFirehosefirehoseloggroupfirehoselogstreamB1C66DFF", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationModerationLabelsFirehoseRoleCE8DFD78", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBModerationLabelsCfnmoderationlabels2D8D0975", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/moderationlabels/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "moderationlabels/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationModerationLabelsFirehoseRoleCE8DFD78", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationModerationLabelsKinesisFirehoseKinesisFirehosePolicyF2E53986": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationModerationLabelsKinesisFirehosefirehoseloggroup82FAD98E", }, ":log-stream:", { "Ref": "IntegrationModerationLabelsKinesisFirehosefirehoseloggroupfirehoselogstreamB1C66DFF", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationModerationLabelsKinesisFirehoseKinesisFirehosePolicyF2E53986", "Roles": [ { "Ref": "IntegrationModerationLabelsKinesisFirehoseKinesisFirehoseRole401AE269", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationModerationLabelsKinesisFirehoseKinesisFirehoseRole401AE269": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationModerationLabelsKinesisFirehosefirehoseloggroup82FAD98E": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationModerationLabelsKinesisFirehosefirehoseloggroupfirehoselogstreamB1C66DFF": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationModerationLabelsKinesisFirehosefirehoseloggroup82FAD98E", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationNewsFeedStorageFirehoseGlueKms20BC4B5C": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationNewsFeedStorageFirehoseGlueKms20BC4B5C", "Roles": [ { "Ref": "IntegrationNewsFeedStorageFirehoseRole452916A8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationNewsFeedStorageFirehoseGlueTablePolicy02CC5F2F": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBNewsFeedCfnnewsfeedstorage1858269F", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationNewsFeedStorageFirehoseGlueTablePolicy02CC5F2F", "Roles": [ { "Ref": "IntegrationNewsFeedStorageFirehoseRole452916A8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationNewsFeedStorageFirehoseRole452916A8": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationNewsFeedStorageFirehoseRoleDefaultPolicy51797843": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationNewsFeedStorageFirehoseRoleDefaultPolicy51797843", "Roles": [ { "Ref": "IntegrationNewsFeedStorageFirehoseRole452916A8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationNewsFeedStorageKinesisFirehose8B6E667E": { "DependsOn": [ "IntegrationNewsFeedStorageFirehoseGlueKms20BC4B5C", "IntegrationNewsFeedStorageFirehoseGlueTablePolicy02CC5F2F", "IntegrationNewsFeedStorageFirehoseRoleDefaultPolicy51797843", "IntegrationNewsFeedStorageFirehoseRole452916A8", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroup25D876B0", }, "LogStreamName": { "Ref": "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstreamD343D310", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationNewsFeedStorageFirehoseRole452916A8", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBNewsFeedCfnnewsfeedstorage1858269F", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/newsfeedstorage/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "newsfeedstorage/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationNewsFeedStorageFirehoseRole452916A8", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationNewsFeedStorageKinesisFirehoseKinesisFirehosePolicyD24D3C7A": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroup25D876B0", }, ":log-stream:", { "Ref": "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstreamD343D310", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationNewsFeedStorageKinesisFirehoseKinesisFirehosePolicyD24D3C7A", "Roles": [ { "Ref": "IntegrationNewsFeedStorageKinesisFirehoseKinesisFirehoseRoleC053D6A1", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationNewsFeedStorageKinesisFirehoseKinesisFirehoseRoleC053D6A1": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroup25D876B0": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstreamD343D310": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationNewsFeedStorageKinesisFirehosefirehoseloggroup25D876B0", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationRedditCommentsFirehoseGlueKms893EB1AB": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationRedditCommentsFirehoseGlueKms893EB1AB", "Roles": [ { "Ref": "IntegrationRedditCommentsFirehoseRoleF335F35C", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationRedditCommentsFirehoseGlueTablePolicy67AC0B91": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBRedditCommentsCfnredditcomments9AF5B12F", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationRedditCommentsFirehoseGlueTablePolicy67AC0B91", "Roles": [ { "Ref": "IntegrationRedditCommentsFirehoseRoleF335F35C", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationRedditCommentsFirehoseRoleDefaultPolicyB84746A4": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationRedditCommentsFirehoseRoleDefaultPolicyB84746A4", "Roles": [ { "Ref": "IntegrationRedditCommentsFirehoseRoleF335F35C", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationRedditCommentsFirehoseRoleF335F35C": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationRedditCommentsKinesisFirehose08A44AB4": { "DependsOn": [ "IntegrationRedditCommentsFirehoseGlueKms893EB1AB", "IntegrationRedditCommentsFirehoseGlueTablePolicy67AC0B91", "IntegrationRedditCommentsFirehoseRoleDefaultPolicyB84746A4", "IntegrationRedditCommentsFirehoseRoleF335F35C", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationRedditCommentsKinesisFirehosefirehoseloggroup8D5772CD", }, "LogStreamName": { "Ref": "IntegrationRedditCommentsKinesisFirehosefirehoseloggroupfirehoselogstreamDBCF0AD3", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationRedditCommentsFirehoseRoleF335F35C", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBRedditCommentsCfnredditcomments9AF5B12F", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/redditcomments/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "redditcomments/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationRedditCommentsFirehoseRoleF335F35C", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationRedditCommentsKinesisFirehoseKinesisFirehosePolicy3D423180": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationRedditCommentsKinesisFirehosefirehoseloggroup8D5772CD", }, ":log-stream:", { "Ref": "IntegrationRedditCommentsKinesisFirehosefirehoseloggroupfirehoselogstreamDBCF0AD3", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationRedditCommentsKinesisFirehoseKinesisFirehosePolicy3D423180", "Roles": [ { "Ref": "IntegrationRedditCommentsKinesisFirehoseKinesisFirehoseRole2B3F4068", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationRedditCommentsKinesisFirehoseKinesisFirehoseRole2B3F4068": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationRedditCommentsKinesisFirehosefirehoseloggroup8D5772CD": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationRedditCommentsKinesisFirehosefirehoseloggroupfirehoselogstreamDBCF0AD3": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationRedditCommentsKinesisFirehosefirehoseloggroup8D5772CD", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationS3Bucket22BFCA0D": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W51", "reason": "This S3 bucket Bucket does not need a bucket policy. The access to the bucket is restricted to Kinesis Fireshose using IAM Role policy", }, ], }, }, "Properties": { "BucketEncryption": { "ServerSideEncryptionConfiguration": [ { "ServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256", }, }, ], }, "LoggingConfiguration": { "DestinationBucketName": { "Ref": "AccessLogD598EBE9", }, "LogFilePrefix": "Integration/", }, "PublicAccessBlockConfiguration": { "BlockPublicAcls": true, "BlockPublicPolicy": true, "IgnorePublicAcls": true, "RestrictPublicBuckets": true, }, }, "Type": "AWS::S3::Bucket", "UpdateReplacePolicy": "Retain", }, "IntegrationS3BucketPolicy3C2EAD91": { "Properties": { "Bucket": { "Ref": "IntegrationS3Bucket22BFCA0D", }, "PolicyDocument": { "Statement": [ { "Action": "s3:*", "Condition": { "Bool": { "aws:SecureTransport": "false", }, }, "Effect": "Deny", "Principal": { "AWS": "*", }, "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, }, "Type": "AWS::S3::BucketPolicy", }, "IntegrationSentimentFirehoseGlueKms369B22FE": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationSentimentFirehoseGlueKms369B22FE", "Roles": [ { "Ref": "IntegrationSentimentFirehoseRoleA51A0692", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationSentimentFirehoseGlueTablePolicyE64C328C": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBSentimentCfnsentiment765321B6", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationSentimentFirehoseGlueTablePolicyE64C328C", "Roles": [ { "Ref": "IntegrationSentimentFirehoseRoleA51A0692", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationSentimentFirehoseRoleA51A0692": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationSentimentFirehoseRoleDefaultPolicyE7E43DB5": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationSentimentFirehoseRoleDefaultPolicyE7E43DB5", "Roles": [ { "Ref": "IntegrationSentimentFirehoseRoleA51A0692", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationSentimentKinesisFirehose1B0D197F": { "DependsOn": [ "IntegrationSentimentFirehoseGlueKms369B22FE", "IntegrationSentimentFirehoseGlueTablePolicyE64C328C", "IntegrationSentimentFirehoseRoleDefaultPolicyE7E43DB5", "IntegrationSentimentFirehoseRoleA51A0692", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationSentimentKinesisFirehosefirehoseloggroup2FEBE358", }, "LogStreamName": { "Ref": "IntegrationSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamCC518130", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationSentimentFirehoseRoleA51A0692", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBSentimentCfnsentiment765321B6", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/sentiment/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "sentiment/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationSentimentFirehoseRoleA51A0692", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationSentimentKinesisFirehoseKinesisFirehosePolicy6257D729": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationSentimentKinesisFirehosefirehoseloggroup2FEBE358", }, ":log-stream:", { "Ref": "IntegrationSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamCC518130", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationSentimentKinesisFirehoseKinesisFirehosePolicy6257D729", "Roles": [ { "Ref": "IntegrationSentimentKinesisFirehoseKinesisFirehoseRole63E5C087", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationSentimentKinesisFirehoseKinesisFirehoseRole63E5C087": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationSentimentKinesisFirehosefirehoseloggroup2FEBE358": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamCC518130": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationSentimentKinesisFirehosefirehoseloggroup2FEBE358", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTextAnalysisLambdaFunctionAF9CFAB5": { "DependsOn": [ "IntegrationTextAnalysisLambdaFunctionServiceRoleDefaultPolicyB94FE1F5", "IntegrationTextAnalysisLambdaFunctionServiceRoleC17422CC", ], "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W58", "reason": "Lambda functions has the required permission to write CloudWatch Logs. It uses custom policy instead of arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole with tighter permissions.", }, { "id": "W89", "reason": "This is not a rule for the general case, just for specific use cases/industries", }, { "id": "W92", "reason": "Impossible for us to define the correct concurrency for clients", }, ], }, }, "Properties": { "Code": { "S3Bucket": { "Ref": "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aS3BucketAB08A02E", }, "S3Key": { "Fn::Join": [ "", [ { "Fn::Select": [ 0, { "Fn::Split": [ "||", { "Ref": "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aS3VersionKey5B136550", }, ], }, ], }, { "Fn::Select": [ 1, { "Fn::Split": [ "||", { "Ref": "AssetParameters09f4ea1cd73c829bbfae7d834c7644c8bb7ae06c10db49af309e099e7328a16aS3VersionKey5B136550", }, ], }, ], }, ], ], }, }, "Environment": { "Variables": { "AWS_NODEJS_CONNECTION_REUSE_ENABLED": "1", "CUSTOMINGESTIONITEM_FEED_STORAGE": { "Ref": "IntegrationCustomIngestionItemKinesisFirehose487658C0", }, "CUSTOMINGESTIONLOUDNESS_FEED_STORAGE": { "Ref": "IntegrationCustomIngestionLoudnessKinesisFirehose5FB054D8", }, "CUSTOMINGESTIONMETADATA_FEED_STORAGE": { "Ref": "IntegrationMetadataKinesisFirehoseC6557973", }, "CUSTOMINGESTION_FEED_STORAGE": { "Ref": "IntegrationCustomIngestionKinesisFirehose9C19C395", }, "ENTITIES_FIREHOSE": { "Ref": "IntegrationEntityKinesisFirehose22678A42", }, "KEYPHRASE_FIREHOSE": { "Ref": "IntegrationKeyPhraseKinesisFirehoseEE44A147", }, "METADATA_NS": "metadata.call_analytics", "MODERATION_LABELS_FIREHOSE": { "Ref": "IntegrationModerationLabelsKinesisFirehoseB9DCFAE5", }, "NEWSFEEDS_FEED_STORAGE": { "Ref": "IntegrationNewsFeedStorageKinesisFirehose8B6E667E", }, "REDDIT_FEED_STORAGE": { "Ref": "IntegrationRedditCommentsKinesisFirehose08A44AB4", }, "SENTIMENT_FIREHOSE": { "Ref": "IntegrationSentimentKinesisFirehose1B0D197F", }, "TEXT_ANALYSIS_NS": "com.test", "TW_FEED_STORAGE": { "Ref": "IntegrationTwFeedStorageKinesisFirehose2717DCB9", }, "TXT_IN_IMG_ENTITY_FIREHOSE": { "Ref": "IntegrationTxtInImgEntityKinesisFirehose9B2B792F", }, "TXT_IN_IMG_KEYPHRASE_FIREHOSE": { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehose33549B85", }, "TXT_IN_IMG_SENTIMENT_FIREHOSE": { "Ref": "IntegrationTxtInImgSentimentKinesisFirehose853B28BB", }, "YOUTUBECOMMENTS_FEED_STORAGE": { "Ref": "IntegrationYouTubeCommentsKinesisFirehose9913E37F", }, }, }, "Handler": "index.handler", "Role": { "Fn::GetAtt": [ "IntegrationTextAnalysisLambdaFunctionServiceRoleC17422CC", "Arn", ], }, "Runtime": "nodejs14.x", "Timeout": 900, "TracingConfig": { "Mode": "Active", }, }, "Type": "AWS::Lambda::Function", }, "IntegrationTextAnalysisLambdaFunctionServiceRoleC17422CC": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com", }, }, ], "Version": "2012-10-17", }, "Policies": [ { "PolicyDocument": { "Statement": [ { "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", ], "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:/aws/lambda/*", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "LambdaFunctionServiceRolePolicy", }, ], }, "Type": "AWS::IAM::Role", }, "IntegrationTextAnalysisLambdaFunctionServiceRoleDefaultPolicyB94FE1F5": { "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W76", "reason": "The lambda role policy requires to access multiple firehose buckets. Hence suppressing the SPCM rule", }, { "id": "W12", "reason": "Lambda needs the following minimum required permissions to send trace data to X-Ray and access ENIs in a VPC.", }, ], }, }, "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "xray:PutTraceSegments", "xray:PutTelemetryRecords", ], "Effect": "Allow", "Resource": "*", }, { "Action": [ "firehose:PutRecord", "firehose:PutRecordBatch", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationSentimentKinesisFirehose1B0D197F", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationEntityKinesisFirehose22678A42", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationKeyPhraseKinesisFirehoseEE44A147", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationTxtInImgSentimentKinesisFirehose853B28BB", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationTxtInImgEntityKinesisFirehose9B2B792F", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationTxtInImgKeyPhraseKinesisFirehose33549B85", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationModerationLabelsKinesisFirehoseB9DCFAE5", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationTwFeedStorageKinesisFirehose2717DCB9", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationNewsFeedStorageKinesisFirehose8B6E667E", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationYouTubeCommentsKinesisFirehose9913E37F", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationCustomIngestionKinesisFirehose9C19C395", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationCustomIngestionLoudnessKinesisFirehose5FB054D8", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationCustomIngestionItemKinesisFirehose487658C0", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationMetadataKinesisFirehoseC6557973", "Arn", ], }, { "Fn::GetAtt": [ "IntegrationRedditCommentsKinesisFirehose08A44AB4", "Arn", ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTextAnalysisLambdaFunctionServiceRoleDefaultPolicyB94FE1F5", "Roles": [ { "Ref": "IntegrationTextAnalysisLambdaFunctionServiceRoleC17422CC", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicAnalysisLambdaFunction60A34A39": { "DependsOn": [ "IntegrationTopicAnalysisLambdaFunctionServiceRoleDefaultPolicyB4E2C8C3", "IntegrationTopicAnalysisLambdaFunctionServiceRoleDDC981DF", ], "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W58", "reason": "Lambda functions has the required permission to write CloudWatch Logs. It uses custom policy instead of arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole with tighter permissions.", }, { "id": "W89", "reason": "This is not a rule for the general case, just for specific use cases/industries", }, { "id": "W92", "reason": "Impossible for us to define the correct concurrency for clients", }, ], }, }, "Properties": { "Code": { "S3Bucket": { "Ref": "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464S3Bucket65363AF3", }, "S3Key": { "Fn::Join": [ "", [ { "Fn::Select": [ 0, { "Fn::Split": [ "||", { "Ref": "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464S3VersionKey16E8A40A", }, ], }, ], }, { "Fn::Select": [ 1, { "Fn::Split": [ "||", { "Ref": "AssetParameters8b91b6e5006db6d50421e4fb40dd5c646cdf2af765de93fc5072c7026fcbf464S3VersionKey16E8A40A", }, ], }, ], }, ], ], }, }, "Environment": { "Variables": { "TOPICS_FIREHOSE": { "Ref": "IntegrationTopicsKinesisFirehose9B7E7760", }, "TOPICS_NS": "com.topic", "TOPIC_MAPPINGS_FIREHOSE": { "Ref": "IntegrationTopicMappingsKinesisFirehoseD34A1F74", }, "TOPIC_MAPPINGS_NS": "com.topic.mappings", }, }, "Handler": "lambda_function.handler", "MemorySize": 256, "Role": { "Fn::GetAtt": [ "IntegrationTopicAnalysisLambdaFunctionServiceRoleDDC981DF", "Arn", ], }, "Runtime": "python3.8", "Timeout": 600, "TracingConfig": { "Mode": "Active", }, }, "Type": "AWS::Lambda::Function", }, "IntegrationTopicAnalysisLambdaFunctionServiceRoleDDC981DF": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com", }, }, ], "Version": "2012-10-17", }, "Policies": [ { "PolicyDocument": { "Statement": [ { "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents", ], "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:/aws/lambda/*", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "LambdaFunctionServiceRolePolicy", }, ], }, "Type": "AWS::IAM::Role", }, "IntegrationTopicAnalysisLambdaFunctionServiceRoleDefaultPolicyB4E2C8C3": { "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W12", "reason": "Lambda needs the following minimum required permissions to send trace data to X-Ray and access ENIs in a VPC.", }, ], }, }, "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "xray:PutTraceSegments", "xray:PutTelemetryRecords", ], "Effect": "Allow", "Resource": "*", }, { "Action": [ "firehose:PutRecord", "firehose:PutRecordBatch", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":firehose:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":deliverystream/", { "Ref": "IntegrationTopicsKinesisFirehose9B7E7760", }, ], ], }, { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":firehose:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":deliverystream/", { "Ref": "IntegrationTopicMappingsKinesisFirehoseD34A1F74", }, ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicAnalysisLambdaFunctionServiceRoleDefaultPolicyB4E2C8C3", "Roles": [ { "Ref": "IntegrationTopicAnalysisLambdaFunctionServiceRoleDDC981DF", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicMappingsFirehoseGlueKms6C9EC083": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicMappingsFirehoseGlueKms6C9EC083", "Roles": [ { "Ref": "IntegrationTopicMappingsFirehoseRoleD0E34058", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicMappingsFirehoseGlueTablePolicyE583D1F0": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTopicMappingsCfntopicmappingsA5BE2E21", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicMappingsFirehoseGlueTablePolicyE583D1F0", "Roles": [ { "Ref": "IntegrationTopicMappingsFirehoseRoleD0E34058", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicMappingsFirehoseRoleD0E34058": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTopicMappingsFirehoseRoleDefaultPolicy3A02C4BF": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicMappingsFirehoseRoleDefaultPolicy3A02C4BF", "Roles": [ { "Ref": "IntegrationTopicMappingsFirehoseRoleD0E34058", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicMappingsKinesisFirehoseD34A1F74": { "DependsOn": [ "IntegrationTopicMappingsFirehoseGlueKms6C9EC083", "IntegrationTopicMappingsFirehoseGlueTablePolicyE583D1F0", "IntegrationTopicMappingsFirehoseRoleDefaultPolicy3A02C4BF", "IntegrationTopicMappingsFirehoseRoleD0E34058", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupDAF73689", }, "LogStreamName": { "Ref": "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupfirehoselogstreamBBFB336F", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTopicMappingsFirehoseRoleD0E34058", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTopicMappingsCfntopicmappingsA5BE2E21", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/topic-mappings/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "topic-mappings/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTopicMappingsFirehoseRoleD0E34058", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTopicMappingsKinesisFirehoseKinesisFirehosePolicy34046622": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupDAF73689", }, ":log-stream:", { "Ref": "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupfirehoselogstreamBBFB336F", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicMappingsKinesisFirehoseKinesisFirehosePolicy34046622", "Roles": [ { "Ref": "IntegrationTopicMappingsKinesisFirehoseKinesisFirehoseRoleEA13EA65", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicMappingsKinesisFirehoseKinesisFirehoseRoleEA13EA65": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupDAF73689": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupfirehoselogstreamBBFB336F": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTopicMappingsKinesisFirehosefirehoseloggroupDAF73689", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTopicsFirehoseGlueKms9019CF4A": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicsFirehoseGlueKms9019CF4A", "Roles": [ { "Ref": "IntegrationTopicsFirehoseRoleAB51B4DE", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicsFirehoseGlueTablePolicy8DC575DB": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTopicsCfntopics92C7BC5A", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicsFirehoseGlueTablePolicy8DC575DB", "Roles": [ { "Ref": "IntegrationTopicsFirehoseRoleAB51B4DE", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicsFirehoseRoleAB51B4DE": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTopicsFirehoseRoleDefaultPolicy5FC191A2": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicsFirehoseRoleDefaultPolicy5FC191A2", "Roles": [ { "Ref": "IntegrationTopicsFirehoseRoleAB51B4DE", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicsKinesisFirehose9B7E7760": { "DependsOn": [ "IntegrationTopicsFirehoseGlueKms9019CF4A", "IntegrationTopicsFirehoseGlueTablePolicy8DC575DB", "IntegrationTopicsFirehoseRoleDefaultPolicy5FC191A2", "IntegrationTopicsFirehoseRoleAB51B4DE", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTopicsKinesisFirehosefirehoseloggroup07A1FEDB", }, "LogStreamName": { "Ref": "IntegrationTopicsKinesisFirehosefirehoseloggroupfirehoselogstream3EB07CDB", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTopicsFirehoseRoleAB51B4DE", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTopicsCfntopics92C7BC5A", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/topics/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "topics/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTopicsFirehoseRoleAB51B4DE", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTopicsKinesisFirehoseKinesisFirehosePolicy0D3750C7": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTopicsKinesisFirehosefirehoseloggroup07A1FEDB", }, ":log-stream:", { "Ref": "IntegrationTopicsKinesisFirehosefirehoseloggroupfirehoselogstream3EB07CDB", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTopicsKinesisFirehoseKinesisFirehosePolicy0D3750C7", "Roles": [ { "Ref": "IntegrationTopicsKinesisFirehoseKinesisFirehoseRoleD9F19588", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTopicsKinesisFirehoseKinesisFirehoseRoleD9F19588": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTopicsKinesisFirehosefirehoseloggroup07A1FEDB": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTopicsKinesisFirehosefirehoseloggroupfirehoselogstream3EB07CDB": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTopicsKinesisFirehosefirehoseloggroup07A1FEDB", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTwFeedStorageFirehoseGlueKms82D47141": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTwFeedStorageFirehoseGlueKms82D47141", "Roles": [ { "Ref": "IntegrationTwFeedStorageFirehoseRoleBDA1E404", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTwFeedStorageFirehoseGlueTablePolicy53C5060D": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTwitterCfntwfeedstorageDC850E99", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTwFeedStorageFirehoseGlueTablePolicy53C5060D", "Roles": [ { "Ref": "IntegrationTwFeedStorageFirehoseRoleBDA1E404", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTwFeedStorageFirehoseRoleBDA1E404": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTwFeedStorageFirehoseRoleDefaultPolicyB6532B5F": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTwFeedStorageFirehoseRoleDefaultPolicyB6532B5F", "Roles": [ { "Ref": "IntegrationTwFeedStorageFirehoseRoleBDA1E404", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTwFeedStorageKinesisFirehose2717DCB9": { "DependsOn": [ "IntegrationTwFeedStorageFirehoseGlueKms82D47141", "IntegrationTwFeedStorageFirehoseGlueTablePolicy53C5060D", "IntegrationTwFeedStorageFirehoseRoleDefaultPolicyB6532B5F", "IntegrationTwFeedStorageFirehoseRoleBDA1E404", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroup945A718C", }, "LogStreamName": { "Ref": "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstream797FAE95", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTwFeedStorageFirehoseRoleBDA1E404", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTwitterCfntwfeedstorageDC850E99", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/twfeedstorage/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "twfeedstorage/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTwFeedStorageFirehoseRoleBDA1E404", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTwFeedStorageKinesisFirehoseKinesisFirehosePolicyDED01CB7": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroup945A718C", }, ":log-stream:", { "Ref": "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstream797FAE95", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTwFeedStorageKinesisFirehoseKinesisFirehosePolicyDED01CB7", "Roles": [ { "Ref": "IntegrationTwFeedStorageKinesisFirehoseKinesisFirehoseRole8ACF28E6", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTwFeedStorageKinesisFirehoseKinesisFirehoseRole8ACF28E6": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroup945A718C": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroupfirehoselogstream797FAE95": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTwFeedStorageKinesisFirehosefirehoseloggroup945A718C", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgEntityFirehoseGlueKms13C70E5E": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgEntityFirehoseGlueKms13C70E5E", "Roles": [ { "Ref": "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgEntityFirehoseGlueTablePolicy9C105C91": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTxtInImgEntityCfntxtinimgentityB1EF7546", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgEntityFirehoseGlueTablePolicy9C105C91", "Roles": [ { "Ref": "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgEntityFirehoseRoleDefaultPolicy85C649FC": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgEntityFirehoseRoleDefaultPolicy85C649FC", "Roles": [ { "Ref": "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgEntityFirehoseRoleF8A843A9": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgEntityKinesisFirehose9B2B792F": { "DependsOn": [ "IntegrationTxtInImgEntityFirehoseGlueKms13C70E5E", "IntegrationTxtInImgEntityFirehoseGlueTablePolicy9C105C91", "IntegrationTxtInImgEntityFirehoseRoleDefaultPolicy85C649FC", "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroup4DE3574A", }, "LogStreamName": { "Ref": "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroupfirehoselogstreamE174C06B", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTxtInImgEntityCfntxtinimgentityB1EF7546", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/txtinimgentity/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "txtinimgentity/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgEntityFirehoseRoleF8A843A9", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTxtInImgEntityKinesisFirehoseKinesisFirehosePolicy197A3750": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroup4DE3574A", }, ":log-stream:", { "Ref": "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroupfirehoselogstreamE174C06B", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgEntityKinesisFirehoseKinesisFirehosePolicy197A3750", "Roles": [ { "Ref": "IntegrationTxtInImgEntityKinesisFirehoseKinesisFirehoseRole16EADE58", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgEntityKinesisFirehoseKinesisFirehoseRole16EADE58": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroup4DE3574A": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroupfirehoselogstreamE174C06B": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTxtInImgEntityKinesisFirehosefirehoseloggroup4DE3574A", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgKeyPhraseFirehoseGlueKms7B90E120": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgKeyPhraseFirehoseGlueKms7B90E120", "Roles": [ { "Ref": "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgKeyPhraseFirehoseGlueTablePolicyF70EA936": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTxtInImgKeyPhraseCfntxtinimgkeyphrase5C32F0E7", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgKeyPhraseFirehoseGlueTablePolicyF70EA936", "Roles": [ { "Ref": "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgKeyPhraseFirehoseRoleDefaultPolicy1518F405": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgKeyPhraseFirehoseRoleDefaultPolicy1518F405", "Roles": [ { "Ref": "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgKeyPhraseKinesisFirehose33549B85": { "DependsOn": [ "IntegrationTxtInImgKeyPhraseFirehoseGlueKms7B90E120", "IntegrationTxtInImgKeyPhraseFirehoseGlueTablePolicyF70EA936", "IntegrationTxtInImgKeyPhraseFirehoseRoleDefaultPolicy1518F405", "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroup990C7C5B", }, "LogStreamName": { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream839D676F", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTxtInImgKeyPhraseCfntxtinimgkeyphrase5C32F0E7", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/txtinimgkeyphrase/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "txtinimgkeyphrase/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgKeyPhraseFirehoseRole7FD100B8", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTxtInImgKeyPhraseKinesisFirehoseKinesisFirehosePolicyD3761F10": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroup990C7C5B", }, ":log-stream:", { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream839D676F", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgKeyPhraseKinesisFirehoseKinesisFirehosePolicyD3761F10", "Roles": [ { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehoseKinesisFirehoseRoleBB967F06", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgKeyPhraseKinesisFirehoseKinesisFirehoseRoleBB967F06": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroup990C7C5B": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroupfirehoselogstream839D676F": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTxtInImgKeyPhraseKinesisFirehosefirehoseloggroup990C7C5B", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgSentimentFirehoseGlueKms4D3BC908": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgSentimentFirehoseGlueKms4D3BC908", "Roles": [ { "Ref": "IntegrationTxtInImgSentimentFirehoseRole3309E38D", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgSentimentFirehoseGlueTablePolicy533DD464": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBTxtInImgSentimentCfntxtinimgsentimentE52337E9", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgSentimentFirehoseGlueTablePolicy533DD464", "Roles": [ { "Ref": "IntegrationTxtInImgSentimentFirehoseRole3309E38D", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgSentimentFirehoseRole3309E38D": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgSentimentFirehoseRoleDefaultPolicyC2C3D583": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgSentimentFirehoseRoleDefaultPolicyC2C3D583", "Roles": [ { "Ref": "IntegrationTxtInImgSentimentFirehoseRole3309E38D", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgSentimentKinesisFirehose853B28BB": { "DependsOn": [ "IntegrationTxtInImgSentimentFirehoseGlueKms4D3BC908", "IntegrationTxtInImgSentimentFirehoseGlueTablePolicy533DD464", "IntegrationTxtInImgSentimentFirehoseRoleDefaultPolicyC2C3D583", "IntegrationTxtInImgSentimentFirehoseRole3309E38D", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroup4E278475", }, "LogStreamName": { "Ref": "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamF4600A5E", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgSentimentFirehoseRole3309E38D", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBTxtInImgSentimentCfntxtinimgsentimentE52337E9", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/txtinimgsentiment/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "txtinimgsentiment/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationTxtInImgSentimentFirehoseRole3309E38D", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationTxtInImgSentimentKinesisFirehoseKinesisFirehosePolicyFD497AB1": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroup4E278475", }, ":log-stream:", { "Ref": "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamF4600A5E", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationTxtInImgSentimentKinesisFirehoseKinesisFirehosePolicyFD497AB1", "Roles": [ { "Ref": "IntegrationTxtInImgSentimentKinesisFirehoseKinesisFirehoseRole86A690E2", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationTxtInImgSentimentKinesisFirehoseKinesisFirehoseRole86A690E2": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroup4E278475": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroupfirehoselogstreamF4600A5E": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationTxtInImgSentimentKinesisFirehosefirehoseloggroup4E278475", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, "IntegrationYouTubeCommentsFirehoseGlueKms6F8A6FCD": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": "kms:Decrypt", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/glue", ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationYouTubeCommentsFirehoseGlueKms6F8A6FCD", "Roles": [ { "Ref": "IntegrationYouTubeCommentsFirehoseRole207AE93B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationYouTubeCommentsFirehoseGlueTablePolicy4203E154": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "glue:GetTable", "glue:GetTableVersion", "glue:GetTableVersions", ], "Effect": "Allow", "Resource": [ { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":table/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "/", { "Ref": "IntegrationInfDBYoutubeCommentsCfnyoutubecommentsE837E65C", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":database/", { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, ], ], }, { "Fn::Join": [ "", [ "arn:aws:glue:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":catalog", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationYouTubeCommentsFirehoseGlueTablePolicy4203E154", "Roles": [ { "Ref": "IntegrationYouTubeCommentsFirehoseRole207AE93B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationYouTubeCommentsFirehoseRole207AE93B": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationYouTubeCommentsFirehoseRoleDefaultPolicy51CF8EFC": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", "s3:DeleteObject*", "s3:PutObject*", "s3:Abort*", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationYouTubeCommentsFirehoseRoleDefaultPolicy51CF8EFC", "Roles": [ { "Ref": "IntegrationYouTubeCommentsFirehoseRole207AE93B", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationYouTubeCommentsKinesisFirehose9913E37F": { "DependsOn": [ "IntegrationYouTubeCommentsFirehoseGlueKms6F8A6FCD", "IntegrationYouTubeCommentsFirehoseGlueTablePolicy4203E154", "IntegrationYouTubeCommentsFirehoseRoleDefaultPolicy51CF8EFC", "IntegrationYouTubeCommentsFirehoseRole207AE93B", ], "Properties": { "DeliveryStreamEncryptionConfigurationInput": { "KeyType": "AWS_OWNED_CMK", }, "DeliveryStreamType": "DirectPut", "ExtendedS3DestinationConfiguration": { "BucketARN": { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "BufferingHints": { "IntervalInSeconds": 600, "SizeInMBs": 128, }, "CloudWatchLoggingOptions": { "Enabled": true, "LogGroupName": { "Ref": "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroup992BBAC5", }, "LogStreamName": { "Ref": "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroupfirehoselogstream648ED089", }, }, "CompressionFormat": "UNCOMPRESSED", "DataFormatConversionConfiguration": { "InputFormatConfiguration": { "Deserializer": { "OpenXJsonSerDe": {}, }, }, "OutputFormatConfiguration": { "Serializer": { "ParquetSerDe": {}, }, }, "SchemaConfiguration": { "DatabaseName": { "Ref": "IntegrationInfDBTweetDBF81EA9E5", }, "RoleARN": { "Fn::GetAtt": [ "IntegrationYouTubeCommentsFirehoseRole207AE93B", "Arn", ], }, "TableName": { "Ref": "IntegrationInfDBYoutubeCommentsCfnyoutubecommentsE837E65C", }, }, }, "DynamicPartitioningConfiguration": { "Enabled": true, "RetryOptions": { "DurationInSeconds": 300, }, }, "EncryptionConfiguration": { "KMSEncryptionConfig": { "AWSKMSKeyARN": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":kms:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":alias/aws/s3", ], ], }, }, }, "ErrorOutputPrefix": "error/youtubecomments/!{firehose:random-string}/!{firehose:error-output-type}/created_at=!{timestamp:yyyy-MM-dd}/", "Prefix": "youtubecomments/created_at=!{partitionKeyFromQuery:created_at}/", "ProcessingConfiguration": { "Enabled": true, "Processors": [ { "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{created_at: .created_at | strptime("%Y-%m-%d %H:%M:%S") | strftime("%Y-%m-%d")}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6", }, ], "Type": "MetadataExtraction", }, ], }, "RoleARN": { "Fn::GetAtt": [ "IntegrationYouTubeCommentsFirehoseRole207AE93B", "Arn", ], }, }, }, "Type": "AWS::KinesisFirehose::DeliveryStream", }, "IntegrationYouTubeCommentsKinesisFirehoseKinesisFirehosePolicy7AF69C21": { "Properties": { "PolicyDocument": { "Statement": [ { "Action": [ "s3:AbortMultipartUpload", "s3:GetBucketLocation", "s3:GetObject", "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:PutObject", ], "Effect": "Allow", "Resource": [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, { "Fn::Join": [ "", [ { "Fn::GetAtt": [ "IntegrationS3Bucket22BFCA0D", "Arn", ], }, "/*", ], ], }, ], }, { "Action": "logs:PutLogEvents", "Effect": "Allow", "Resource": { "Fn::Join": [ "", [ "arn:", { "Ref": "AWS::Partition", }, ":logs:", { "Ref": "AWS::Region", }, ":", { "Ref": "AWS::AccountId", }, ":log-group:", { "Ref": "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroup992BBAC5", }, ":log-stream:", { "Ref": "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroupfirehoselogstream648ED089", }, ], ], }, }, ], "Version": "2012-10-17", }, "PolicyName": "IntegrationYouTubeCommentsKinesisFirehoseKinesisFirehosePolicy7AF69C21", "Roles": [ { "Ref": "IntegrationYouTubeCommentsKinesisFirehoseKinesisFirehoseRoleEED13794", }, ], }, "Type": "AWS::IAM::Policy", }, "IntegrationYouTubeCommentsKinesisFirehoseKinesisFirehoseRoleEED13794": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ { "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "firehose.amazonaws.com", }, }, ], "Version": "2012-10-17", }, }, "Type": "AWS::IAM::Role", }, "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroup992BBAC5": { "DeletionPolicy": "Retain", "Metadata": { "cfn_nag": { "rules_to_suppress": [ { "id": "W86", "reason": "Retention period for CloudWatchLogs LogGroups are set to 'Never Expire' to preserve customer data indefinitely", }, { "id": "W84", "reason": "By default CloudWatchLogs LogGroups data is encrypted using the CloudWatch server-side encryption keys (AWS Managed Keys)", }, ], }, }, "Type": "AWS::Logs::LogGroup", "UpdateReplacePolicy": "Retain", }, "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroupfirehoselogstream648ED089": { "DeletionPolicy": "Retain", "Properties": { "LogGroupName": { "Ref": "IntegrationYouTubeCommentsKinesisFirehosefirehoseloggroup992BBAC5", }, }, "Type": "AWS::Logs::LogStream", "UpdateReplacePolicy": "Retain", }, }, } `;