AWSTemplateFormatVersion: "2010-09-09"
Description: Media2Cloud Backend Stack - create backend resources such as Amazon DynamoDB tables, AWS Step Functions, AWS Lambda functions and layers, Amazon EventBridge, Amazon CloudWatch Events, Amazon SNS, Amazon S3 notification, and so forth. Version %%VERSION%%
Mappings:
Workflow:
# Backlog workflow
BacklogStatusUpdater:
Package: "%%PKG_BACKLOG_STATUS_UPDATER%%"
Name: backlog-status-updater
BacklogStreamConnector:
Package: "%%PKG_BACKLOG_STREAM_CONNECTOR%%"
Name: backlog-stream-connector
BacklogCustomLabels:
Package: "%%PKG_BACKLOG_CUSTOMLABELS%%"
Name: backlog-custom-labels
# Ingest workflow
IngestFixity:
Package: "%%PKG_INGEST_FIXITY%%"
Name: ingest-fixity
IngestVideo:
Package: "%%PKG_INGEST_VIDEO%%"
Name: ingest-video
IngestAudio:
Package: "%%PKG_INGEST_AUDIO%%"
Name: ingest-audio
IngestImage:
Package: "%%PKG_INGEST_IMAGE%%"
Name: ingest-image
IngestDocument:
Package: "%%PKG_INGEST_DOCUMENT%%"
Name: ingest-document
IngestMain:
Package: "%%PKG_INGEST_MAIN%%"
Name: ingest-main
IngestStatusUpdater:
Package: "%%PKG_INGEST_STATUS_UPDATER%%"
Name: ingest-status-updater
# Analysis workflow
AnalysisVideo:
Package: "%%PKG_ANALYSIS_VIDEO%%"
Name: analysis-video
AnalysisAudio:
Package: "%%PKG_ANALYSIS_AUDIO%%"
Name: analysis-audio
AnalysisImage:
Package: "%%PKG_ANALYSIS_IMAGE%%"
Name: analysis-image
# Docker image containerized lambda
BlipModel:
Name: blip-model
AnalysisDocument:
Package: "%%PKG_ANALYSIS_DOCUMENT%%"
Name: analysis-document
AnalysisMain:
Package: "%%PKG_ANALYSIS_MAIN%%"
Name: analysis-main
AnalysisStatusUpdater:
Package: "%%PKG_ANALYSIS_STATUS_UPDATER%%"
Name: analysis-status-updater
# Main Workflow
Main:
Name: main
StateMachineErrorHandler:
Package: "%%PKG_ERROR_HANDLER%%"
Name: main-error-handler
IngestS3Event:
Package: "%%PKG_MAIN_S3EVENT%%"
Name: main-s3event
Layer:
MediaInfoLib:
Package: "%%LAYER_MEDIAINFO%%"
Name: mediainfo
ImageProcessLib:
Package: "%%LAYER_IMAGE_PROCESS%%"
Name: image-process-lib
FixityLib:
Package: "%%LAYER_FIXITY_LIB%%"
Name: fixity-lib
PDFLib:
Package: "%%LAYER_PDF_LIB%%"
Name: pdf-lib
CanvasLib:
Package: "%%LAYER_CANVAS_LIB%%"
Name: canvas-lib
BacklogLib:
Package: "%%LAYER_BACKLOG%%"
Name: service-backlog-lib
DynamoDB:
Backlog:
Suffix: service-backlog
PartitionKey: id
SortKey: serviceApi
# status-timestamp global secondary index
GSIStatusIndex: gsi-status-timestamp
GSIStatusPartitionKey: status
GSIStatusSortKey: timestamp
# jobId global secondary index
GSIJobIdIndex: gsi-jobId
GSIJobIdPartitionKey: jobId
AtomicLock:
Suffix: atomic-lock
PartitionKey: lockId
ServiceToken:
Suffix: service-token
PartitionKey: uuid
SortKey: keyword
Ingest:
Suffix: ingest
PartitionKey: uuid
# schemaversion-timestamp global secondary index
SchemaVersionIndexName: gsi-schemaversion-timestamp
SchemaVersionIndexKey: schemaVersion
GSISortKey: timestamp
# type-timestamp global secondary index
TypeIndexName: gsi-type-timestamp
TypeIndexKey: type
# group-timestamp global secondary index
GroupIndexName: gsi-group-timestamp
GroupIndexKey: group
# overallstatus-timestamp global secondary index
StatusIndexName: gsi-overallstatus-timestamp
StatusIndexKey: overallStatus
Analysis:
Suffix: aiml
PartitionKey: uuid
SortKey: type
EventBridge:
Bus:
Name: service-backlog
Rule:
Name: StatusChange
Detail: Service Backlog Status Change
Source: custom.servicebacklog
SNS:
Topic:
Name: status
DisplayName: M2CStatus
BacklogTopic:
Name: service-backlog-topic
DisplayName: BacklogSNS
Node:
Runtime:
Version: nodejs14.x
Parameters:
S3Bucket:
Type: String
Description: S3Bucket
KeyPrefix:
Type: String
Description: KeyPrefix
SolutionId:
Type: String
Description: SolutionId
SolutionLowerCaseId:
Type: String
Description: SolutionLowerCaseId
ResourcePrefix:
Type: String
Description: ResourcePrefix
CustomUserAgent:
Type: String
Description: CustomUserAgent
AnonymousUsage:
Type: String
Description: AnonymousUsage
CustomResourcesLambdaArn:
Type: String
Description: CustomResourcesLambdaArn
CustomResourcesRoleName:
Type: String
Description: CustomResourcesRoleName
AwsSdkLayer:
Type: String
Description: AwsSdkLayer
CoreLibLayer:
Type: String
Description: CoreLibLayer
SolutionUuid:
Type: String
Description: SolutionUuid
IotHost:
Type: String
Description: IotHost
IotTopic:
Type: String
Description: IotTopic
IngestBucket:
Type: String
Description: IngestBucket
ProxyBucket:
Type: String
Description: ProxyBucket
OpenSearchDomainName:
Type: String
Description: OpenSearchDomainName
OpenSearchDomainEndpoint:
Type: String
Description: OpenSearchDomainEndpoint
DefaultAIOptions:
Type: String
Description: DefaultAIOptions
DefaultMinConfidence:
Type: String
Description: DefaultMinConfidence
MediaConvertEndpoint:
Type: String
Description: MediaConvertEndpoint
StartOnObjectCreation:
Type: String
Description: StartOnObjectCreation
AIOptionsS3Key:
Type: String
Description: AIOptionsS3Key
BlipImageArn:
Type: String
Description: BlipImageArn
Conditions:
bStartOnObjectCreation: !Equals
- !Ref StartOnObjectCreation
- "YES"
bBlipImageArn: !Not
- !Equals
- !Ref BlipImageArn
- ""
Resources:
################################################################################
#
# Lambda Layers for backend
# * MediaInfoLayer
# * ImageProcessLayer
# * FixityLibLayer
# * PDFLibLayer
# * CanvasLibLayer
# * BacklogLayer
#
################################################################################
MediaInfoLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- MediaInfoLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- MediaInfoLib
- Package
Description: !Sub (${SolutionLowerCaseId}) mediainfo library layer
LicenseInfo: Apache-2.0
ImageProcessLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- ImageProcessLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- ImageProcessLib
- Package
Description: !Sub (${SolutionLowerCaseId}) image process library layer (EXIFtool and JIMP)
LicenseInfo: Apache-2.0
FixityLibLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- FixityLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- FixityLib
- Package
Description: !Sub (${SolutionLowerCaseId}) fixity library layer (SPARK and RUSHA)
LicenseInfo: Apache-2.0
PDFLibLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- PDFLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- PDFLib
- Package
Description: !Sub (${SolutionLowerCaseId}) pdf library layer (PDF.JS)
LicenseInfo: Apache-2.0
CanvasLibLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- CanvasLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- CanvasLib
- Package
Description: !Sub (${SolutionLowerCaseId}) nodeJS canvas layer
LicenseInfo: Apache-2.0
BacklogLibLayer:
Type: AWS::Lambda::LayerVersion
Properties:
LayerName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Layer
- BacklogLib
- Name
CompatibleRuntimes:
- !FindInMap
- Node
- Runtime
- Version
Content:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Layer
- BacklogLib
- Package
Description: !Sub (${SolutionLowerCaseId}) service backlog queue management library layer
LicenseInfo: Apache-2.0
################################################################################
#
# Amazon DynamoDB tables
# * BacklogTable: Queue service requests
# * AtomicLockTable: Serialize service request, used by Custom Labels workflow
# * IngestTable: Stores ingested metadata
# * ServiceTokenTable: Part of the Service Integration, temporarily stores
# Step Functions execution token to resume
# * AnalysisTable: Stores AI/ML metadata (pointers to S3 locations)
#
################################################################################
BacklogTable:
Type: AWS::DynamoDB::Table
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W28
reason: Tables are explicitly assigned with predefined names
Properties:
TableName: !Sub
- ${ResourcePrefix}-${suffix}
- suffix: !FindInMap
- DynamoDB
- Backlog
- Suffix
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- PartitionKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- SortKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIStatusPartitionKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIStatusSortKey
AttributeType: N
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIJobIdPartitionKey
AttributeType: S
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- PartitionKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- SortKey
KeyType: RANGE
SSESpecification:
SSEEnabled: true
PointInTimeRecoverySpecification:
PointInTimeRecoveryEnabled: true
TimeToLiveSpecification:
AttributeName: ttl
Enabled: true
GlobalSecondaryIndexes:
-
IndexName: !FindInMap
- DynamoDB
- Backlog
- GSIStatusIndex
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIStatusPartitionKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIStatusSortKey
KeyType: RANGE
Projection:
ProjectionType: INCLUDE
NonKeyAttributes:
- serviceParams
-
IndexName: !FindInMap
- DynamoDB
- Backlog
- GSIJobIdIndex
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Backlog
- GSIJobIdPartitionKey
KeyType: HASH
Projection:
ProjectionType: INCLUDE
NonKeyAttributes:
- serviceParams
StreamSpecification:
StreamViewType: NEW_AND_OLD_IMAGES
AtomicLockTable:
Type: AWS::DynamoDB::Table
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W28
reason: Tables are explicitly assigned with predefined names
Properties:
TableName: !Sub
- ${ResourcePrefix}-${suffix}
- suffix: !FindInMap
- DynamoDB
- AtomicLock
- Suffix
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
-
AttributeName: !FindInMap
- DynamoDB
- AtomicLock
- PartitionKey
AttributeType: S
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- AtomicLock
- PartitionKey
KeyType: HASH
SSESpecification:
SSEEnabled: true
PointInTimeRecoverySpecification:
PointInTimeRecoveryEnabled: true
TimeToLiveSpecification:
AttributeName: ttl
Enabled: true
ServiceTokenTable:
Type: AWS::DynamoDB::Table
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W28
reason: Tables are explicitly assigned with predefined names
Properties:
TableName: !Sub
- ${ResourcePrefix}-${suffix}
- suffix: !FindInMap
- DynamoDB
- ServiceToken
- Suffix
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
-
AttributeName: !FindInMap
- DynamoDB
- ServiceToken
- PartitionKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- ServiceToken
- SortKey
AttributeType: S
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- ServiceToken
- PartitionKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- ServiceToken
- SortKey
KeyType: RANGE
SSESpecification:
SSEEnabled: true
PointInTimeRecoverySpecification:
PointInTimeRecoveryEnabled: true
TimeToLiveSpecification:
AttributeName: ttl
Enabled: true
IngestTable:
Type: AWS::DynamoDB::Table
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W28
reason: Tables are explicitly assigned with predefined names
Properties:
TableName: !Sub
- ${ResourcePrefix}-${suffix}
- suffix: !FindInMap
- DynamoDB
- Ingest
- Suffix
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- PartitionKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- SchemaVersionIndexKey
AttributeType: N
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GSISortKey
AttributeType: N
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- TypeIndexKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GroupIndexKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- StatusIndexKey
AttributeType: S
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- PartitionKey
KeyType: HASH
SSESpecification:
SSEEnabled: true
PointInTimeRecoverySpecification:
PointInTimeRecoveryEnabled: true
GlobalSecondaryIndexes:
-
IndexName: !FindInMap
- DynamoDB
- Ingest
- SchemaVersionIndexName
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- SchemaVersionIndexKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GSISortKey
KeyType: RANGE
Projection:
ProjectionType: INCLUDE
NonKeyAttributes:
- !FindInMap
- DynamoDB
- Ingest
- TypeIndexKey
-
IndexName: !FindInMap
- DynamoDB
- Ingest
- TypeIndexName
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- TypeIndexKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GSISortKey
KeyType: RANGE
Projection:
ProjectionType: KEYS_ONLY
-
IndexName: !FindInMap
- DynamoDB
- Ingest
- GroupIndexName
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GroupIndexKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GSISortKey
KeyType: RANGE
Projection:
ProjectionType: KEYS_ONLY
-
IndexName: !FindInMap
- DynamoDB
- Ingest
- StatusIndexName
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- StatusIndexKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Ingest
- GSISortKey
KeyType: RANGE
Projection:
ProjectionType: KEYS_ONLY
StreamSpecification:
StreamViewType: NEW_AND_OLD_IMAGES
AnalysisTable:
Type: AWS::DynamoDB::Table
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W28
reason: Tables are explicitly assigned with predefined names
Properties:
TableName: !Sub
- ${ResourcePrefix}-${suffix}
- suffix: !FindInMap
- DynamoDB
- Analysis
- Suffix
BillingMode: PAY_PER_REQUEST
AttributeDefinitions:
-
AttributeName: !FindInMap
- DynamoDB
- Analysis
- PartitionKey
AttributeType: S
-
AttributeName: !FindInMap
- DynamoDB
- Analysis
- SortKey
AttributeType: S
KeySchema:
-
AttributeName: !FindInMap
- DynamoDB
- Analysis
- PartitionKey
KeyType: HASH
-
AttributeName: !FindInMap
- DynamoDB
- Analysis
- SortKey
KeyType: RANGE
SSESpecification:
SSEEnabled: true
PointInTimeRecoverySpecification:
PointInTimeRecoveryEnabled: true
################################################################################
#
# Service Data Access Role
# * Read/Write to Bucket (Amazon Transcribe, Amazon Comprehend, AWS Elemental
# MediaConvert)
#
################################################################################
ServiceDataAccessRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service:
- comprehend.amazonaws.com
- transcribe.amazonaws.com
- mediaconvert.amazonaws.com
# Note: MediaConvert only takes '/' or '/service-role/' paths.
# '/some-random-string/' will throw an error 'Invalid role provided in RESOURCE_ROLE_ARN' with code 1432
Path: /service-role/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-data-access
PolicyDocument:
Version: "2012-10-17"
Statement:
# S3 - read from ingest bucket
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 list/read/write to proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
################################################################################
#
# Amazon EventBridge
# * an EventBridge bus that interfaces with upstream/downstream processes
#
################################################################################
EventBridgeLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/events/${ResourcePrefix}-${name}
- name: !FindInMap
- EventBridge
- Bus
- Name
RetentionInDays: 7
EventBridgeBus:
Type: AWS::Events::EventBus
Properties:
Name: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- EventBridge
- Bus
- Name
EventBridgeRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: events.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-eventbridge
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt EventBridgeLogGroup.Arn
EventBridgeRule:
Type: AWS::Events::Rule
Properties:
Description: !FindInMap
- EventBridge
- Rule
- Detail
EventBusName: !GetAtt EventBridgeBus.Name
EventPattern:
detail-type:
- !FindInMap
- EventBridge
- Rule
- Detail
source:
- !FindInMap
- EventBridge
- Rule
- Source
account:
- !Ref AWS::AccountId
region:
- !Ref AWS::Region
Name: !FindInMap
- EventBridge
- Rule
- Name
RoleArn: !GetAtt EventBridgeRole.Arn
State: ENABLED
Targets:
-
Arn: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${EventBridgeLogGroup}
Id: !Sub ${ResourcePrefix}-loggroup-target
################################################################################
#
# Amazon Simple Notification Service (SNS)
#
################################################################################
SNSTopic:
Type: AWS::SNS::Topic
Properties:
TopicName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- SNS
- Topic
- Name
DisplayName: !FindInMap
- SNS
- Topic
- DisplayName
KmsMasterKeyId: alias/aws/sns
SNSTopicPolicy:
Type: AWS::SNS::TopicPolicy
Properties:
Topics:
- !Ref SNSTopic
PolicyDocument:
Id: !Sub ${ResourcePrefix}-status-policy
Version: "2012-10-17"
Statement:
-
Sid: AllowLamdaToPublish
Effect: Allow
Principal:
Service: lambda.amazonaws.com
Action: sns:Publish
Resource: !Ref SNSTopic
################################################################################
#
# Backlog Service SNS Topic and Data Access
# * SNS Topic (Amazon Rekognition, Amazon Textract)
#
################################################################################
BacklogTopic:
Type: AWS::SNS::Topic
Properties:
TopicName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- SNS
- BacklogTopic
- Name
DisplayName: !FindInMap
- SNS
- BacklogTopic
- DisplayName
KmsMasterKeyId: alias/aws/sns
BacklogTopicPolicy:
Type: AWS::SNS::TopicPolicy
Properties:
Topics:
- !Ref BacklogTopic
PolicyDocument:
Id: !Sub ${ResourcePrefix}-backlog-topic-policy
Version: "2012-10-17"
Statement:
-
Sid: allow-services-publish-sns
Effect: Allow
Principal:
Service:
- rekognition.amazonaws.com
- textract.amazonaws.com
Action: sns:Publish
Resource: !Ref BacklogTopic
BacklogTopicRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service:
- rekognition.amazonaws.com
- textract.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-service-backlog-topic-role
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sns:Publish
Resource: !Ref BacklogTopic
################################################################################
#
# Backlog Custom Labels State Machine
#
################################################################################
BacklogCustomLabelsLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogCustomLabels
- Name
RetentionInDays: 7
BacklogCustomLabelsRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-backlog-customlabels
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt BacklogCustomLabelsLogGroup.Arn
# S3
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
- s3:PutObjectAcl
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !GetAtt AtomicLockTable.Arn
# Rekognition Custom Labels
-
Effect: Allow
Action: rekognition:DescribeProjectVersions
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:project/*/*
-
Effect: Allow
Action:
- rekognition:DetectCustomLabels
- rekognition:StartProjectVersion
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:project/*/version/*/*
BacklogCustomLabelsLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogCustomLabels
- Name
Description: !Sub (${SolutionLowerCaseId}) Backlog Custom Labels state machine lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt BacklogCustomLabelsRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- BacklogCustomLabels
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
# Service Backlog
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_ATOMICLOCK_TABLE: !Ref AtomicLockTable
BacklogCustomLabelsStateMachineServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: !Sub states.${AWS::Region}.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-customlabels-service-role
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: lambda:InvokeFunction
Resource: !GetAtt BacklogCustomLabelsLambda.Arn
BacklogCustomLabelsStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogCustomLabels
- Name
RoleArn: !GetAtt BacklogCustomLabelsStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Check project version status",
"States": {
"Check project version status": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "check-project-version-status",
"jobTag.$": "$.jobTag",
"input.$": "$.input",
"output.$": "$.output"
},
"Next": "Project version started?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Project version started?": {
"Type": "Choice",
"Choices": [
{
"Or": [
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "TRAINING_FAILED"
},
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "FAILED"
},
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "DELETING"
}
],
"Next": "Project version failed"
},
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "RUNNING",
"Next": "Detect custom labels"
},
{
"Or": [
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "TRAINING_COMPLETED"
},
{
"Variable": "$.output['check-project-version-status'].status",
"StringEquals": "STOPPED"
}
],
"Next": "Start project version"
}
],
"Default": "Wait for project version status (3mins)"
},
"Wait for project version status (3mins)": {
"Type": "Wait",
"Seconds": 180,
"Next": "Check project version status"
},
"Start project version": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "start-project-version",
"jobTag.$": "$.jobTag",
"input.$": "$.input",
"output.$": "$.output"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
],
"Next": "Wait for project version status (3mins)"
},
"Detect custom labels": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "detect-custom-labels",
"jobTag.$": "$.jobTag",
"input.$": "$.input",
"output.$": "$.output"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
],
"Next": "More custom labels?"
},
"More custom labels?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.output['detect-custom-labels'].status",
"StringEquals": "completed",
"Next": "Custom labels completed"
}
],
"Default": "Detect custom labels"
},
"Custom labels completed": {
"Type": "Succeed"
},
"Project version failed": {
"Type": "Fail"
}
}
}
-
{
x0: !GetAtt BacklogCustomLabelsLambda.Arn
}
################################################################################
#
# Backlog Stream connector
# * Data Access Role (Amazon Transcribe, Amazon Comprehend
# * Connector lambda role
# * Connector lambda
# * Configure Lambda Event Source Mapping to trigger on DDB Stream
#
################################################################################
BacklogStreamConnectorLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogStreamConnector
- Name
RetentionInDays: 7
BacklogStreamConnectorRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed with the solution and stack id
-
id: W76
reason: Suppress SPCM higher than 25 warning
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Principal:
Service: lambda.amazonaws.com
Action: sts:AssumeRole
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-service-backlog-stream
PolicyDocument:
Version: "2012-10-17"
Statement:
# Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt BacklogStreamConnectorLogGroup.Arn
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Query
- dynamodb:UpdateItem
Resource:
- !GetAtt BacklogTable.Arn
- !Sub
- ${BacklogTable.Arn}/index/${index}
- index: !FindInMap
- DynamoDB
- Backlog
- GSIStatusIndex
# DynamoDB stream permission
-
Effect: Allow
Action:
- dynamodb:DescribeStream
- dynamodb:GetRecords
- dynamodb:GetShardIterator
- dynamodb:ListStreams
Resource: !GetAtt BacklogTable.StreamArn
# EventBridge
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
# Rekognition
-
Effect: Allow
Action:
- rekognition:StartContentModeration
- rekognition:StartCelebrityRecognition
- rekognition:StartFaceDetection
- rekognition:StartFaceSearch
- rekognition:StartLabelDetection
- rekognition:StartPersonTracking
- rekognition:StartSegmentDetection
- rekognition:StartTextDetection
Resource: "*"
-
Effect: Allow
Action: rekognition:StopProjectVersion
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:project/*/version/*/*
# Textract
-
Effect: Allow
Action:
- textract:StartDocumentAnalysis
- textract:StartDocumentTextDetection
Resource: "*"
# Comprehend
-
Effect: Allow
Action:
- comprehend:StartEntitiesDetectionJob
- comprehend:StartKeyPhrasesDetectionJob
- comprehend:StartDominantLanguageDetectionJob
- comprehend:StartSentimentDetectionJob
- comprehend:StartTopicsDetectionJob
Resource: "*"
# Transcribe
-
Effect: Allow
Action: transcribe:StartTranscriptionJob
Resource: "*"
# MediaConvert
-
Effect: Allow
Action: mediaConvert:CreateJob
Resource: !Sub arn:aws:mediaconvert:${AWS::Region}:${AWS::AccountId}:*
# S3 - StreamConnectorLambda starts AI/ML job on behalf
# which requires GetObject when the services assume role
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# Step Functions (Custom Labels)
-
Effect: Allow
Action: states:StartExecution
Resource: !Ref BacklogCustomLabelsStateMachine
# DynamoDB - AtomicLock table to acquire lock
-
Effect: Allow
Action:
- dynamodb:Query
- dynamodb:DeleteItem
- dynamodb:PutItem
- dynamodb:UpdateItem
Resource: !GetAtt AtomicLockTable.Arn
# IAM - Rekognition/Textract pass SNS topic role for notification
-
Effect: Allow
Action: iam:PassRole
Resource: !GetAtt BacklogTopicRole.Arn
# IAM - Comprehend/Transcribe pass data access role
-
Effect: Allow
Action:
- iam:GetRole
- iam:PassRole
Resource: !GetAtt ServiceDataAccessRole.Arn
BacklogStreamConnectorLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogStreamConnector
- Name
Description: !Sub (${SolutionLowerCaseId}) backlog stream connector (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt BacklogStreamConnectorRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- BacklogStreamConnector
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
ENV_DATA_ACCESS_ROLE: !GetAtt ServiceDataAccessRole.Arn
ENV_ATOMICLOCK_TABLE: !Ref AtomicLockTable
ENV_MEDIACONVERT_HOST: !Ref MediaConvertEndpoint
BacklogStreamEventSource:
Type: AWS::Lambda::EventSourceMapping
Properties:
Enabled: true
EventSourceArn: !GetAtt BacklogTable.StreamArn
FunctionName: !Ref BacklogStreamConnectorLambda
BatchSize: 1
MaximumRetryAttempts: 10
StartingPosition: LATEST
################################################################################
#
# Backlog Status Updater
# * Updater lambda role
# * Updater lambda
# * Amazon SNS (Amazon Rekognition, Amazon Textract)
# * Amazon EventBridge / CloudWatch Event
# * AWS Elemental MediaConvert
# * Amazon Transcribe
# * Amazon Step Functions of Custom Labels
#
################################################################################
BacklogStatusUpdaterLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogStatusUpdater
- Name
RetentionInDays: 7
BacklogStatusUpdaterRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-service-backlog-status-updater
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt BacklogStatusUpdaterLogGroup.Arn
# DynamoDB - query and delete item from backlog table / jobId index
-
Effect: Allow
Action:
- dynamodb:DeleteItem
- dynamodb:Query
Resource:
- !GetAtt BacklogTable.Arn
- !Sub
- ${BacklogTable.Arn}/index/${index}
- index: !FindInMap
- DynamoDB
- Backlog
- GSIJobIdIndex
# EventBridge
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
# DynamoDB - Atomic Lock Table to delete lock
-
Effect: Allow
Action:
- dynamodb:DeleteItem
- dynamodb:Query
Resource: !GetAtt AtomicLockTable.Arn
# Step Functions - describe custom labels execution
-
Effect: Allow
Action: states:DescribeExecution
Resource: !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${BacklogCustomLabelsStateMachine.Name}:*
BacklogStatusUpdaterLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogStatusUpdater
- Name
Description: !Sub (${SolutionLowerCaseId}) service backlog status updater lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt BacklogStatusUpdaterRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- BacklogStatusUpdater
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
ENV_ATOMICLOCK_TABLE: !Ref AtomicLockTable
################################################################################
# Backlog Rekognition / Textract SNS Notification
BacklogTopicSubscription:
Type: AWS::SNS::Subscription
Properties:
Protocol: lambda
TopicArn: !Ref BacklogTopic
Endpoint: !GetAtt BacklogStatusUpdaterLambda.Arn
BacklogTopicPermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref BacklogStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: sns.amazonaws.com
SourceArn: !Ref BacklogTopic
################################################################################
# Backlog MediaConvert Job Status Change Event
BacklogMediaConvertStatusChangeEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-MediaConvertJobStatusChangeEventBacklog
Description: !Sub "(${SolutionLowerCaseId}) MediaConvert Job Status Change Event (Backlog)"
EventPattern:
source:
- aws.mediaconvert
region:
- !Ref AWS::Region
detail-type:
- MediaConvert Job State Change
detail:
status:
- COMPLETE
- CANCELED
- ERROR
userMetadata:
solutionUuid:
- !Ref SolutionUuid
State: ENABLED
Targets:
-
Id: !Sub Id-${BacklogStatusUpdaterLambda}
Arn: !GetAtt BacklogStatusUpdaterLambda.Arn
BacklogMediaConvertStatusChangePermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref BacklogStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt BacklogMediaConvertStatusChangeEvent.Arn
################################################################################
# Backlog Transcribe Job Status Change Event
BacklogTranscribeStatusChangeEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-TranscribeJobStateChangeBacklog
Description: !Sub (${SolutionLowerCaseId}) Transcribe Job State Change Event (Backlog)
EventPattern:
source:
- aws.transcribe
region:
- !Ref AWS::Region
detail-type:
- Transcribe Job State Change
detail:
TranscriptionJobStatus:
- COMPLETED
- FAILED
TranscriptionJobName:
- prefix: !Sub ${SolutionUuid}
State: ENABLED
Targets:
-
Id: !Sub Id-${BacklogStatusUpdaterLambda}
Arn: !GetAtt BacklogStatusUpdaterLambda.Arn
BacklogTranscribeStatusChangePermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref BacklogStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt BacklogTranscribeStatusChangeEvent.Arn
################################################################################
# Backlog Custom Labels State Machine Event
BacklogCustomLabelsStateMachineStatusEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BacklogCustomLabels
- Name
Description: !Sub (${SolutionLowerCaseId}) backlog custom labels state machine status change
EventPattern:
source:
- aws.states
detail-type:
- Step Functions Execution Status Change
detail:
status:
- FAILED
- ABORTED
- TIMED_OUT
- SUCCEEDED
stateMachineArn:
- !Ref BacklogCustomLabelsStateMachine
State: ENABLED
Targets:
-
Id: !Sub Id-${BacklogStatusUpdaterLambda}
Arn: !GetAtt BacklogStatusUpdaterLambda.Arn
BacklogCustomLabelsStateMachineStatusPermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref BacklogStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt BacklogCustomLabelsStateMachineStatusEvent.Arn
################################################################################
#
# Ingest Workflow
# * State Machine Service Role
# * Fixity (Nested) State Machine
# * Audio (Nested) State Machine
# * Video (Nested) State Machine
# * Image (Nested) State Machine
# * Document (Nested) State Machine
# * Ingest (Main) State Machine
#
################################################################################
IngestStateMachineServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: !Sub states.${AWS::Region}.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-statemachine-role
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: lambda:InvokeFunction
Resource:
- !GetAtt IngestFixityLambda.Arn
- !GetAtt IngestVideoLambda.Arn
- !GetAtt IngestAudioLambda.Arn
- !GetAtt IngestImageLambda.Arn
- !GetAtt IngestDocumentLambda.Arn
- !GetAtt IngestMainLambda.Arn
# Below polices are needed for Service Integration of nested workflows
# https://docs.aws.amazon.com/step-functions/latest/dg/stepfunctions-iam.html
-
Effect: Allow
Action: states:StartExecution
Resource:
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestFixity
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestVideo
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestAudio
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestImage
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestDocument
- Name
-
Effect: Allow
Action:
- states:DescribeExecution
- states:StopExecution
Resource:
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- IngestFixity
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- IngestVideo
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- IngestAudio
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- IngestImage
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- IngestDocument
- Name
-
Effect: Allow
Action:
- events:PutTargets
- events:PutRule
- events:DescribeRule
Resource: !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule
################################################################################
# Ingest Fixty State Machine resources
IngestFixityLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestFixity
- Name
RetentionInDays: 7
IngestFixityRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-fixity
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestFixityLogGroup.Arn
# S3
-
Effect: Allow
Action:
- s3:GetObject
- s3:GetObjectTagging
- s3:GetObjectVersionTagging
- s3:PutObject
- s3:PutObjectTagging
- s3:PutObjectVersionTagging
- s3:RestoreObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
IngestFixityLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestFixity
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest fixity state machine lambda (2048MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 2048
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestFixityRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestFixity
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref FixityLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
IngestFixityStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestFixity
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "Ingest Fixity state machine auto restores s3 object from GLAICER or DEEP_ARCHIVE and to run checksum",
"StartAt": "Check restore status",
"States": {
"Check restore status": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "check-restore-status",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Restore completed?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
]
},
"Restore completed?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Compute checksum"
},
{
"Variable": "$.data.restore.tier",
"StringEquals": "Expedited",
"Next": "Wait 4 mins"
},
{
"And": [
{
"Variable": "$.data.restore.storageClass",
"StringEquals": "DEEP_ARCHIVE"
},
{
"Variable": "$.data.restore.tier",
"StringEquals": "Bulk"
}
],
"Next": "Wait 12 hrs"
}
],
"Default": "Wait 4 hrs"
},
"Wait 4 mins": {
"Type": "Wait",
"Seconds": 240,
"Next": "Check restore status"
},
"Wait 4 hrs": {
"Type": "Wait",
"Seconds": 14400,
"Next": "Check restore status"
},
"Wait 12 hrs": {
"Type": "Wait",
"Seconds": 43200,
"Next": "Check restore status"
},
"Compute checksum": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "compute-checksum",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "More data?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
]
},
"More data?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Validate checksum"
}
],
"Default": "Compute checksum"
},
"Validate checksum": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "validate-checksum",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
]
}
}
}
-
{
x0: !GetAtt IngestFixityLambda.Arn
}
################################################################################
# Ingest Video State Machine resources
IngestVideoLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestVideo
- Name
RetentionInDays: 7
IngestVideoRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-video
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestVideoLogGroup.Arn
# S3 - ingest bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${IngestBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# MediaConvert
-
Effect: Allow
Action:
- mediaConvert:CreateJob
- mediaConvert:GetJob
Resource: !Sub arn:aws:mediaconvert:${AWS::Region}:${AWS::AccountId}:*
# IAM - pass data access role to MediaConvert
-
Effect: Allow
Action:
- iam:GetRole
- iam:PassRole
Resource: !GetAtt ServiceDataAccessRole.Arn
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# EventBridge (ServiceBacklog)
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
IngestVideoLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestVideo
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest video state machine lambda (2048MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 2048
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestVideoRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestVideo
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref MediaInfoLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_MEDIACONVERT_HOST: !Ref MediaConvertEndpoint
ENV_DATA_ACCESS_ROLE: !GetAtt ServiceDataAccessRole.Arn
## Service Backlog ##
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
IngestVideoStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestVideo
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "video ingest state machine to run mediainfo and start mediaconvert job to create proxy",
"StartAt": "Run mediainfo",
"States": {
"Run mediainfo": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "run-mediainfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Start and wait for mediaconvert job",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Start and wait for mediaconvert job": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"operation": "start-transcode",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
}
},
"TimeoutSeconds": 86400,
"End": true
}
}
}
-
{
x0: !GetAtt IngestVideoLambda.Arn,
a0: !Ref IngestVideoLambda
}
################################################################################
# Ingest Audio State Machine resources
IngestAudioLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestAudio
- Name
RetentionInDays: 7
IngestAudioRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-audio
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestAudioLogGroup.Arn
# S3 - ingest bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${IngestBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# MediaConvert
-
Effect: Allow
Action:
- mediaConvert:CreateJob
- mediaConvert:GetJob
Resource: !Sub arn:aws:mediaconvert:${AWS::Region}:${AWS::AccountId}:*
# IAM - pass data access role to MediaConvert
-
Effect: Allow
Action:
- iam:GetRole
- iam:PassRole
Resource: !GetAtt ServiceDataAccessRole.Arn
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# EventBridge (ServiceBacklog)
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
IngestAudioLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestAudio
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest audio state machine lambda (256MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 256
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestAudioRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestAudio
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref MediaInfoLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_MEDIACONVERT_HOST: !Ref MediaConvertEndpoint
ENV_DATA_ACCESS_ROLE: !GetAtt ServiceDataAccessRole.Arn
## Service Backlog ##
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
IngestAudioStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestAudio
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "audio ingest state machine to run mediainfo and start elastictranscode job to create proxy",
"StartAt": "Run mediainfo",
"States": {
"Run mediainfo": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "run-mediainfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Start and wait for transcode job",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Start and wait for transcode job": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"operation": "start-transcode",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
}
},
"TimeoutSeconds": 86400,
"End": true
}
}
}
-
{
x0: !GetAtt IngestAudioLambda.Arn,
a0: !Ref IngestAudioLambda
}
################################################################################
# Ingest Image State Machine resources
IngestImageLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestImage
- Name
RetentionInDays: 7
IngestImageRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-image
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestImageLogGroup.Arn
# S3 - ingest bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${IngestBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
IngestImageLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestImage
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest image state machine lambda (1024MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 3008
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestImageRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestImage
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref ImageProcessLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
IngestImageStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestImage
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "image ingest state machine to run exiftool and extract thumbnail image",
"StartAt": "Run imageinfo",
"States": {
"Run imageinfo": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "run-imageinfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
}
}
}
-
{
x0: !GetAtt IngestImageLambda.Arn
}
################################################################################
# Ingest Document State Machine resources
IngestDocumentLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestDocument
- Name
RetentionInDays: 7
IngestDocumentRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-document
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestDocumentLogGroup.Arn
# S3 - ingest bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${IngestBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
IngestDocumentLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestDocument
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest document state machine lambda (1024MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 1024
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestDocumentRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestDocument
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref PDFLibLayer
- !Ref CanvasLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
IngestDocumentStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestDocument
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "document ingest state machine lambda to run pdf info to extract metadata and convert pages to PNG images",
"StartAt": "Run PDFInfo and extract pages",
"States": {
"Run PDFInfo and extract pages": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "run-docinfo",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "More pages?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"More pages?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "PDFInfo completed"
}
],
"Default": "Run PDFInfo and extract pages"
},
"PDFInfo completed": {
"Type": "Succeed"
}
}
}
-
{
x0: !GetAtt IngestDocumentLambda.Arn
}
################################################################################
# Ingest Main State Machine resources
IngestMainLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestMain
- Name
RetentionInDays: 7
IngestMainRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-main
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestMainLogGroup.Arn
# S3 - ingest bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${IngestBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:GetObjectTagging
- s3:GetObjectVersionTagging
- s3:PutObjectTagging
- s3:PutObjectVersionTagging
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# IoT
-
Effect: Allow
Action: iot:Publish
Resource: !Sub arn:aws:iot:${AWS::Region}:${AWS::AccountId}:topic/${IotTopic}
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
# SNS
-
Effect: Allow
Action: sns:Publish
Resource: !Ref SNSTopic
IngestMainLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestMain
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest main state machine lambda (512MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 512
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestMainRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestMain
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref MediaInfoLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_SNS_TOPIC_ARN: !Ref SNSTopic
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
IngestMainStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestMain
- Name
RoleArn: !GetAtt IngestStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "ingest state machine to create record, to extract media technical metadata, and to index results to elasticsearch",
"StartAt": "Create record",
"States": {
"Create record": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "create-record",
"executionArn.$": "$$.Execution.Id",
"input.$": "$.input"
},
"Next": "Start fixity (nested)",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Start fixity (nested)": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"operation": "check-restore-status",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${IngestFixityStateMachine}"
},
"Next": "Fixity completed"
},
"Fixity completed": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "fixity-completed",
"nestedStateOutput.$": "$"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
],
"Next": "Choose by media type"
},
"Choose by media type": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.input.type",
"StringEquals": "image",
"Next": "Start image ingest (nested)"
},
{
"Variable": "$.input.type",
"StringEquals": "video",
"Next": "Start video ingest (nested)"
},
{
"Variable": "$.input.type",
"StringEquals": "audio",
"Next": "Start audio ingest (nested)"
},
{
"Variable": "$.input.type",
"StringEquals": "document",
"Next": "Start document ingest (nested)"
}
],
"Default": "Media type not supported"
},
"Start image ingest (nested)": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"operation": "run-imageinfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${IngestImageStateMachine}"
},
"Next": "Update record"
},
"Start video ingest (nested)": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"operation": "run-mediainfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${IngestVideoStateMachine}"
},
"Next": "Update record"
},
"Start audio ingest (nested)": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"operation": "run-mediainfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${IngestAudioStateMachine}"
},
"Next": "Update record"
},
"Start document ingest (nested)": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"operation": "run-docinfo",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${IngestDocumentStateMachine}"
},
"Next": "Update record"
},
"Media type not supported": {
"Type": "Fail"
},
"Update record": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "update-record",
"nestedStateOutput.$": "$"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
],
"Next": "Index ingest results"
},
"Index ingest results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-ingest-results",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
],
"Next": "Completed"
},
"Completed": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "job-completed",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true
}
}
}
-
{
x0: !GetAtt IngestMainLambda.Arn
}
################################################################################
#
# Ingest Automation
# * Ingest status updater
# * Ingest Table DDB Stream Event
# * Backlog Status Change Event (Ingest) supports AWS Elemental MediaConvert
#
################################################################################
IngestStatusUpdaterLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestStatusUpdater
- Name
RetentionInDays: 7
IngestStatusUpdaterRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-BacklogEventBridgeEvent
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestStatusUpdaterLogGroup.Arn
# Step Functions - send task results on MediaConvert Job Status Change Event
-
Effect: Allow
Action:
- states:SendTaskSuccess
- states:SendTaskFailure
Resource:
- !Ref IngestVideoStateMachine
- !Ref IngestAudioStateMachine
# DynamoDB - get and delete token from ServiceToken
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# IoT (TO BE REMOVED)
-
Effect: Allow
Action: iot:Publish
Resource: !Sub arn:aws:iot:${AWS::Region}:${AWS::AccountId}:topic/${IotTopic}
# SNS
-
Effect: Allow
Action: sns:Publish
Resource: !Ref SNSTopic
-
PolicyName: !Sub ${ResourcePrefix}-DDBStreamEvent
PolicyDocument:
Version: "2012-10-17"
Statement:
# DynamoDB stream permission
-
Effect: Allow
Action:
- dynamodb:DescribeStream
- dynamodb:GetRecords
- dynamodb:GetShardIterator
- dynamodb:ListStreams
Resource: !GetAtt IngestTable.StreamArn
# S3 - allow to list objects on REMOVE event
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
# S3 - allow to delete objects on REMOVE event
-
Effect: Allow
Action:
- s3:GetObject
- s3:DeleteObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# OpenSearch - allow delete document on REMOVE event
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
IngestStatusUpdaterLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestStatusUpdater
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest status updater lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestStatusUpdaterRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestStatusUpdater
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_SNS_TOPIC_ARN: !Ref SNSTopic
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
################################################################################
# DynamoDB Ingest Table DDB Stream Event
IngestTableStreamEventSource:
Type: AWS::Lambda::EventSourceMapping
Properties:
Enabled: true
EventSourceArn: !GetAtt IngestTable.StreamArn
FunctionName: !Ref IngestStatusUpdaterLambda
BatchSize: 1
MaximumRetryAttempts: 10
StartingPosition: LATEST
################################################################################
# Backlog Ingest Status Change Event
# * events fired by our Service Backlog EventBus
IngestBacklogStatusChangeEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-IngestBacklogStatusChangeEvent
Description: !Sub "(${SolutionId}) Backlog Ingest Status Change Event"
EventBusName: !GetAtt EventBridgeBus.Name
EventPattern:
source:
- !FindInMap
- EventBridge
- Rule
- Source
region:
- !Ref AWS::Region
detail-type:
- !FindInMap
- EventBridge
- Rule
- Detail
detail:
status:
# MediaConvert
- COMPLETE
- CANCELED
- ERROR
serviceApi:
- mediaconvert:createjob
State: ENABLED
Targets:
-
Id: !Sub Id-${IngestStatusUpdaterLambda}
Arn: !GetAtt IngestStatusUpdaterLambda.Arn
IngestBacklogStatusChangePermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref IngestStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt IngestBacklogStatusChangeEvent.Arn
################################################################################
#
# Analysis Workflow
# * State Machine Service Role
# * Audio (Nested) State Machine
# * Video (Nested) State Machine
# * Image (Nested) State Machine
# * Document (Nested) State Machine
# * Analysis (Main) State Machine
#
################################################################################
AnalysisStateMachineServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: !Sub states.${AWS::Region}.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-statemachine-role
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: lambda:InvokeFunction
Resource:
- !GetAtt AnalysisVideoLambda.Arn
- !GetAtt AnalysisAudioLambda.Arn
- !GetAtt AnalysisImageLambda.Arn
- !GetAtt AnalysisDocumentLambda.Arn
- !GetAtt AnalysisMainLambda.Arn
# Below polices are needed for Service Integration of nested workflows
# https://docs.aws.amazon.com/step-functions/latest/dg/stepfunctions-iam.html
-
Effect: Allow
Action: states:StartExecution
Resource:
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisVideo
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisAudio
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisImage
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisDocument
- Name
-
Effect: Allow
Action:
- states:DescribeExecution
- states:StopExecution
Resource:
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- AnalysisVideo
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- AnalysisAudio
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- AnalysisImage
- Name
- !Sub
- arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${ResourcePrefix}-${name}:*
- name: !FindInMap
- Workflow
- AnalysisDocument
- Name
-
Effect: Allow
Action:
- events:PutTargets
- events:PutRule
- events:DescribeRule
Resource: !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule
################################################################################
# Analysis Video State Machine resources
AnalysisVideoLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisVideo
- Name
RetentionInDays: 7
AnalysisVideoRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-video
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisVideoLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# Rekognition
-
Effect: Allow
Action:
- rekognition:DescribeCollection
- rekognition:StartContentModeration
- rekognition:StartCelebrityRecognition
- rekognition:StartFaceDetection
- rekognition:StartFaceSearch
- rekognition:StartLabelDetection
- rekognition:StartPersonTracking
- rekognition:StartSegmentDetection
- rekognition:StartTextDetection
- rekognition:GetContentModeration
- rekognition:GetCelebrityRecognition
- rekognition:GetFaceDetection
- rekognition:GetFaceSearch
- rekognition:GetLabelDetection
- rekognition:GetPersonTracking
- rekognition:GetSegmentDetection
- rekognition:GetTextDetection
- rekognition:DetectFaces
- rekognition:DetectLabels
- rekognition:DetectModerationLabels
- rekognition:DetectText
- rekognition:RecognizeCelebrities
- rekognition:SearchFacesByImage
Resource: "*"
# IAM - Rekognition/Textract pass SNS topic role for notification
-
Effect: Allow
Action: iam:PassRole
Resource: !GetAtt BacklogTopicRole.Arn
# EventBridge
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
# Backlog Custom Labels state machine
-
Effect: Allow
Action: states:StartExecution
Resource: !Ref BacklogCustomLabelsStateMachine
-
Effect: Allow
Action: states:DescribeExecution
Resource: !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${BacklogCustomLabelsStateMachine.Name}:*
# Rekognition Custom Labels
-
Effect: Allow
Action: rekognition:DescribeProjectVersions
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:project/*/*
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
AnalysisVideoLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisVideo
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis video state machine lambda (2048MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 2048
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisVideoRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisVideo
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref CanvasLibLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
## Service Backlog ##
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
ENV_ATOMICLOCK_TABLE: !Ref AtomicLockTable
AnalysisVideoStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisVideo
- Name
RoleArn: !GetAtt AnalysisStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Start video analysis",
"States": {
"Start video analysis": {
"Type": "Parallel",
"Branches": [
{
"StartAt": "Frame-based detection iterators",
"States": {
"Frame-based detection iterators": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "prepare-frame-detection-iterators",
"uuid.$": "$.uuid",
"input.$": "$.input",
"data.$": "$.data",
"status": "NOT_STARTED",
"progress": 0
},
"Next": "Frame based analysis",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Frame based analysis": {
"Type": "Map",
"ItemsPath": "$.data.iterators",
"MaxConcurrency": 10,
"Iterator": {
"StartAt": "Detect frame (Iterator)",
"States": {
"Detect frame (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "detect-frame-iterator",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"Next": "More frames (Iterator)?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"More frames (Iterator)?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Detect frame completed"
}
],
"Default": "Detect frame (Iterator)"
},
"Detect frame completed": {
"Type": "Succeed"
}
}
},
"ResultPath": "$.data.iterators",
"Next": "Frame-based track iterators"
},
"Frame-based track iterators": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "prepare-frame-track-iterators",
"uuid.$": "$.uuid",
"data.$": "$.data",
"status": "NOT_STARTED",
"progress": 0
},
"Next": "Frame based track iterators",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Frame based track iterators": {
"Type": "Map",
"ItemsPath": "$.data.iterators",
"MaxConcurrency": 10,
"Iterator": {
"StartAt": "Create frame-based track (Iterator)",
"States": {
"Create frame-based track (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "create-track-iterator",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"Next": "More frame-based tracks (Iterator)?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"More frame-based tracks (Iterator)?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Index frame-based analysis (Iterator)"
}
],
"Default": "Create frame-based track (Iterator)"
},
"Index frame-based analysis (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-analysis-results",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
}
}
},
"ResultPath": "$.data.iterators",
"End": true
}
}
},
{
"StartAt": "Video-based detection iterators",
"States": {
"Video-based detection iterators": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "prepare-video-detection-iterators",
"uuid.$": "$.uuid",
"input.$": "$.input",
"data.$": "$.data",
"status": "NOT_STARTED",
"progress": 0
},
"Next": "Video based analysis",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Video based analysis": {
"Type": "Map",
"ItemsPath": "$.data.iterators",
"MaxConcurrency": 10,
"Iterator": {
"StartAt": "Start detection and wait (Iterator)",
"States": {
"Start detection and wait (Iterator)": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"operation": "start-detection-iterator",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"data.$": "$.data"
}
},
"TimeoutSeconds": 86400,
"Next": "Collect detection results (Iterator)"
},
"Collect detection results (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"operation": "collect-results-iterator",
"data.$": "$.data"
},
"Next": "Create video-based track (Iterator)",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Create video-based track (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "create-track-iterator",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"Next": "More video-based tracks (Iterator)?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"More video-based tracks (Iterator)?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Index video-based analysis (Iterator)"
}
],
"Default": "Create video-based track (Iterator)"
},
"Index video-based analysis (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-analysis-results",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
}
}
},
"ResultPath": "$.data.iterators",
"End": true
}
}
},
{
"StartAt": "Custom detection iterators",
"States": {
"Custom detection iterators": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "prepare-custom-detection-iterators",
"uuid.$": "$.uuid",
"input.$": "$.input",
"data.$": "$.data",
"status": "NOT_STARTED",
"progress": 0
},
"Next": "Custom analysis",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Custom analysis": {
"Type": "Map",
"ItemsPath": "$.data.iterators",
"MaxConcurrency": 10,
"Iterator": {
"StartAt": "Start custom and wait (Iterator)",
"States": {
"Start custom and wait (Iterator)": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"operation": "start-detection-iterator",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"data.$": "$.data"
}
},
"TimeoutSeconds": 86400,
"Next": "Collect custom results (Iterator)"
},
"Collect custom results (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"operation": "collect-results-iterator",
"data.$": "$.data"
},
"Next": "Create custom track (Iterator)",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Create custom track (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "create-track-iterator",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"Next": "More custom tracks (Iterator)?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"More custom tracks (Iterator)?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Index custom analysis (Iterator)"
}
],
"Default": "Create custom track (Iterator)"
},
"Index custom analysis (Iterator)": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-analysis-results",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
}
}
},
"ResultPath": "$.data.iterators",
"End": true
}
}
}
],
"Next": "Video analysis completed"
},
"Video analysis completed": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "job-completed",
"stateExecution.$": "$$.Execution",
"parallelStateOutputs.$": "$"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
}
}
}
-
{
a0: !Ref AnalysisVideoLambda,
x0: !GetAtt AnalysisVideoLambda.Arn
}
################################################################################
# Analysis Audio State Machine resources
AnalysisAudioLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisAudio
- Name
RetentionInDays: 7
AnalysisAudioRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-audio
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisAudioLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:PutItem
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# Comprehend
-
Effect: Allow
Action:
- comprehend:BatchDetectDominantLanguage
- comprehend:BatchDetectEntities
- comprehend:BatchDetectKeyPhrases
- comprehend:BatchDetectSentiment
- comprehend:BatchDetectSyntax
- comprehend:StartEntitiesDetectionJob
- comprehend:StartKeyPhrasesDetectionJob
- comprehend:StartDominantLanguageDetectionJob
- comprehend:StartSentimentDetectionJob
- comprehend:StartTopicsDetectionJob
- comprehend:StopDominantLanguageDetectionJob
- comprehend:StopEntitiesDetectionJob
- comprehend:StopKeyPhrasesDetectionJob
- comprehend:StopSentimentDetectionJob
- comprehend:DescribeDominantLanguageDetectionJob
- comprehend:DescribeEntitiesDetectionJob
- comprehend:DescribeKeyPhrasesDetectionJob
- comprehend:DescribeSentimentDetectionJob
- comprehend:DescribeTopicsDetectionJob
Resource: "*"
-
Effect: Allow
Action: comprehend:DescribeEntityRecognizer
Resource: !Sub arn:aws:comprehend:${AWS::Region}:${AWS::AccountId}:entity-recognizer/*
# Transcribe
-
Effect: Allow
Action:
- transcribe:GetTranscriptionJob
- transcribe:StartTranscriptionJob
- transcribe:GetVocabulary
- transcribe:DescribeLanguageModel
Resource: "*"
# IAM - Comprehend/Transcribe pass data access role
-
Effect: Allow
Action: iam:PassRole
Resource: !GetAtt ServiceDataAccessRole.Arn
# EventBridge
-
Effect: Allow
Action: events:PutEvents
Resource: !GetAtt EventBridgeBus.Arn
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
AnalysisAudioLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisAudio
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis audio state machine lambda (512MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 512
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisAudioRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisAudio
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
- !Ref BacklogLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_DATA_ACCESS_ROLE: !GetAtt ServiceDataAccessRole.Arn
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
# Service Backlog ##
ENV_BACKLOG_EB_BUS: !GetAtt EventBridgeBus.Name
ENV_BACKLOG_TABLE: !Ref BacklogTable
ENV_BACKLOG_TOPIC_ARN: !Ref BacklogTopic
ENV_BACKLOG_TOPIC_ROLE_ARN: !GetAtt BacklogTopicRole.Arn
AnalysisAudioStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisAudio
- Name
RoleArn: !GetAtt AnalysisStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Start transcribe and wait",
"States": {
"Start transcribe and wait": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"operation": "start-transcribe",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input"
}
},
"TimeoutSeconds": 86400,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 1,
"BackoffRate": 1.2
}
],
"Next": "Collect transcribe results"
},
"Collect transcribe results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "collect-transcribe-results",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Got transcription data?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.1
}
]
},
"Got transcription data?": {
"Type": "Choice",
"Choices": [
{
"And": [
{
"Variable": "$.data.transcribe",
"IsPresent": true
},
{
"Variable": "$.data.transcribe.languageCode",
"IsPresent": true
},
{
"Variable": "$.data.transcribe.output",
"IsPresent": true
},
{
"Variable": "$.data.transcribe.vtt",
"IsPresent": true
}
],
"Next": "Index transcribe results"
}
],
"Default": "Analysis completed"
},
"Index transcribe results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-transcribe-results",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Start comprehend analysis",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.1
}
]
},
"Start comprehend analysis": {
"Type": "Parallel",
"Branches": [
{
"StartAt": "Batch detect entities",
"States": {
"Batch detect entities": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress.$": "$.progress",
"operation": "start-entity",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Index entity results",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 8,
"BackoffRate": 1.2
}
]
},
"Index entity results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "index-entity-results",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
}
}
},
{
"StartAt": "Batch detect keyphrases",
"States": {
"Batch detect keyphrases": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress.$": "$.progress",
"operation": "start-keyphrase",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Index keyphrase results",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 8,
"BackoffRate": 1.2
}
]
},
"Index keyphrase results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "index-keyphrase-results",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
}
}
},
{
"StartAt": "Batch detect sentiments",
"States": {
"Batch detect sentiments": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress.$": "$.progress",
"operation": "start-sentiment",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Index sentiment results",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 8,
"BackoffRate": 1.2
}
]
},
"Index sentiment results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "index-sentiment-results",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
}
}
},
{
"StartAt": "Check custom entity criteria",
"States": {
"Check custom entity criteria": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress.$": "$.progress",
"operation": "check-custom-entity-criteria",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Can start custom entity?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 1,
"BackoffRate": 1.2
}
]
},
"Can start custom entity?" : {
"Type": "Choice",
"Choices": [
{
"And": [
{
"Variable": "$.data.comprehend",
"IsPresent": true
},
{
"Variable": "$.data.comprehend.customentity",
"IsPresent": true
},
{
"Variable": "$.data.comprehend.customentity.prefix",
"IsPresent": true
}
],
"Next": "Start and wait custom entity"
}
],
"Default": "Custom entity skipped"
},
"Start and wait custom entity": {
"Type": "Task",
"Resource":"arn:aws:states:::lambda:invoke.waitForTaskToken",
"Parameters": {
"FunctionName": "${a0}",
"Payload": {
"token.$":"$$.Task.Token",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress.$": "$.progress",
"operation": "start-custom-entity",
"input.$": "$.input",
"data.$": "$.data"
}
},
"TimeoutSeconds": 86400,
"Next": "Wait for custom entity status (3mins)",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
},
"Wait for custom entity status (3mins)": {
"Type": "Wait",
"Seconds": 180,
"Next": "Check custom entity status"
},
"Check custom entity status": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "check-custom-entity-status",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Custom entity completed?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 1,
"BackoffRate": 1.1
}
]
},
"Custom entity completed?": {
"Type": "Choice",
"Choices": [
{
"Or": [
{
"Variable": "$.status",
"StringEquals": "NO_DATA"
},
{
"Variable": "$.status",
"StringEquals": "ERROR"
}
],
"Next": "Custom entity skipped"
},
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Create custom entity track"
}
],
"Default": "Wait for custom entity status (3mins)"
},
"Create custom entity track": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "create-custom-entity-track",
"input.$": "$.input",
"data.$": "$.data"
},
"Next": "Index custom entity results",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
},
"Index custom entity results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"operation": "index-custom-entity-results",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
},
"Custom entity skipped": {
"Type": "Succeed"
}
}
}
],
"Next": "Analysis completed"
},
"Analysis completed": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "job-completed",
"stateExecution.$": "$$.Execution",
"parallelStateOutputs.$": "$"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.1
}
]
}
}
}
-
{
a0: !Ref AnalysisAudioLambda,
x0: !GetAtt AnalysisAudioLambda.Arn
}
################################################################################
# Analysis Image State Machine resources
AnalysisImageLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisImage
- Name
RetentionInDays: 7
AnalysisImageRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-image
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisImageLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# Rekognition
-
Effect: Allow
Action:
- rekognition:DetectFaces
- rekognition:DetectLabels
- rekognition:DetectModerationLabels
- rekognition:DetectText
- rekognition:RecognizeCelebrities
- rekognition:DescribeCollection
- rekognition:SearchFacesByImage
Resource: "*"
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
AnalysisImageLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisImage
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis image state machine lambda (384MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 384
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisImageRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisImage
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
# BLIP model lambda
BlipModelLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BlipModel
- Name
RetentionInDays: 7
BlipModelRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-image
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt BlipModelLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
BlipModelLambda:
Condition: bBlipImageArn
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- BlipModel
- Name
Description: !Sub (${SolutionLowerCaseId}) BLIP model lambda (5308MB)
MemorySize: 5308
Timeout: 900
PackageType: Image
Role: !GetAtt BlipModelRole.Arn
Code:
ImageUri: !Ref BlipImageArn
Architectures:
- x86_64
TracingConfig:
Mode: Active
# Placeholder if BlipImageArn not defined
BlipModelPlaceholderLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}-placeholder
- name: !FindInMap
- Workflow
- BlipModel
- Name
RetentionInDays: 7
BlipModelPlaceholderLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}-placeholder
- name: !FindInMap
- Workflow
- BlipModel
- Name
Description: !Sub (${SolutionLowerCaseId}) BLIP model (placeholder) lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 3
Handler: index.handler
Role: !GetAtt BlipModelRole.Arn
Code:
ZipFile: |
exports.handler = async (event, context) => {
return {
caption: undefined,
};
};
# Allow AnalysisStateMachine to run Blip model
BlipModelAnalysisPolicy:
Type: AWS::IAM::Policy
Properties:
Roles:
- !Ref AnalysisStateMachineServiceRole
PolicyName: AllowAnalysisStateMachineInvokeBlipModel
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: lambda:InvokeFunction
Resource: !If
- bBlipImageArn
- !GetAtt BlipModelLambda.Arn
- !GetAtt BlipModelPlaceholderLambda.Arn
AnalysisImageStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisImage
- Name
RoleArn: !GetAtt AnalysisStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Run parallel states",
"States": {
"Run parallel states": {
"Type": "Parallel",
"Branches": [
{
"StartAt": "Start image analysis",
"States": {
"Start image analysis": {
"Type": "Task",
"Resource": "${AnalysisImageLambda.Arn}",
"Parameters": {
"operation": "start-image-analysis",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
}
}
},
{
"StartAt": "Run BLIP model",
"States": {
"Run BLIP model": {
"Type": "Task",
"Resource": "${blipLambda}",
"Parameters": {
"bucket.$": "$.input.destination.bucket",
"key.$": "$.input.image.key"
},
"ResultPath": "$.data.image",
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.1
}
]
}
}
}
],
"Next": "Index analysis results"
},
"Index analysis results": {
"Type": "Task",
"Resource": "${AnalysisImageLambda.Arn}",
"Parameters": {
"operation": "index-analysis-results",
"parallelStateOutputs.$": "$",
"stateExecution.$": "$$.Execution"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 6,
"BackoffRate": 1.1
}
]
}
}
}
-
{
blipLambda: !If [
bBlipImageArn,
!GetAtt BlipModelLambda.Arn,
!GetAtt BlipModelPlaceholderLambda.Arn
]
}
################################################################################
# Analysis Document State Machine resources
AnalysisDocumentLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisDocument
- Name
RetentionInDays: 7
AnalysisDocumentRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-document
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisDocumentLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action:
- s3:GetObject
- s3:PutObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# Textract
-
Effect: Allow
Action: textract:AnalyzeDocument
Resource: "*"
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
AnalysisDocumentLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisDocument
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis document state machine lambda (384MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 384
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisDocumentRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisDocument
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
AnalysisDocumentStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisDocument
- Name
RoleArn: !GetAtt AnalysisStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Analyze document",
"States": {
"Analyze document": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "start-document-analysis",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data",
"stateExecution.$": "$$.Execution"
},
"Next": "More pages?",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
]
},
"More pages?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.status",
"StringEquals": "COMPLETED",
"Next": "Index analysis results"
}
],
"Default": "Analyze document"
},
"Index analysis results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "index-analysis-results",
"uuid.$": "$.uuid",
"status.$": "$.status",
"progress.$": "$.progress",
"input.$": "$.input",
"data.$": "$.data"
},
"End": true,
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 4,
"BackoffRate": 1.2
}
]
}
}
}
-
{
x0: !GetAtt AnalysisDocumentLambda.Arn
}
################################################################################
# Analysis Main State Machine resources
AnalysisMainLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisMain
- Name
RetentionInDays: 7
AnalysisMainRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-main
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisMainLogGroup.Arn
# S3 - proxy bucket
-
Effect: Allow
Action: s3:ListBucket
Resource: !Sub arn:aws:s3:::${ProxyBucket}
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${ProxyBucket}/*
# DynamoDB
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# OpenSearch
-
Effect: Allow
Action:
- es:ESHttpGet
- es:ESHttpHead
- es:ESHttpPost
- es:ESHttpPut
- es:ESHttpDelete
Resource:
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}
- !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${OpenSearchDomainName}/*
# IoT
-
Effect: Allow
Action: iot:Publish
Resource: !Sub arn:aws:iot:${AWS::Region}:${AWS::AccountId}:topic/${IotTopic}
# SNS
-
Effect: Allow
Action: sns:Publish
Resource: !Ref SNSTopic
# Rekognition
-
Effect: Allow
Action: rekognition:ListFaces
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:collection/*
-
Effect: Allow
Action: rekognition:DescribeProjectVersions
Resource: !Sub arn:aws:rekognition:${AWS::Region}:${AWS::AccountId}:project/*/*
# Comprehend
-
Effect: Allow
Action: comprehend:DescribeEntityRecognizer
Resource: !Sub arn:aws:comprehend:${AWS::Region}:${AWS::AccountId}:entity-recognizer/*
# Transcribe
-
Effect: Allow
Action:
- transcribe:GetVocabulary
- transcribe:DescribeLanguageModel
Resource: "*"
AnalysisMainLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisMain
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis main state machine lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisMainRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisMain
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_ES_DOMAIN_ENDPOINT: !Ref OpenSearchDomainEndpoint
ENV_SNS_TOPIC_ARN: !Ref SNSTopic
## Default AI/ML options ##
ENV_DEFAULT_AI_OPTIONS: !Ref DefaultAIOptions
ENV_DEFAULT_MINCONFIDENCE: !Ref DefaultMinConfidence
ENV_AI_OPTIONS_S3KEY: !Ref AIOptionsS3Key
AnalysisMainStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisMain
- Name
RoleArn: !GetAtt AnalysisStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"StartAt": "Prepare analysis",
"States": {
"Prepare analysis": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "prepare-analysis",
"executionArn.$": "$$.Execution.Id",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input"
},
"Next": "Run parallel states",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Run parallel states": {
"Type": "Parallel",
"Branches": [
{
"StartAt": "Video analysis enabled?",
"States": {
"Video analysis enabled?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.input.video.enabled",
"BooleanEquals": false,
"Next": "Skip video analysis"
}
],
"Default": "Start video analysis and wait"
},
"Start video analysis and wait": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${AnalysisVideoStateMachine}"
},
"End": true
},
"Skip video analysis": {
"Type": "Succeed"
}
}
},
{
"StartAt": "Audio analysis enabled?",
"States": {
"Audio analysis enabled?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.input.audio.enabled",
"BooleanEquals": false,
"Next": "Skip audio analysis"
}
],
"Default": "Start audio analysis and wait"
},
"Start audio analysis and wait": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${AnalysisAudioStateMachine}"
},
"End": true
},
"Skip audio analysis": {
"Type": "Succeed"
}
}
},
{
"StartAt": "Image analysis enabled?",
"States": {
"Image analysis enabled?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.input.image.enabled",
"BooleanEquals": false,
"Next": "Skip image analysis"
}
],
"Default": "Start image analysis and wait"
},
"Start image analysis and wait": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${AnalysisImageStateMachine}"
},
"End": true
},
"Skip image analysis": {
"Type": "Succeed"
}
}
},
{
"StartAt": "Document analysis enabled?",
"States": {
"Document analysis enabled?": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.input.document.enabled",
"BooleanEquals": false,
"Next": "Skip document analysis"
}
],
"Default": "Start document analysis and wait"
},
"Start document analysis and wait": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"StateMachineArn": "${AnalysisDocumentStateMachine}"
},
"End": true
},
"Skip document analysis": {
"Type": "Succeed"
}
}
}
],
"Next": "Collect analysis results"
},
"Collect analysis results": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "collect-analysis-results",
"status": "NOT_STARTED",
"progress": 0,
"parallelStateOutputs.$": "$"
},
"Next": "Analysis completed",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 1,
"MaxAttempts": 2,
"BackoffRate": 1.2
}
]
},
"Analysis completed": {
"Type": "Task",
"Resource": "${x0}",
"Parameters": {
"operation": "job-completed",
"uuid.$": "$.uuid",
"status": "NOT_STARTED",
"progress": 0,
"input.$": "$.input",
"data.$": "$.data"
},
"End": true
}
}
}
-
{
x0: !GetAtt AnalysisMainLambda.Arn
}
################################################################################
#
# Analysis Automation
# * Analysis status updater
# * Backlog Status Change Event (Analysis) includes
# Amazon Rekognition, Amazon Transcribe,
# Amazon Comprehend, & Custom Labels state machine
#
################################################################################
AnalysisStatusUpdaterLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisStatusUpdater
- Name
RetentionInDays: 7
AnalysisStatusUpdaterRole:
Type: AWS::IAM::Role
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W11
reason: Wildcard character is prefixed and scoped with the ResourcePrefix
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-analysis-status-updater
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt AnalysisStatusUpdaterLogGroup.Arn
# Step Functions - send task results on MediaConvert Job Status Change Event
-
Effect: Allow
Action:
- states:SendTaskSuccess
- states:SendTaskFailure
Resource:
- !Ref AnalysisVideoStateMachine
- !Ref AnalysisAudioStateMachine
# DynamoDB - get token from ServiceToken / delete items from AIML table on REMOVE event
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${ResourcePrefix}-*
# SNS
-
Effect: Allow
Action: sns:Publish
Resource: !Ref SNSTopic
# Transcribe
-
Effect: Allow
Action: transcribe:GetTranscriptionJob
Resource: "*"
AnalysisStatusUpdaterLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- AnalysisStatusUpdater
- Name
Description: !Sub (${SolutionLowerCaseId}) analysis status updater lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt AnalysisStatusUpdaterRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- AnalysisStatusUpdater
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_SNS_TOPIC_ARN: !Ref SNSTopic
################################################################################
# Analysis Backlog Status Change Event
# * events fired by our Service Backlog EventBus
AnalysisBacklogStatusChangeEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-AnalysisBacklogStatusChangeEvent
Description: !Sub "(${SolutionId}) Backlog Analysis Status Change Event"
EventBusName: !GetAtt EventBridgeBus.Name
EventPattern:
source:
- !FindInMap
- EventBridge
- Rule
- Source
region:
- !Ref AWS::Region
detail-type:
- !FindInMap
- EventBridge
- Rule
- Detail
detail:
status:
# Rekognition (SUCCEEDED, FAILED)
- SUCCEEDED
- FAILED
# Transcribe (COMPLETED, FAILED)
- COMPLETED
# Comprehend (PROCESSING)
- PROCESSING
# Custom Labels state machine (SUCCEEDED, FAILED, ABORTED, TIMED_OUT)
- ABORTED
- TIMED_OUT
serviceApi:
# Rekognition APIs supported by backlog management
- rekognition:startcontentmoderation
- rekognition:startcelebrityrecognition
- rekognition:startfacedetection
- rekognition:startfacesearch
- rekognition:startlabeldetection
- rekognition:startpersontracking
- rekognition:startsegmentdetection
- rekognition:starttextdetection
# Transcribe APIs supported by backlog management
- transcribe:startmedicaltranscriptionjob
- transcribe:starttranscriptionjob
# Comprehend APIs supported by backlog management
- comprehend:startdocumentclassificationjob
- comprehend:startdominantlanguagedetectionjob
- comprehend:startentitiesdetectionjob
- comprehend:startkeyphrasesdetectionjob
- comprehend:startsentimentdetectionjob
- comprehend:starttopicsdetectionjob
# Custom Custom Labels API supported by backlog management
- custom:startcustomlabelsdetection
State: ENABLED
Targets:
-
Id: !Sub Id-${AnalysisStatusUpdaterLambda}
Arn: !GetAtt AnalysisStatusUpdaterLambda.Arn
AnalysisBacklogStatusChangePermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref AnalysisStatusUpdaterLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt AnalysisBacklogStatusChangeEvent.Arn
################################################################################
#
# Main (Grand) Workflow - chain Ingest and Analysis workflows
#
################################################################################
MainStateMachineServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: !Sub states.${AWS::Region}.amazonaws.com
Path: !Sub /${ResourcePrefix}/
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-main-statemachine-role
PolicyDocument:
Version: "2012-10-17"
Statement:
# Below polices are needed for Service Integration of nested workflows
# https://docs.aws.amazon.com/step-functions/latest/dg/stepfunctions-iam.html
-
Effect: Allow
Action: states:StartExecution
Resource:
- !Ref IngestMainStateMachine
- !Ref AnalysisMainStateMachine
-
Effect: Allow
Action:
- states:DescribeExecution
- states:StopExecution
Resource:
- !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${IngestMainStateMachine.Name}:*
- !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${AnalysisMainStateMachine.Name}:*
-
Effect: Allow
Action:
- events:PutTargets
- events:PutRule
- events:DescribeRule
Resource: !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule
MainStateMachine:
Type: AWS::StepFunctions::StateMachine
Properties:
StateMachineName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- Main
- Name
RoleArn: !GetAtt MainStateMachineServiceRole.Arn
DefinitionString:
!Sub
- |-
{
"Comment": "main state machine to run ingest and anlysis sub state machines",
"StartAt": "Start ingest state machine",
"States": {
"Start ingest state machine": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"input.$": "$.input"
},
"StateMachineArn": "${IngestStateMachine}"
},
"ResultSelector": {
"ingestOutput.$": "States.StringToJson($.Input)"
},
"Next": "Start analysis state machine"
},
"Start analysis state machine": {
"Type": "Task",
"Resource": "arn:aws:states:::states:startExecution.sync",
"Parameters": {
"Input": {
"input.$": "$.ingestOutput.input"
},
"StateMachineArn": "${AnalysisStateMachine}"
},
"End": true
}
}
}
-
{
IngestStateMachine: !Ref IngestMainStateMachine,
AnalysisStateMachine: !Ref AnalysisMainStateMachine
}
################################################################################
#
# State Machine Error Handling
# * Handle errors from AnalysisMain and IngestMain state machines
#
################################################################################
StateMachineErrorHandlerLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- StateMachineErrorHandler
- Name
RetentionInDays: 7
StateMachineErrorHandlerRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-state-machine-error-handler
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt StateMachineErrorHandlerLogGroup.Arn
# Step Functions
-
Effect: Allow
Action: states:DescribeStateMachine
Resource:
- !Ref IngestMainStateMachine
- !Ref AnalysisMainStateMachine
-
Effect: Allow
Action:
- states:DescribeExecution
- states:GetExecutionHistory
Resource:
- !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${IngestMainStateMachine.Name}:*
- !Sub arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${AnalysisMainStateMachine.Name}:*
# DynamoDB - allow to update executions column on ingest table
-
Effect: Allow
Action:
- dynamodb:Scan
- dynamodb:Query
- dynamodb:UpdateItem
- dynamodb:DeleteItem
Resource: !GetAtt IngestTable.Arn
# IoT
-
Effect: Allow
Action: iot:Publish
Resource: !Sub arn:aws:iot:${AWS::Region}:${AWS::AccountId}:topic/${IotTopic}
# SNS
-
Effect: Allow
Action: sns:Publish
Resource: !Ref SNSTopic
StateMachineErrorHandlerLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- StateMachineErrorHandler
- Name
Description: !Sub (${SolutionLowerCaseId}) state machine error handler lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt StateMachineErrorHandlerRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- StateMachineErrorHandler
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
ENV_SNS_TOPIC_ARN: !Ref SNSTopic
StateMachineStatusChangeEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-StateMachineStatusChange
Description: !Sub (${SolutionLowerCaseId}) State Machine Status Change Event
EventPattern:
source:
- aws.states
region:
- !Ref AWS::Region
detail-type:
- Step Functions Execution Status Change
detail:
status:
- FAILED
- ABORTED
- TIMED_OUT
stateMachineArn:
- !Ref IngestMainStateMachine
- !Ref AnalysisMainStateMachine
State: ENABLED
Targets:
-
Id: !Sub Id-${StateMachineErrorHandlerLambda}
Arn: !GetAtt StateMachineErrorHandlerLambda.Arn
StateMachineStatusChangePermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref StateMachineErrorHandlerLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt StateMachineStatusChangeEvent.Arn
################################################################################
#
# (OPTIONAL) Ingest Automation
# * Ingest S3 Event - trigger main workflow on s3:objectcreated event
#
################################################################################
IngestS3EventLogGroup:
Type: AWS::Logs::LogGroup
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W84
reason: Use default encryption. Disable additional KMS encryption requirement. https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/encrypt-log-data-kms.html
Properties:
LogGroupName: !Sub
- /aws/lambda/${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestS3Event
- Name
RetentionInDays: 7
IngestS3EventRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: Allow
Action: sts:AssumeRole
Principal:
Service: lambda.amazonaws.com
Path: !Sub /${ResourcePrefix}/
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess
Policies:
-
PolicyName: !Sub ${ResourcePrefix}-ingest-s3event
PolicyDocument:
Version: "2012-10-17"
Statement:
# CloudWatch Logs
-
Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: !GetAtt IngestS3EventLogGroup.Arn
# S3 - read object on ingest bucket
-
Effect: Allow
Action: s3:GetObject
Resource: !Sub arn:aws:s3:::${IngestBucket}/*
# Step Functions - start ingest main state machine
-
Effect: Allow
Action:
- states:DescribeStateMachine
- states:StartExecution
Resource: !Ref MainStateMachine
IngestS3EventLambda:
Type: AWS::Lambda::Function
Metadata:
cfn_nag:
rules_to_suppress:
-
id: W89
reason: Workflow not using VPC
-
id: W92
reason: Workflow not limiting simultaneous executions
Properties:
FunctionName: !Sub
- ${ResourcePrefix}-${name}
- name: !FindInMap
- Workflow
- IngestS3Event
- Name
Description: !Sub (${SolutionLowerCaseId}) ingest s3 event lambda (128MB)
Runtime: !FindInMap
- Node
- Runtime
- Version
MemorySize: 128
Timeout: 900
Handler: index.handler
Role: !GetAtt IngestS3EventRole.Arn
Code:
S3Bucket: !Ref S3Bucket
S3Key: !Sub
- ${KeyPrefix}/${package}
- package: !FindInMap
- Workflow
- IngestS3Event
- Package
Layers:
- !Ref AwsSdkLayer
- !Ref CoreLibLayer
TracingConfig:
Mode: Active
Environment:
Variables:
ENV_EXPECTED_BUCKET_OWNER: !Ref AWS::AccountId
ENV_CUSTOM_USER_AGENT: !Ref CustomUserAgent
ENV_ANONYMOUS_USAGE: !Ref AnonymousUsage
ENV_SOLUTION_UUID: !Ref SolutionUuid
ENV_SOLUTION_ID: !Ref SolutionId
ENV_RESOURCE_PREFIX: !Ref ResourcePrefix
ENV_IOT_HOST: !Ref IotHost
ENV_IOT_TOPIC: !Ref IotTopic
ENV_INGEST_BUCKET: !Ref IngestBucket
ENV_PROXY_BUCKET: !Ref ProxyBucket
IngestObjectCreatedEvent:
Type: AWS::Events::Rule
Properties:
Name: !Sub ${ResourcePrefix}-IngestObjectCreated
Description: !Sub (${SolutionLowerCaseId}) Ingest Bucket Object Created Event
EventPattern:
source:
- aws.s3
region:
- !Ref AWS::Region
detail-type:
- Object Created
detail:
bucket:
name:
- !Ref IngestBucket
State: !If
- bStartOnObjectCreation
- ENABLED
- DISABLED
Targets:
-
Id: !Sub Id-${IngestS3EventLambda}
Arn: !GetAtt IngestS3EventLambda.Arn
IngestObjectCreatedPermission:
Type: AWS::Lambda::Permission
Properties:
FunctionName: !Ref IngestS3EventLambda
Action: lambda:InvokeFunction
Principal: events.amazonaws.com
SourceArn: !GetAtt IngestObjectCreatedEvent.Arn
CustomResourcesPolicyBackend:
Type: AWS::IAM::Policy
Properties:
Roles:
- !Ref CustomResourcesRoleName
PolicyName: !Sub ${ResourcePrefix}-custom-resources-backend-stack
PolicyDocument:
Version: "2012-10-17"
Statement:
# S3 - put bucket notification
-
Effect: Allow
Action: s3:PutBucketNotification
Resource: !Sub arn:aws:s3:::${IngestBucket}
# SageMaker - describe model endpoint
-
Effect: Allow
Action: sagemaker:DescribeEndpoint
Resource: !Sub arn:aws:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint/*
ConfigureBucketNotification:
Condition: bStartOnObjectCreation
DependsOn: CustomResourcesPolicyBackend
Type: Custom::ConfigureBucketNotification
Properties:
ServiceToken: !Ref CustomResourcesLambdaArn
Data:
Bucket: !Ref IngestBucket
NotificationConfiguration:
EventBridgeConfiguration:
EventBridgeEnabled: !Ref AWS::NoValue
Outputs:
# SNS Topic
SNSTopicArn:
Value: !Ref SNSTopic
Description: SNSTopicArn
# DyanmoDB
BacklogTable:
Value: !Ref BacklogTable
Description: BacklogTable
BacklogTableArn:
Value: !GetAtt BacklogTable.Arn
Description: BacklogTableArn
AtomicLockTable:
Value: !Ref AtomicLockTable
Description: AtomicLockTable
AtomicLockTableArn:
Value: !GetAtt AtomicLockTable.Arn
Description: AtomicLockTableArn
ServiceTokenTable:
Value: !Ref ServiceTokenTable
Description: ServiceTokenTable
ServiceTokenTableArn:
Value: !GetAtt ServiceTokenTable.Arn
Description: ServiceTokenTableArn
IngestTable:
Value: !Ref IngestTable
Description: IngestTable
IngestTableArn:
Value: !GetAtt IngestTable.Arn
Description: IngestTableArn
AnalysisTable:
Value: !Ref AnalysisTable
Description: AnalysisTable
AnalysisTableArn:
Value: !GetAtt AnalysisTable.Arn
Description: AnalysisTableArn
# State Machines
BacklogCustomLabelsStateMachine:
Value: !Ref BacklogCustomLabelsStateMachine
Description: BacklogCustomLabelsStateMachine
BacklogCustomLabelsStateMachineName:
Value: !GetAtt BacklogCustomLabelsStateMachine.Name
Description: BacklogCustomLabelsStateMachineName
IngestMainStateMachine:
Value: !Ref IngestMainStateMachine
Description: IngestMainStateMachine
IngestMainStateMachineName:
Value: !GetAtt IngestMainStateMachine.Name
Description: IngestMainStateMachine
AnalysisMainStateMachine:
Value: !Ref AnalysisMainStateMachine
Description: AnalysisMainStateMachine
AnalysisMainStateMachineName:
Value: !GetAtt AnalysisMainStateMachine.Name
Description: AnalysisMainStateMachine
MainStateMachine:
Value: !Ref MainStateMachine
Description: MainStateMachine
MainStateMachineName:
Value: !GetAtt MainStateMachine.Name
Description: MainStateMachine
# Misc.
DefaultAIOptions:
Value: !Ref DefaultAIOptions
Description: DefaultAIOptions
DefaultMinConfidence:
Value: !Ref DefaultMinConfidence
Description: DefaultMinConfidence