AWSTemplateFormatVersion: '2010-09-09' Transform: 'AWS::Serverless-2016-10-31' Description: 'Meter Data Lake prediction pipeline (qs-1r18anahd)' Mappings: TrainingImageMap: us-west-1: image: "632365934929.dkr.ecr.us-west-1.amazonaws.com" us-west-2: image: "156387875391.dkr.ecr.us-west-2.amazonaws.com" us-east-1: image: "522234722520.dkr.ecr.us-east-1.amazonaws.com" us-east-2: image: "566113047672.dkr.ecr.us-east-2.amazonaws.com" ap-east-1: image: "286214385809.dkr.ecr.ap-east-1.amazonaws.com" ap-northeast-1: image: "633353088612.dkr.ecr.ap-northeast-1.amazonaws.com" ap-northeast-2: image: "204372634319.dkr.ecr.ap-northeast-2.amazonaws.com" ap-south-1: image: "991648021394.dkr.ecr.ap-south-1.amazonaws.com" ap-southeast-1: image: "475088953585.dkr.ecr.ap-southeast-1.amazonaws.com" ap-southeast-2: image: "514117268639.dkr.ecr.ap-southeast-2.amazonaws.com" ca-central-1: image: "469771592824.dkr.ecr.ca-central-1.amazonaws.com" cn-north-1: image: "390948362332.dkr.ecr.cn-north-1.amazonaws.com.cn" cn-northwest-1: image: "387376663083.dkr.ecr.cn-northwest-1.amazonaws.com.cn" eu-north-1: image: "669576153137.dkr.ecr.eu-north-1.amazonaws.com" eu-central-1: image: "495149712605.dkr.ecr.eu-central-1.amazonaws.com" eu-west-1: image: "224300973850.dkr.ecr.eu-west-1.amazonaws.com" eu-west-2: image: "644912444149.dkr.ecr.eu-west-2.amazonaws.com" eu-west-3: image: "749696950732.dkr.ecr.eu-west-3.amazonaws.com" me-south-1: image: "249704162688.dkr.ecr.me-south-1.amazonaws.com" sa-east-1: image: "855470959533.dkr.ecr.sa-east-1.amazonaws.com" us-gov-west-1: image: "226302683700.dkr.ecr.us-gov-west-1.amazonaws.com" Globals: Function: Runtime: python3.7 Environment: Variables: Db_schema: !Ref DBName Athena_bucket: !Ref AthenaQueryBucket Working_bucket: !Ref WorkingBucket With_weather_data: !Ref WithWeather config_table: !Ref ConfigTable Resources: AthenaQueryBucket: Type: AWS::S3::Bucket WorkingBucket: Type: AWS::S3::Bucket MachineLearningPipelineOrchestratorStateMachine: Type: 'AWS::Serverless::StateMachine' Properties: DefinitionUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/statemachine/ml-pipeline-orchestrator.asl.json' DefinitionSubstitutions: InitialTrainingStateMachine: !Ref MachineLearningPipelineModelTrainingStateMachine PredictionPipelineStateMachine: !Ref MachineLearningPipelinePreCalculationStateMachine CheckInitialPipelineRun: !Ref CheckInitialPipelineRun Policies: - Version: 2012-10-17 Statement: - Effect: Allow Action: - events:PutTargets - events:PutRule - events:DescribeRule Resource: !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule - StepFunctionsExecutionPolicy: StateMachineName: !GetAtt MachineLearningPipelineModelTrainingStateMachine.Name - StepFunctionsExecutionPolicy: StateMachineName: !GetAtt MachineLearningPipelinePreCalculationStateMachine.Name - LambdaInvokePolicy: FunctionName: !Ref CheckInitialPipelineRun # init training statemachine MachineLearningPipelineModelTrainingStateMachine: Type: 'AWS::Serverless::StateMachine' Properties: DefinitionUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/statemachine/ml-pipeline-model-training.asl.json' DefinitionSubstitutions: WorkingBucket: !Ref WorkingBucket LambdaPrepareTrainingName: !Ref PrepareTrainingFunction AccountId: !Sub ${AWS::AccountId} SageMakerRole: !GetAtt SageMakerRole.Arn TrainingImage: !FindInMap [TrainingImageMap, !Ref "AWS::Region", "image"] LoadPipelineParameter: !Ref LoadPipelineParameter HasEndpoint: !Ref HasEndpoint Policies: - Version: 2012-10-17 Statement: - Effect: Allow Action: - "iam:PassRole" Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/*" - Effect: Allow Action: - events:DescribeRule - events:PutRule - events:PutTargets Resource: - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForSageMakerTrainingJobsRule" - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" - Effect: Allow Action: - sagemaker:CreateTrainingJob - sagemaker:DescribeTrainingJob - sagemaker:StopTrainingJob - sagemaker:CreateHyperParameterTuningJob - sagemaker:DescribeHyperParameterTuningJob - sagemaker:StopHyperParameterTuningJob - sagemaker:CreateModel - sagemaker:CreateEndpointConfig - sagemaker:CreateEndpoint - sagemaker:DeleteEndpointConfig - sagemaker:DeleteEndpoint - sagemaker:UpdateEndpoint - sagemaker:AddTags - sagemaker:ListTags Resource: - !Sub "arn:${AWS::Partition}:sagemaker:${AWS::Region}:${AWS::AccountId}:training-job/*" - !Sub "arn:${AWS::Partition}:sagemaker:${AWS::Region}:${AWS::AccountId}:model/*" - !Sub "arn:${AWS::Partition}:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint/*" - !Sub "arn:${AWS::Partition}:sagemaker:${AWS::Region}:${AWS::AccountId}:endpoint-config/*" - LambdaInvokePolicy: FunctionName: !Ref PrepareTrainingFunction - LambdaInvokePolicy: FunctionName: !Ref LoadPipelineParameter - LambdaInvokePolicy: FunctionName: !Ref HasEndpoint # Step functions state machine MachineLearningPipelinePreCalculationStateMachine: Type: 'AWS::Serverless::StateMachine' Properties: DefinitionUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/statemachine/ml-pipeline-pre-calculation.asl.json' DefinitionSubstitutions: LambdaSplitFunctionName: !Ref SplitBatchFunction LambdaPrepareFunctionName: !Ref PrepareBatchFunction LambdaResultFunctionName: !Ref UploadResultFunction LambdaAnomalyFunctionName: !Ref BatchAnomalyDetection LambdaTriggerCrawlerFunctionName: !Ref TriggerGlueCrawlerFunction LambdaGetCrawlerStateFunctionName: !Ref GetGlueCrawlerStateFunction LoadPipelineParameter: !Ref LoadPipelineParameter Policies: - Version: 2012-10-17 Statement: - Effect: Allow Action: - sagemaker:CreateTransformJob - sagemaker:DescribeTransformJob - sagemaker:StopTransformJob Resource: !Sub "arn:${AWS::Partition}:sagemaker:${AWS::Region}:${AWS::AccountId}:transform-job/*" - Effect: Allow Action: - events:DescribeRule - events:PutRule - events:PutTargets Resource: - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForSageMakerTransformJobsRule" - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForStepFunctionsExecutionRule" - LambdaInvokePolicy: FunctionName: !Ref SplitBatchFunction - LambdaInvokePolicy: FunctionName: !Ref BatchAnomalyDetection - LambdaInvokePolicy: FunctionName: !Ref PrepareBatchFunction - LambdaInvokePolicy: FunctionName: !Ref UploadResultFunction - LambdaInvokePolicy: FunctionName: !Ref TriggerGlueCrawlerFunction - LambdaInvokePolicy: FunctionName: !Ref GetGlueCrawlerStateFunction - LambdaInvokePolicy: FunctionName: !Ref LoadPipelineParameter # # Lambda functions # LoadPipelineParameter: Type: 'AWS::Serverless::Function' Properties: MemorySize: 128 Timeout: 30 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_load_pipeline_parameter.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer PrepareTrainingFunction: Type: 'AWS::Serverless::Function' Properties: MemorySize: 512 Timeout: 600 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_prepare_training.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer CheckInitialPipelineRun: Type: 'AWS::Serverless::Function' Properties: MemorySize: 128 Timeout: 30 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_check_initial_pipeline_run.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer StateTopicSubscription: Type: 'AWS::Serverless::Function' Properties: MemorySize: 128 Timeout: 30 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_state_topic_subscription.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Environment: Variables: ml_pipeline_trigger_state_machine: !Ref MachineLearningPipelineOrchestratorStateMachine Events: EventBridgeRule: Type: EventBridgeRule Properties: Pattern: source: - "aws.glue" detail-type: - "Glue Crawler State Change" detail: state: - "Succeeded" crawlerName: - !Ref GlueCrawlerNameTrigger Layers: - !Ref DependencyLayer SplitBatchFunction: Type: 'AWS::Serverless::Function' Properties: Timeout: 300 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_split_batch.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer PrepareBatchFunction: Type: 'AWS::Serverless::Function' Properties: MemorySize: 512 Timeout: 900 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_prepare_batch.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer UploadResultFunction: Type: 'AWS::Serverless::Function' Properties: Timeout: 180 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_upload_result.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref DependencyLayer BatchAnomalyDetection: Type: 'AWS::Serverless::Function' Properties: Handler: app.lambda_handler Runtime: python3.7 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_batch_anomaly_detection.zip' MemorySize: 2048 Timeout: 600 Role: !GetAtt 'RunLambdaRole.Arn' Layers: - !Ref BatchAnomalyDependencyLayer TriggerGlueCrawlerFunction: Type: 'AWS::Serverless::Function' Properties: Timeout: 60 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_crawler_trigger_glue_crawler.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' GetGlueCrawlerStateFunction: Type: 'AWS::Serverless::Function' Properties: Timeout: 60 CodeUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_crawler_get_glue_crawler_state.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' HasEndpoint: Type: 'AWS::Serverless::Function' Properties: Timeout: 60 CodeUri: Bucket: !If [ UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName ] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/ml_pipeline_has_endpoint.zip' Handler: app.lambda_handler Role: !GetAtt 'RunLambdaRole.Arn' # # Lambda Dependency Layer # BatchAnomalyDependencyLayer: Type: 'AWS::Serverless::LayerVersion' Properties: LayerName: !Sub "ml-app-dependencies-${AWS::Region}" Description: Dependencies for sam app ContentUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/dependencies/anomaly_detection/layer.zip' CompatibleRuntimes: - python3.7 LicenseInfo: 'Available under the MIT-0 license.' DependencyLayer: Type: AWS::Serverless::LayerVersion Properties: LayerName: !Sub "common-app-dependencies-${AWS::Region}" ContentUri: Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName] Key: Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/dependencies/common/common-layer.zip' CompatibleRuntimes: - python3.7 LicenseInfo: 'MIT' # # Crawler # ForecastCrawler: Type: "AWS::Glue::Crawler" Properties: Role: !Sub "service-role/${CrawlerRole}" Targets: S3Targets: - Path: !Sub "${WorkingBucket}/meteranalytics/forecast" DatabaseName: !Ref DBName SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "DEPRECATE_IN_DATABASE" AnomalyCrawler: Type: "AWS::Glue::Crawler" Properties: Role: !Sub "service-role/${CrawlerRole}" Targets: S3Targets: - Path: !Sub "${WorkingBucket}/meteranalytics/anomaly" DatabaseName: !Ref DBName SchemaChangePolicy: UpdateBehavior: "UPDATE_IN_DATABASE" DeleteBehavior: "DEPRECATE_IN_DATABASE" # # glue database # GlueTableGeodata: Type: "AWS::Glue::Table" Properties: DatabaseName: !Ref DBName CatalogId: !Ref "AWS::AccountId" TableInput: Name: "geodata" TableType: "EXTERNAL_TABLE" StorageDescriptor: Columns: - Name: "col0" Type: "string" - Name: "col1" Type: "double" - Name: "col2" Type: "double" Location: !Sub "s3://${BusinessZoneS3Bucket}/geodata/" InputFormat: "org.apache.hadoop.mapred.TextInputFormat" OutputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" Compressed: false NumberOfBuckets: -1 SerdeInfo: SerializationLibrary: "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe" Parameters: "field.delim": "," "skip.header.line.count": 0 Parameters: CrawlerSchemaDeserializerVersion: "1.0" CrawlerSchemaSerializerVersion: "1.0" areColumnsQuoted: "false" GlueTableAnomaly: Type: "AWS::Glue::Table" Properties: DatabaseName: !Ref DBName CatalogId: !Ref "AWS::AccountId" TableInput: Name: "anomaly" TableType: "EXTERNAL_TABLE" StorageDescriptor: Columns: - Name: "meter_id" Type: "string" - Name: "ds" Type: "string" - Name: "consumption" Type: "double" - Name: "yhat_lower" Type: "double" - Name: "yhat_upper" Type: "double" - Name: "anomaly" Type: "bigint" - Name: "importance" Type: "double" Location: !Sub "s3://${WorkingBucket}/meteranalytics/anomaly/" InputFormat: "org.apache.hadoop.mapred.TextInputFormat" OutputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" Compressed: false NumberOfBuckets: -1 SerdeInfo: SerializationLibrary: "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe" Parameters: "field.delim": "," "skip.header.line.count": 1 Parameters: CrawlerSchemaDeserializerVersion: "1.0" CrawlerSchemaSerializerVersion: "1.0" areColumnsQuoted: "false" UPDATED_BY_CRAWLER: !Ref AnomalyCrawler # # IAM roles # CrawlerRole: Type: "AWS::IAM::Role" Properties: Path: "/service-role/" AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"glue.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" MaxSessionDuration: 3600 ManagedPolicyArns: - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess" - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole" RunLambdaRole: Type: 'AWS::IAM::Role' Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: 'lambda.amazonaws.com' Action: 'sts:AssumeRole' ManagedPolicyArns: - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonAthenaFullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonSageMakerFullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AWSStepFunctionsFullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonDynamoDBFullAccess' SageMakerRole: Type: 'AWS::IAM::Role' Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: 'sagemaker.amazonaws.com' Action: 'sts:AssumeRole' ManagedPolicyArns: - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonAthenaFullAccess' - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonSageMakerFullAccess' Metadata: AWS::CloudFormation::Interface: ParameterGroups: - Label: default: AWS Quick Start configuration Parameters: - QSS3BucketName - QSS3KeyPrefix - DBName ParameterLabels: QSS3BucketName: default: Quick Start S3 bucket name QSS3KeyPrefix: default: Quick Start S3 key prefix DBName: default: Glue data catalog database name Parameters: QSS3BucketName: AllowedPattern: '^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$' ConstraintDescription: >- Quick Start bucket name can include numbers, lowercase letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-). Default: aws-quickstart Description: >- S3 bucket name for the Quick Start assets. Only change this value if you customize or extend the Quick Start for your own use. This string can include numbers, lowercase letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen (-). Type: String QSS3KeyPrefix: AllowedPattern: '^[0-9a-zA-Z-/]*[/]$' ConstraintDescription: >- Quick Start key prefix can include numbers, lowercase letters, uppercase letters, hyphens (-), and forward slash (/) and must terminate in a forward slash. Default: quickstart-aws-utility-meter-data-analytics-platform/ Type: String Description: S3 key prefix for the Quick Start assets. Quick Start key prefix can include numbers, lowercase letters, uppercase letters, hyphens (-), and forward slash (/). DBName: ConstraintDescription: >- Name of the glue data catalog database Default: meterdata Type: String GlueCrawlerNameTrigger: Type: String BusinessZoneS3Bucket: Type: String WithWeather: Description: >- Enables or disables the use of weather data. 0 = weather data won't be used 1 = enable weather data use Type: Number ConfigTable: Type: String Conditions: UsingDefaultBucket: !Equals [!Ref QSS3BucketName, 'aws-quickstart'] Outputs: AthenaQueryBucket: Value: !Ref AthenaQueryBucket WorkingBucket: Value: !Ref WorkingBucket