AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 Description: Deploys infrastructure to run well analytics Parameters: # Takes in the username and passwords for the Kibana Dashboard Email: Description: Serves as username and email for system Type: String Default: test@example.com AllowedPattern: (?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\]) DeploymentType: Description: Deploy as either a test or a workshop. Selecting workshop will exponentially increase the cost of the deployment Type: String Default: Test AllowedValues: ["Test","Workshop"] Resources: #S3 Bucket where all the data files will go S3Bucket: Type: AWS::S3::Bucket Properties: BucketName: !Sub ${AWS::AccountId}-bsee-well-analytics NotificationConfiguration: LambdaConfigurations: - Event: s3:ObjectCreated:* Function: Fn::GetAtt: - MvWarMainDynamo - Arn Filter: S3Key: Rules: - Name: prefix Value: mv_war_main/ - Event: s3:ObjectCreated:* Function: Fn::GetAtt: - MvWarMainPropDynamo - Arn Filter: S3Key: Rules: - Name: prefix Value: mv_war_main_prop/ - Event: s3:ObjectCreated:* Function: Fn::GetAtt: - WellLocationDynamo - Arn Filter: S3Key: Rules: - Name: prefix Value: mv_boreholes/ - Event: s3:ObjectCreated:* Function: Fn::GetAtt: - SubmitComprehendLambda - Arn Filter: S3Key: Rules: - Name: prefix Value: model-input/ - Event: s3:ObjectCreated:* Function: Fn::GetAtt: - ProcessEventRecordLambda - Arn Filter: S3Key: Rules: - Name: prefix Value: model-output/ # Cognito Setup with ES CognitoUserPool: Type: AWS::Cognito::UserPool Properties: UserPoolName: CognitoUserPool UsernameAttributes: - email EmailVerificationSubject: Fn::Sub: Kibana Dashboard User Creation AutoVerifiedAttributes: - email Schema: - Name: email AttributeDataType: String Mutable: true Required: true UserPoolDomain: Type: AWS::Cognito::UserPoolDomain Properties: UserPoolId: !Ref CognitoUserPool Domain: !Sub ${AWS::AccountId}-well-analytics-user-pool-domain CognitoUserPoolClient: Type: AWS::Cognito::UserPoolClient Properties: ClientName: CognitoUserPoolClient UserPoolId: !Ref CognitoUserPool IdentityPool: Type: "AWS::Cognito::IdentityPool" Properties: IdentityPoolName: KibanaIdentity AllowUnauthenticatedIdentities: true CognitoIdentityProviders: - ClientId: !Ref CognitoUserPoolClient ProviderName: !GetAtt CognitoUserPool.ProviderName CognitoUserPoolAdmin: Type: AWS::Cognito::UserPoolUser Properties: Username: !Ref Email UserPoolId: !Ref CognitoUserPool UserAttributes: - Name: email Value: !Ref Email AuthenticatedPolicy: Type: AWS::IAM::ManagedPolicy Properties: PolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Action: - "es:ESHttp*" Resource: - "*" CognitoAuthorizedRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Federated: "cognito-identity.amazonaws.com" Action: - "sts:AssumeRoleWithWebIdentity" Condition: StringEquals: "cognito-identity.amazonaws.com:aud": !Ref IdentityPool "ForAnyValue:StringLike": "cognito-identity.amazonaws.com:amr": authenticated ManagedPolicyArns: - !Ref AuthenticatedPolicy # Assigns the roles to the Identity Pool IdentityPoolRoleMapping: Type: "AWS::Cognito::IdentityPoolRoleAttachment" Properties: IdentityPoolId: !Ref IdentityPool Roles: authenticated: !GetAtt CognitoAuthorizedRole.Arn # Creates the ES Cluster ElasticsearchDomain: Type: AWS::Elasticsearch::Domain Properties: DomainName: "drilling-reports" ElasticsearchClusterConfig: InstanceCount: 1 InstanceType: "m4.large.elasticsearch" ElasticsearchVersion: 7.1 CognitoOptions: Enabled: true UserPoolId: !Ref CognitoUserPool IdentityPoolId: !Ref IdentityPool RoleArn: Fn::GetAtt: - EsCognitoRoleARN - Arn EBSOptions: EBSEnabled: true Iops: 0 VolumeSize: 10 VolumeType: "gp2" AccessPolicies: Version: "2012-10-17" Statement: - Effect: Allow Principal: AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root" Action: "es:*" Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/drilling-reports/* - Effect: Allow Principal: AWS: Fn::GetAtt: - ESLambdaRole - Arn Action: "es:*" Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/drilling-reports/* # Just a queue to decouple writing to the ES cluster and reading from comprehend results MyQueue: Type: AWS::SQS::Queue Properties: QueueName: Ingest-ES-Queue VisibilityTimeout: 150 # Next few items are the dynamo db tables MvWarMainTable: Type: AWS::DynamoDB::Table Properties: KeySchema: - AttributeName: SN_WAR KeyType: HASH AttributeDefinitions: - AttributeName: SN_WAR AttributeType: S BillingMode: PAY_PER_REQUEST TableName: mv-war-main MvWarMainPropsTable: Type: AWS::DynamoDB::Table Properties: AttributeDefinitions: - AttributeName: SN_WAR AttributeType: S KeySchema: - AttributeName: SN_WAR KeyType: HASH BillingMode: PAY_PER_REQUEST TableName: mv-war-main-prop BseeWellLocationsTable: Type: AWS::DynamoDB::Table Properties: KeySchema: - AttributeName: API_WELL_NUMBER KeyType: HASH AttributeDefinitions: - AttributeName: API_WELL_NUMBER AttributeType: S BillingMode: PAY_PER_REQUEST TableName: BSEE-WellLocations # Next few are the invoke permissions S3 needs to trigger Lambdas MvWarMainDynamoInvokePermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - MvWarMainDynamo - Arn Principal: s3.amazonaws.com SourceAccount: !Sub ${AWS::AccountId} SourceArn: !Sub arn:aws:s3:::${AWS::AccountId}-bsee-well-analytics MvWarMainPropDyanmoInvokePermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - MvWarMainPropDynamo - Arn Principal: s3.amazonaws.com SourceAccount: !Sub ${AWS::AccountId} SourceArn: !Sub arn:aws:s3:::${AWS::AccountId}-bsee-well-analytics WellLocationInvokePermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - WellLocationDynamo - Arn Principal: s3.amazonaws.com SourceAccount: !Sub ${AWS::AccountId} SourceArn: !Sub arn:aws:s3:::${AWS::AccountId}-bsee-well-analytics SubmitComprehendInvokePermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - SubmitComprehendLambda - Arn Principal: s3.amazonaws.com SourceAccount: !Sub ${AWS::AccountId} SourceArn: !Sub arn:aws:s3:::${AWS::AccountId}-bsee-well-analytics ProcessEventRecordInvokePermission: Type: AWS::Lambda::Permission Properties: Action: lambda:InvokeFunction FunctionName: Fn::GetAtt: - ProcessEventRecordLambda - Arn Principal: s3.amazonaws.com SourceAccount: !Sub ${AWS::AccountId} SourceArn: !Sub arn:aws:s3:::${AWS::AccountId}-bsee-well-analytics # The Roles needed to execute stuff GetDataRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3FullAccess EsCognitoRoleARN: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - es.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonESCognitoAccess UploadDynamoRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess ComprehendTrainingRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - comprehend.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/ComprehendFullAccess ComprehendLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/ComprehendFullAccess - arn:aws:iam::aws:policy/AWSLambdaFullAccess ProcessEventRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: sts:AssumeRole Path: / ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess - arn:aws:iam::aws:policy/AmazonSQSFullAccess StatesExecutionRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - !Sub states.${AWS::Region}.amazonaws.com Action: "sts:AssumeRole" Path: "/" Policies: - PolicyName: StatesExecutionPolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "lambda:InvokeFunction" Resource: "*" ESLambdaRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - lambda.amazonaws.com Action: "sts:AssumeRole" Path: "/" ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonSQSFullAccess Policies: - PolicyName: UploadToElasticsearch PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "es:*" Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/drilling-reports BasicExecutionPolicy: Type: AWS::IAM::Policy Properties: PolicyName: lambda-log PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: arn:aws:logs:*:*:* Roles: - Ref: GetDataRole - Ref: ComprehendLambdaRole - Ref: ComprehendTrainingRole - Ref: StatesExecutionRole - Ref: UploadDynamoRole - Ref: ESLambdaRole - Ref: ProcessEventRole # Layers for lambdas RequestsLayer: Type: AWS::Serverless::LayerVersion Properties: LayerName: Requests Description: Requests Library for api calls ContentUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: Layers/requests.zip CompatibleRuntimes: - python3.7 PandasLayer: Type: AWS::Serverless::LayerVersion Properties: LayerName: Pandas Description: Pandas dataframe library ContentUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: Layers/Pandas.zip CompatibleRuntimes: - python3.7 AWSauthLayer: Type: AWS::Serverless::LayerVersion Properties: LayerName: AWS4Auth Description: Auth for AWS role ContentUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: Layers/AWS4Auth-Lambda.zip CompatibleRuntimes: - python3.7 ElasticsearchPythonLayer: Type: AWS::Serverless::LayerVersion Properties: LayerName: ElasticsearchPackage Description: Elasticsearch python package ContentUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: Layers/ElasticSearch-LambdaLayer.zip CompatibleRuntimes: - python3.7 # Actual Lambda Functions DownloadBseeData: Type: AWS::Serverless::Function Properties: Handler: getBSEEdata.lambda_handler Description: Downloads the BSEE Data Set MemorySize: 640 Timeout: 900 Runtime: python3.7 Layers: - !Ref RequestsLayer Role: Fn::GetAtt: - GetDataRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: getBSEEdata.py.zip Environment: Variables: S3Bucket: Ref: S3Bucket DownloadWellLocationData: Type: AWS::Serverless::Function Properties: Handler: DownloadWellLocationData.wellLocation Description: Downloads the well geolocation Data MemorySize: 128 Timeout: 300 Runtime: python3.7 Layers: - !Ref RequestsLayer Role: Fn::GetAtt: - GetDataRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: DownloadWellLocationData.py.zip Environment: Variables: S3Bucket: Ref: S3Bucket ComprehendTrainingLambda: Type: AWS::Serverless::Function Properties: Handler: ComprehendTraining.lambda_handler Description: Creates Comprehend Training Set MemorySize: 768 Timeout: 600 Runtime: python3.7 Layers: - !Ref PandasLayer Role: Fn::GetAtt: - ComprehendLambdaRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: ComprehendTraining.py.zip Environment: Variables: data_access_role_arn: Fn::GetAtt: - ComprehendTrainingRole - Arn S3Bucket: Ref: S3Bucket CheckTrainingStatusLambda: Type: AWS::Serverless::Function Properties: Handler: CheckComprehend.lambda_handler Description: Checks whether the training is complete MemorySize: 128 Timeout: 5 Runtime: python3.7 Role: Fn::GetAtt: - ComprehendLambdaRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: CheckComprehend.py.zip SNWARSplit: Type: AWS::Serverless::Function Properties: Handler: SNWARSplit.lambda_handler Description: Splits remark data into chunks that comprehend can process MemorySize: 640 Timeout: 300 Layers: - !Ref PandasLayer Runtime: python3.7 Role: Fn::GetAtt: - GetDataRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: SNWARSplit.py.zip Environment: Variables: S3Bucket: Ref: S3Bucket Type: Ref: DeploymentType MvWarMainDynamo: Type: AWS::Serverless::Function Properties: Handler: MvWarMainDynamo.lambda_handler Description: Uploads mv war main data to dynamo MemorySize: 640 Timeout: 900 Layers: - !Ref PandasLayer Runtime: python3.7 Role: Fn::GetAtt: - UploadDynamoRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: MvWarMainDynamo.py.zip Environment: Variables: DynamoDbTable: Ref: MvWarMainTable MvWarMainPropDynamo: Type: AWS::Serverless::Function Properties: Handler: MvWarMainPropDynamo.lambda_handler Description: Uploads mv war main prop data to dynamo MemorySize: 128 Timeout: 300 Layers: - !Ref PandasLayer Runtime: python3.7 Role: Fn::GetAtt: - UploadDynamoRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: MvWarMainPropDynamo.py.zip Environment: Variables: DynamoDbTable: Ref: MvWarMainPropsTable WellLocationDynamo: Type: AWS::Serverless::Function Properties: Handler: WellLocationDynamo.lambda_handler Description: Uploads mv war main prop data to dynamo MemorySize: 128 Timeout: 300 Layers: - !Ref PandasLayer Runtime: python3.7 Role: Fn::GetAtt: - UploadDynamoRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: WellLocationDynamo.py.zip Environment: Variables: DynamoDbTable: Ref: BseeWellLocationsTable SubmitComprehendLambda: Type: AWS::Serverless::Function Properties: Handler: SubmitComprehendJob.lambda_handler Description: Submits comprehend jobs MemorySize: 256 Timeout: 900 Runtime: python3.7 Role: Fn::GetAtt: - ComprehendLambdaRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: SubmitComprehendJob.py.zip Environment: Variables: data_access_arn: Fn::GetAtt: - ComprehendTrainingRole - Arn recognizer_arn: !Sub arn:aws:comprehend:${AWS::Region}:${AWS::AccountId}:entity-recognizer/well-events-blog ProcessEventRecordLambda: Type: AWS::Serverless::Function Properties: Handler: ProcessEventRecord.lambda_handler Description: Process the Comprehend output MemorySize: 640 Timeout: 300 Runtime: python3.7 Role: Fn::GetAtt: - ProcessEventRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: ProcessEventRecord.py.zip Environment: Variables: DynamoMvWarMain: !Ref MvWarMainTable DynamoMvWarMainProp: !Ref MvWarMainPropsTable QueueName: Fn::GetAtt: - MyQueue - QueueName IngestESLambda: Type: AWS::Serverless::Function Properties: Handler: IngestES.lambda_handler Description: Pushes the messages to the ES domain. MemorySize: 128 Timeout: 15 Runtime: python3.7 Layers: - !Ref AWSauthLayer - !Ref ElasticsearchPythonLayer Role: Fn::GetAtt: - ESLambdaRole - Arn CodeUri: Bucket: !Sub "${AWS::AccountId}-bsee-sourcecode" Key: IngestES.py.zip Environment: Variables: es_endpoint: Fn::GetAtt: - ElasticsearchDomain - DomainEndpoint Region: !Sub ${AWS::Region} IngestESTrigger: Type: AWS::Lambda::EventSourceMapping Properties: BatchSize: 1 FunctionName: Fn::GetAtt: - IngestESLambda - Arn EventSourceArn: Fn::GetAtt: - MyQueue - Arn MyStateMachine: Type: AWS::StepFunctions::StateMachine Properties: StateMachineName: DataCollectionStateMachine DefinitionString: !Sub - |- { "Comment": "A Hello World example of the Amazon States Language using Pass states", "StartAt": "DataDownloads", "States": { "DataDownloads": { "Type": "Parallel", "Branches": [ { "StartAt": "BSEE Zip", "States": { "BSEE Zip": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${downloadBseeARN}", "Payload": { "Input.$": "$" } }, "Next": "ComprehendTraining" }, "ComprehendTraining": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${comprehendTrainingARN}", "Payload": { "Input.$": "$.Payload" } }, "Next": "WaitForTranscriptionComplete" }, "WaitForTranscriptionComplete": { "Type": "Wait", "Seconds": 30, "Next": "Get Job Status" }, "Get Job Status": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${checkComprehendTrainingARN}", "Payload": { "Input.$": "$.Payload" } }, "Next": "Transcript Complete?" }, "Transcript Complete?": { "Type": "Choice", "Choices": [ { "Variable": "$.Payload.jobStatus", "StringEquals": "FAILED", "Next": "Fail to train data" }, { "Variable": "$.Payload.jobStatus", "StringEquals": "COMPLETED", "Next": "Completed Training" } ], "Default": "WaitForTranscriptionComplete" }, "Completed Training": { "Type": "Succeed" }, "Fail to train data": { "Type": "Fail" } } }, { "StartAt": "Well Location Download", "States": { "Well Location Download": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${wellLocationDownloadARN}", "Payload": { "Input.$": "$" } }, "End": true } } } ], "Next": "Split SNWAR" }, "Split SNWAR": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": "${splitSnwarARN}", "Payload": { "Input.$": "$" } }, "End": true } } } - { downloadBseeARN: !GetAtt [DownloadBseeData, Arn], comprehendTrainingARN: !GetAtt [ComprehendTrainingLambda, Arn], checkComprehendTrainingARN: !GetAtt [CheckTrainingStatusLambda, Arn], wellLocationDownloadARN: !GetAtt [DownloadWellLocationData, Arn], splitSnwarARN: !GetAtt [SNWARSplit, Arn], } RoleArn: Fn::GetAtt: - StatesExecutionRole - Arn Outputs: KibanaDashboard: Description: The web link for the Kibana Dashboard, use this to view the data Value: !Sub https://${ElasticsearchDomain.DomainEndpoint}/_plugin/kibana/