AWSTemplateFormatVersion: 2010-09-09

Transform: AWS::Serverless-2016-10-31

Description: Workload Discovery on AWS Amazon Athena and AWS Glue Stack

Parameters:
  CostAndUsageBucket:
    Type: String
    Description: 'The cost and usage bucket created by Perspective at deploy time e.g. aws-perspective-v(DEPLOYED-VERSION)-costandusagereportbucket-(ID-STRING)'
  CURCrawlerKey:
    Type: String
    Default: 'perspective_glue_crawler'
  DeploymentBucket:
    Type: String
    Description: "The bucket that contains all the deployment code for Perspective"
  DeploymentBucketKey:
    Type: String
    Description: "The key to the deployment code for Perspective"

Resources:
  AWSCURDatabase:
    Type: 'AWS::Glue::Database'
    Properties:
      DatabaseInput:
        Name: 'athenacurcfn_aws_perspective_cost_and_usage'
      CatalogId: !Ref AWS::AccountId

  AWSCURCrawlerComponentFunction:
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W11
            reason: 'This is an autogenerated template from the CUR set up'
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      ManagedPolicyArns:
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole'
      Policies:
        - PolicyName: AWSCURCrawlerComponentFunction
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 'glue:UpdateDatabase'
                  - 'glue:UpdatePartition'
                  - 'glue:CreateTable'
                  - 'glue:UpdateTable'
                  - 'glue:ImportCatalogToGlue'
                Resource: '*'
              - Effect: Allow
                Action:
                  - 's3:GetObject'
                  - 's3:PutObject'
                Resource: !Sub 'arn:${AWS::Partition}:s3:::${CostAndUsageBucket}/${CURCrawlerKey}/*'

  AWSCURCrawlerLambdaExecutor:
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W11
            reason: 'This is an autogenerated template from the CUR set up'
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      Policies:
        - PolicyName: AWSCURCrawlerLambdaExecutor
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - s3:PutObject*
                  - s3:ListBucket*
                  - s3:GetObject*
                  - s3:PutBucketLogging
                Resource:
                  - !Sub arn:aws:s3:::${CostAndUsageBucket}
                  - !Sub arn:aws:s3:::${CostAndUsageBucket}/*
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 'glue:StartCrawler'
                Resource: '*'

  AWSCURCrawler:
    Type: 'AWS::Glue::Crawler'
    Properties:
      Name: AWSCURCrawler-aws-perspective-cost-and-usage
      Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
      Role: !GetAtt AWSCURCrawlerComponentFunction.Arn
      DatabaseName: !Ref AWSCURDatabase
      Targets:
        S3Targets:
          - Path: !Sub 's3://${CostAndUsageBucket}/${CURCrawlerKey}'
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.gz'
              - '**.zip'
      SchemaChangePolicy:
        UpdateBehavior: UPDATE_IN_DATABASE
        DeleteBehavior: DELETE_FROM_DATABASE

  AWSCURInitializer:
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W89
            reason: 'Not applicable'
          - id: W92
            reason: 'Not applicable'
    Type: AWS::Serverless::Function
    DependsOn: AWSCURCrawler
    Properties:
      CodeUri:
        Bucket: !Ref DeploymentBucket
        Key: !Sub ${DeploymentBucketKey}/cur-setup.zip
      Runtime: nodejs16.x
      MemorySize: 1024
      Tracing: Active
      Environment:
        Variables:          
          CURCrawlerKey: !Ref CURCrawlerKey
      Handler: 'index.handler'
      Timeout: 30
      ReservedConcurrentExecutions: 1
      Role: !GetAtt AWSCURCrawlerLambdaExecutor.Arn

  AWSStartCURCrawler:
    Type: 'Custom::AWSStartCURCrawler'
    Properties:
      ServiceToken: !GetAtt AWSCURInitializer.Arn

  AWSS3CUREventLambdaPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !GetAtt AWSCURInitializer.Arn
      Principal: 's3.amazonaws.com'
      SourceAccount: !Ref AWS::AccountId
      SourceArn: !Sub 'arn:${AWS::Partition}:s3:::${CostAndUsageBucket}'

  AWSS3CURLambdaExecutor:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /
      Policies:
        - PolicyName: AWSS3CURLambdaExecutor
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Resource: !Sub 'arn:${AWS::Partition}:logs:*:*:*'
              - Effect: Allow
                Action:
                  - 's3:PutBucketNotification'
                Resource: !Sub 'arn:${AWS::Partition}:s3:::${CostAndUsageBucket}'

  AWSS3CURNotification:
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: W89
            reason: 'Not applicable'
          - id: W92
            reason: 'Not applicable'
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - AWSCURInitializer
      - AWSS3CUREventLambdaPermission
    Properties:
      Code:
        ZipFile: >
          const AWS = require('aws-sdk');
          const response = require('./cfn-response');
          exports.handler = function(event, context, callback) {
            const s3 = new AWS.S3();
            const putConfigRequest = function(notificationConfiguration) {
              return new Promise(function(resolve, reject) {
                s3.putBucketNotificationConfiguration({
                  Bucket: event.ResourceProperties.BucketName,
                  NotificationConfiguration: notificationConfiguration
                }, function(err, data) {
                  if (err) reject({ msg: this.httpResponse.body.toString(), error: err, data: data });
                  else resolve(data);
                });
              });
            };
            const newNotificationConfig = {};
            if (event.RequestType !== 'Delete') {
              newNotificationConfig.LambdaFunctionConfigurations = [{
                Events: [ 's3:ObjectCreated:*' ],
                LambdaFunctionArn: event.ResourceProperties.TargetLambdaArn || 'missing arn',
                Filter: { Key: { FilterRules: [ { Name: 'prefix', Value: 'aws-perspective' }, { Name: 'suffix', Value: '.snappy.parquet' } ] } }
              }];
            }
            putConfigRequest(newNotificationConfig).then(function(result) {
              response.send(event, context, response.SUCCESS, result);
              callback(null, result);
            }).catch(function(error) {
              response.send(event, context, response.FAILED, error);
              console.log(error);
              callback(error);
            });
          };
      Handler: 'index.handler'
      Timeout: 30
      Runtime: nodejs16.x
      ReservedConcurrentExecutions: 1
      Role: !GetAtt AWSS3CURLambdaExecutor.Arn

  AWSPutS3CURNotification:
    Type: 'Custom::AWSPutS3CURNotification'
    Properties:
      ServiceToken: !GetAtt AWSS3CURNotification.Arn
      TargetLambdaArn: !GetAtt AWSCURInitializer.Arn
      BucketName: !Sub '${CostAndUsageBucket}'
      ReportKey: 'perspective_glue_crawler'

  AWSCURReportStatusTable:
    Type: 'AWS::Glue::Table'
    DependsOn: AWSCURDatabase
    Properties:
      DatabaseName: athenacurcfn_aws_perspective_cost_and_usage
      CatalogId: !Ref AWS::AccountId
      TableInput:
        Name: 'cost_and_usage_data_status'
        TableType: 'EXTERNAL_TABLE'
        StorageDescriptor:
          Columns:
            - Name: status
              Type: 'string'
          InputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
          OutputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
          SerdeInfo:
            SerializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
          Location: !Sub 's3://${CostAndUsageBucket}/aws-perspective/aws-perspective-cost-and-usage-${AWS::AccountId}/cost_and_usage_${AWS::AccountId}_data_status/'