AWSTemplateFormatVersion: "2010-09-09"
Transform: 'AWS::Serverless-2016-10-31'
Description: "Create Redshift cluster (qs-1r18anahd)"
Globals:
  Function:
    Runtime: python3.7

Resources:
    RedshiftCluster:
      Type: AWS::CloudFormation::Stack
      Properties:
        TemplateURL:
          !Sub
          - 'https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}submodules/quickstart-amazon-redshift/templates/redshift.template.yaml'
          - S3Region: !If [ UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion ]
            S3Bucket: !If [ UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName ]
        Parameters:
          MasterUsername: !Ref AdminUsername
          MasterUserPassword: !Ref AdminUserPassword
          VPCID: !Ref VPCID
          Subnet1ID: !Ref Subnet1ID
          Subnet2ID: !Ref Subnet2ID
          RemoteAccessCIDR: !Ref RemoteAccessCIDR
          NumberOfNodes: !Ref ClusterNodeCount
          NodeType: !Ref ClusterNodeType
          DatabaseName: !Ref DbName
          ConcurrencyScaling: "off"

    RedshiftGlueConnectSecurityGroup:
      Type: 'AWS::EC2::SecurityGroup'
      Properties:
        GroupDescription: Security group for Redshift cluster to help Glue connect to the cluster
        VpcId: !Ref VPCID
        SecurityGroupIngress:
          - IpProtocol: tcp
            FromPort: !GetAtt RedshiftCluster.Outputs.RedshiftPort
            ToPort: !GetAtt RedshiftCluster.Outputs.RedshiftPort
            CidrIp: !Ref RemoteAccessCIDR
            Description: 'Redshift Access to VPC CIDR'

    SelfReferencingIngress:
      Type: AWS::EC2::SecurityGroupIngress
      Properties:
        Description: Self referencing security group ingress rule for redshift cluster
        FromPort: -1
        GroupId: !Ref RedshiftGlueConnectSecurityGroup
        IpProtocol: "-1"
        SourceSecurityGroupId: !Ref RedshiftGlueConnectSecurityGroup
        ToPort: -1

    #
    # SecretsManager
    #
    RedshiftSecret:
      Type: 'AWS::SecretsManager::Secret'
      Properties:
        Name: redshift_db_credentials
        Description: Redshift secret
        SecretString: !Sub '{"username":"${AdminUsername}","password":"${AdminUserPassword}"}'

    SecretRedshiftAttachment:
      Type: "AWS::SecretsManager::SecretTargetAttachment"
      Properties:
        SecretId: !Ref RedshiftSecret
        TargetId: !GetAtt RedshiftCluster.Outputs.RedshiftCluster
        TargetType: AWS::Redshift::Cluster

    SecretManagerVpcEndpoint:
      Type: AWS::EC2::VPCEndpoint
      Properties:
        PrivateDnsEnabled: true
        PolicyDocument: '{
           "Version":"2012-10-17",
           "Statement":[{
             "Effect":"Allow",
             "Principal": "*",
             "Action": "*",
             "Resource":"*"
           }]
        }'
        SecurityGroupIds:
          - !Ref LambdaVpcSecurityGroup
        SubnetIds:
          - !Ref Subnet1ID
          - !Ref Subnet2ID
        ServiceName: !Sub com.amazonaws.${AWS::Region}.secretsmanager
        VpcEndpointType: "Interface"
        VpcId: !Ref VPCID

    #
    # Network
    #
    LambdaVpcSecurityGroup:
      Type: 'AWS::EC2::SecurityGroup'
      Properties:
        GroupDescription: Security group for Lambda Function
        VpcId: !Ref VPCID
        SecurityGroupIngress:
          - IpProtocol: tcp
            FromPort: 0
            ToPort: 65535
            CidrIp: !Ref RemoteAccessCIDR
            Description: 'Lambda to VPC CIDR'

    #
    # Http API
    #
    HttpApi:
      Type: AWS::Serverless::HttpApi

    #
    # Dependency Layer
    #
    PostgresLayer:
      Type: 'AWS::Serverless::LayerVersion'
      Properties:
        LayerName: !Sub "api-dependencies-${AWS::Region}"
        Description: Dependencies for sam app
        ContentUri:
          Bucket: !If [ UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName ]
          Key:
            Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/dependencies/postgres/layer.zip'
        CompatibleRuntimes:
          - python3.7
        LicenseInfo: 'Available under the MIT-0 license.'

    #
    # Lambda
    #
    ConsumptionApi:
      Type: 'AWS::Serverless::Function'
      Properties:
        Handler: app.lambda_handler
        CodeUri:
          Bucket: !If [ UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName ]
          Key:
            Fn::Sub: '${QSS3KeyPrefix}assets/functions/packages/redshift_consumption.zip'
        Description: Lambda function to get consumption for certain meter in different aggregation levels (daily, weekly, monthly)
        MemorySize: 256
        Timeout: 180
        Role: !GetAtt 'RunRedshiftLambdaRole.Arn'
        Environment:
          Variables:
            SECRET_NAME: !Ref RedshiftSecret
            Db_schema: !GetAtt [RedshiftCluster, Outputs.RedshiftDatabaseName]
        Layers:
          - !Ref PostgresLayer
        VpcConfig:
          SecurityGroupIds:
            - !Ref LambdaVpcSecurityGroup
          SubnetIds:
            - !Ref Subnet1ID
            - !Ref Subnet2ID
        Events:
          ApiEvent:
            Type: HttpApi
            Properties:
              ApiId: !Ref HttpApi
              Method: GET
              Path: /consumption/{requested_aggregation}/{year}/{meter_id}

    RunRedshiftLambdaRole:
      Type: 'AWS::IAM::Role'
      Metadata:
        cfn-lint:
          config:
            ignore_checks:
              - E9101
            ignore_reasons:
              E9101: Managed policy name
      Properties:
        AssumeRolePolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Principal:
                Service: 'lambda.amazonaws.com'
              Action: 'sts:AssumeRole'
        ManagedPolicyArns:
          - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
          - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole'
        Policies:
          - PolicyName: secre_manager_read_access
            PolicyDocument:
              Version: 2012-10-17
              Statement:
                - Effect: Allow
                  Action:
                    - "secretsmanager:GetSecretValue"
                    - "secretsmanager:DescribeSecret"
                    - "secretsmanager:ListSecrets"
                    - "kms:Decrypt"
                  Resource:
                    - !Sub '${RedshiftSecret}'
                    - !Sub 'arn:${AWS::Partition}:kms:::*'

    #
    # Glue Connection, Job, Trigger
    #
    GlueWorkflow:
      Type: AWS::Glue::Workflow
      Properties:
        Name: !Sub 'meter-data-redshift-pipeline-${AWS::Region}'
        Description: Workflow for copieing meter reads to readshift

    GlueConnection:
      Type: "AWS::Glue::Connection"
      DependsOn:
        - RedshiftCluster
        - IAMRole
      Properties:
        ConnectionInput:
          Description: "A connection to the meter data Redshift cluster"
          ConnectionType: "JDBC"
          PhysicalConnectionRequirements:
            SubnetId: !Ref Subnet1ID
            SecurityGroupIdList:
              - !Ref RedshiftGlueConnectSecurityGroup
            AvailabilityZone: !Select
              - 0
              - !GetAZs
                Ref: 'AWS::Region'
          ConnectionProperties:
            "JDBC_CONNECTION_URL": !Sub "jdbc:redshift://${RedshiftCluster.Outputs.RedshiftClusterEndpoint}/${RedshiftCluster.Outputs.RedshiftDatabaseName}"
            "JDBC_ENFORCE_SSL": "true"
            PASSWORD: !Ref AdminUserPassword
            USERNAME: !Ref AdminUsername
          Name: !Ref ClusterName
        CatalogId: !Ref "AWS::AccountId"

    IAMRole:
      Type: "AWS::IAM::Role"
      Properties:
        Path: "/service-role/"
        AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"glue.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}"
        MaxSessionDuration: 3600
        ManagedPolicyArns:
          - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonS3FullAccess"
          - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSGlueServiceRole"
          - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonRedshiftFullAccess"

    GlueJob4BusinessDailyToRedshift:
      Type: "AWS::Glue::Job"
      Properties:
        Name: !Sub "business_daily_to_redshift-${AWS::Region}"
        Role: !GetAtt IAMRole.Arn
        ExecutionProperty:
          MaxConcurrentRuns: 1
        Command:
          Name: "glueetl"
          ScriptLocation: !Sub "s3://${GlueScriptsS3Bucket}/admin/business_daily_to_redshift.py"
          PythonVersion: "3"
        DefaultArguments:
          --TempDir: !Sub "s3://${GlueTempS3Bucket}/admin"
          --enable-metrics: ""
          --enable-continuous-cloudwatch-log: "true"
          --enable-continuous-log-filter: "true"
          --job-bookmark-option: "job-bookmark-enable"
          --job-language: "python"
          --enable-s3-parquet-optimized-committer: true
          --db_name: !Ref DbName
          --redshift_connection: !Ref GlueConnection
          --temp_workflow_bucket: !Ref TempWorkflowS3Bucket
        Connections:
          Connections:
            - !Ref GlueConnection
        MaxRetries: 0
        #AllocatedCapacity: !Ref DPU
        MaxCapacity: !Ref GlueDPU
        Timeout: 2880
        GlueVersion: "2.0"

    GlueJobImportDemoDataToRedshift:
      Type: "AWS::Glue::Job"
      Properties:
        Name: !Sub "import_demo_data_to_redshift-${AWS::Region}"
        Role: !GetAtt IAMRole.Arn
        ExecutionProperty:
          MaxConcurrentRuns: 1
        Command:
          Name: "glueetl"
          ScriptLocation: !Sub "s3://${GlueScriptsS3Bucket}/admin/import_demo_data_to_redshift.py"
          PythonVersion: "3"
        DefaultArguments:
          --TempDir: !Sub "s3://${GlueTempS3Bucket}/admin"
          --enable-metrics: ""
          --enable-continuous-cloudwatch-log: "true"
          --enable-continuous-log-filter: "true"
          --job-bookmark-option: "job-bookmark-enable"
          --job-language: "python"
          --db_name: !GetAtt [RedshiftCluster, Outputs.RedshiftDatabaseName]
          --redshift_connection: !Ref GlueConnection
          --cis_bucket: !Sub "${BusinessZoneS3Bucket}/cisdata"
          --geo_bucket: !Sub "${BusinessZoneS3Bucket}/geodata"
        Connections:
          Connections:
            - !Ref GlueConnection
        MaxRetries: 0
        Timeout: 2880
        GlueVersion: "2.0"
        #AllocatedCapacity: !Ref DPU
        MaxCapacity: !Ref GlueDPU

    GlueReadshiftLoadTrigger:
      Type: "AWS::Glue::Trigger"
      Properties:
        Name: !Sub "export-to-redshift-${AWS::Region}"
        Type: "EVENT"
        WorkflowName: !Ref GlueWorkflow
        Actions:
          - JobName: !Ref GlueJob4BusinessDailyToRedshift

    InvokeGlueReadshiftWflRole:
      Type: 'AWS::IAM::Role'
      Properties:
        AssumeRolePolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Principal:
                Service: events.amazonaws.com
              Action: sts:AssumeRole
        Policies:
          - PolicyName: Glue-notify-events
            PolicyDocument:
              Version: "2012-10-17"
              Statement:
                - Effect: Allow
                  Action: 'glue:notifyEvent'
                  Resource: !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:workflow/*"

    EventRule:
      Type: AWS::Events::Rule
      Properties:
        Description: "EventRule"
        EventPattern:
          source:
            - "aws.glue"
          detail-type:
            - "Glue Crawler State Change"
          detail:
            state:
              - "Succeeded"
            crawlerName:
              - !Ref GlueCrawlerToTriggerAfter
        State: "ENABLED"
        Targets:
          -  Arn: !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:workflow/${GlueWorkflow}"
             Id: !Ref GlueWorkflow
             RoleArn: !GetAtt InvokeGlueReadshiftWflRole.Arn

Outputs:
  StackName:
    Description: 'Stack name'
    Value: !Sub '${AWS::StackName}'
  RedshiftSecret:
    Description: SecretManager secret reference
    Value: !Ref RedshiftSecret
  ConsumptionApi:
    Description: Prediction function.
    Value: !Ref ConsumptionApi
  RedshiftClusterEndpoint:
    Value: !GetAtt RedshiftCluster.Outputs.RedshiftClusterEndpoint
  RedshiftEndpoint:
    Value: !GetAtt RedshiftCluster.Outputs.RedshiftEndpoint

Conditions:
  UsingDefaultBucket: !Equals [!Ref QSS3BucketName, 'aws-quickstart']

Parameters:
  QSS3BucketName:
    AllowedPattern: '^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$'
    ConstraintDescription: >-
      Quick Start bucket name can include numbers, lowercase letters, uppercase
      letters, and hyphens (-). It cannot start or end with a hyphen (-).
    Default: aws-quickstart
    Description: >-
      S3 bucket name for the Quick Start assets.
      Only change this value if you customize or extend the Quick Start for your own use.
      This string can include numbers, lowercase letters, uppercase letters, and hyphens (-).
      It cannot start or end with a hyphen (-).
    Type: String
  QSS3KeyPrefix:
    AllowedPattern: '^[0-9a-zA-Z-/]*[/]$'
    ConstraintDescription: >-
      Quick Start key prefix can include numbers, lowercase letters, uppercase
      letters, hyphens (-), and forward slash (/) and must terminate in a forward slash.
    Default: quickstart-aws-utility-meter-data-analytics-platform/
    Type: String
    Description: S3 key prefix for the Quick Start assets. Quick Start key prefix
      can include numbers, lowercase letters, uppercase letters, hyphens (-), and
      forward slash (/).
  AdminUsername:
    Type: String

  AdminUserPassword:
    Type: String
    NoEcho: True
  
  Subnet1ID:
    Type: AWS::EC2::Subnet::Id
  
  Subnet2ID:
    Type: AWS::EC2::Subnet::Id

  VPCID:
    Type: AWS::EC2::VPC::Id
  
  RemoteAccessCIDR:
    Type: String
  
  ClusterName:
    Type: String
    Default: redshift-cluster-1

  ClusterNodeType:
    Type: String
    Default: ra3.4xlarge

  ClusterNodeCount:
    Type: Number
    Default: 2

  GlueCrawlerToTriggerAfter:
    Type: String

  GlueDPU:
    Type: String
    Description: No. of DPUs for Glue Job
    Default: 10

  GlueScriptsS3Bucket:
    Type: String

  GlueTempS3Bucket:
    Type: String

  BusinessZoneS3Bucket:
    Type: String

  TempWorkflowS3Bucket:
    Type: String

  QSS3BucketRegion:
    Type: String

  DbName:
    Type: String