Description: "AWS CloudFormation template to create an AWS Elasticsearch Service domain and Kinesis Firehoses to read from the input streams and write to the domain. **WARNING** This template creates an Amazon Elasticsearch domain and a Kinesis Firehose. You will be billed for the AWS resources used if you create a stack from this template."
Parameters:
  NodeType: 
    Description: "The node type to be provisioned for the Elasticsearch cluster"
    Type: String
    Default: "t2.small.elasticsearch"
    AllowedValues: 
      - 
        "t2.small.elasticsearch"
      - 
        "m4.large.elasticsearch"
      - 
        "m4.xlarge.elasticsearch"
      - 
        "c4.large.elasticsearch"
      - 
        "c4.xlarge.elasticsearch"
      - 
        "r4.large.elasticsearch"
      - 
        "r4.xlarge.elasticsearch"
    ConstraintDescription: "must be a valid Elasticsearch node type."
  NodeCount: 
    Description: "The number of nodes in the Elasticsearch cluster."
    Type: Number
    Default: 1
  ElasticsearchAccessIP:
    Description: "The IP or IP range (in CIDR notation) where you want to test Kibana and Elasticsearch APIs from, you will likely want to update the access policies created by the stack to fit your needs.  This IP access configuration will get you started w/o requiring opening your instance to public access"
    Type: String
    Default: None
  EventStreamARN:
     Description: The name of the stream to get Event data from
     Type: String
     Default: None
  JobStreamARN:
    Description: The name of the stream to Job get data from
    Type: String
    Default: None
  MetricStreamARN:
    Description: The name of the stream to Job get data from
    Type: String
    Default: None
  StreamConsumerRoleARN:
     Description: The name of the role to grant firehose access to the input streams
     Type: String
     Default: None  

Mappings:
  SourceCode:
    General:
      S3Bucket: "rodeolabz"
      KeyPrefix: "vodtk/3-mediaconvert-workload-monitoring/pipeline-es"

Resources:
   
  FirehoseBucket:
    Type : "AWS::S3::Bucket"

  FirehoseBucketRole:
    Type: "AWS::IAM::Role"
    Properties:
      RoleName: !Sub "${AWS::StackName}-FirehoseBucketRole"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - 
            Effect: Allow
            Principal:
              Service:
                - firehose.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - 
          PolicyName: !Sub "${AWS::StackName}-FirehoseBucketPolicy"
          PolicyDocument:
            Statement: 
              - Effect: Allow
                Action: 
                  - s3:*
                Resource: 
                  - !Join ["", ["arn:aws:s3:::", !Ref FirehoseBucket]] 
                  - !Join ["", ["arn:aws:s3:::", !Ref FirehoseBucket, "/*"]]
  
  ESDomain:
    Type: "AWS::Elasticsearch::Domain"
    Properties: 
      AccessPolicies: 
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              AWS:
                - !Sub "${AWS::AccountId}"
            Action:
              - es:*
            Resource: 
              - !Join ["", ["arn:aws:es:", !Ref "AWS::Region", ":", !Ref "AWS::AccountId", ":domain/", !Sub "${AWS::StackName}-esdomain"]]
          - Effect: Allow
            Principal:  
              AWS: "*"
            Action:
              - es:*
            Condition:
              IpAddress:
                aws:SourceIp: 
                  - !Ref ElasticsearchAccessIP
            Resource:
              - !Join ["", ["arn:aws:es:", !Ref "AWS::Region", ":", !Ref "AWS::AccountId", ":domain/", !Sub "${AWS::StackName}-esdomain/*"]] 
      DomainName: !Sub "${AWS::StackName}-esdomain"
      EBSOptions:
        EBSEnabled: true
        Iops: 0
        VolumeSize: 10
        VolumeType: gp2 
      ElasticsearchClusterConfig:
        DedicatedMasterEnabled: false
        InstanceCount: 
          !Ref NodeCount
        ZoneAwarenessEnabled: false
        InstanceType: 
          !Ref NodeType
      ElasticsearchVersion: 5.5
      SnapshotOptions: 
        AutomatedSnapshotStartHour: 0      
  
  EventStreamToEsFirehose:
    Type: "AWS::KinesisFirehose::DeliveryStream"
    Properties: 
      DeliveryStreamName: !Sub "${AWS::StackName}-EventStreamToEsFirehose"
      DeliveryStreamType: KinesisStreamAsSource
      ElasticsearchDestinationConfiguration:
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 20
        CloudWatchLoggingOptions:
          Enabled: False
        DomainARN: !GetAtt ESDomain.DomainArn
        IndexName: "eventindex"
        IndexRotationPeriod: OneWeek
        RetryOptions:
          DurationInSeconds: 300
        RoleARN: !GetAtt FirehoseDeliveryRole.Arn
        S3BackupMode: FailedDocumentsOnly
        S3Configuration:
          CompressionFormat: UNCOMPRESSED
          BufferingHints: 
            IntervalInSeconds: 60
            SizeInMBs: 50
          BucketARN: !GetAtt FirehoseBucket.Arn
          Prefix: !Sub "${AWS::StackName}/events"
          RoleARN: !GetAtt FirehoseBucketRole.Arn
        TypeName: EventMapping
      KinesisStreamSourceConfiguration:
        KinesisStreamARN: !Ref EventStreamARN
        RoleARN: !Ref StreamConsumerRoleARN
  
  JobStreamToEsFirehose:
    DependsOn: EsConfigure
    Type: "AWS::KinesisFirehose::DeliveryStream"
    Properties: 
      DeliveryStreamName: !Sub "${AWS::StackName}-JobStreamToEsFirehose"
      DeliveryStreamType: KinesisStreamAsSource
      ElasticsearchDestinationConfiguration:
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 20
        CloudWatchLoggingOptions:
          Enabled: False
        DomainARN: !GetAtt ESDomain.DomainArn
        IndexName: "jobindex"
        IndexRotationPeriod: OneWeek
        RetryOptions:
          DurationInSeconds: 300
        RoleARN: !GetAtt FirehoseDeliveryRole.Arn
        S3BackupMode: FailedDocumentsOnly
        S3Configuration:
          CompressionFormat: UNCOMPRESSED
          BufferingHints: 
            IntervalInSeconds: 60
            SizeInMBs: 50
          BucketARN: !GetAtt FirehoseBucket.Arn
          Prefix: !Sub "${AWS::StackName}/jobs"
          RoleARN: !GetAtt FirehoseBucketRole.Arn
        TypeName: JobMapping
      KinesisStreamSourceConfiguration:
        KinesisStreamARN: !Ref JobStreamARN
        RoleARN: !Ref StreamConsumerRoleARN
  
  MetricStreamToEsFirehose:
    DependsOn: EsConfigure
    Type: "AWS::KinesisFirehose::DeliveryStream"
    Properties: 
      DeliveryStreamName: !Sub "${AWS::StackName}-MetricStreamToEsFirehose"
      DeliveryStreamType: KinesisStreamAsSource
      ElasticsearchDestinationConfiguration:
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 20
        CloudWatchLoggingOptions:
          Enabled: False
        DomainARN: !GetAtt ESDomain.DomainArn
        IndexName: "metricindex"
        IndexRotationPeriod: OneWeek
        RetryOptions:
          DurationInSeconds: 300
        RoleARN: !GetAtt FirehoseDeliveryRole.Arn
        S3BackupMode: FailedDocumentsOnly
        S3Configuration:
          CompressionFormat: UNCOMPRESSED
          BufferingHints: 
            IntervalInSeconds: 60
            SizeInMBs: 50
          BucketARN: !GetAtt FirehoseBucket.Arn
          Prefix: !Sub "${AWS::StackName}/metrics"
          RoleARN: !GetAtt FirehoseBucketRole.Arn
        TypeName: MetricMapping
      KinesisStreamSourceConfiguration:
        KinesisStreamARN: !Ref MetricStreamARN
        RoleARN: !Ref StreamConsumerRoleARN

  FirehoseDeliveryRole:
    Type: "AWS::IAM::Role"
    Properties:
      RoleName: !Sub "${AWS::StackName}-FirehoseDeliveryRole"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - 
            Effect: Allow
            Principal:
              Service:
                - firehose.amazonaws.com
            Action:
              - sts:AssumeRole
            # Condition: 
            #   StringEquals: 
            #     sts:ExternalId:
            #       !Sub "${AWS::AccountId}"
      Policies:
        - 
          PolicyName: !Sub "${AWS::StackName}-FirehoseDeliveryPolicy"
          PolicyDocument:
            Statement: 
              - Effect: Allow
                Action: 
                  - s3:AbortMultipartUpload
                  - s3:GetBucketLocation
                  - s3:GetObject
                  - s3:ListBucket
                  - s3:ListBucketMultipartUploads
                  - s3:PutObject
                Resource: 
                  - !Join ["", ["arn:aws:s3:::", !Ref FirehoseBucket]] 
                  - !Join ["", ["arn:aws:s3:::", !Ref FirehoseBucket, "/*"]]
              - Effect: Allow
                Action: 
                  - es:ESHttpGet
                Resource: 
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_all/_settings"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_cluster/stats"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/eventindex*/_mapping/*"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_nodes"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_nodes/stats"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_nodes/*/stats"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/_stats"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/*/_stats"
              - Effect: Allow
                Action: 
                  - es:DescribeElasticsearchDomain
                  - es:DescribeElasticsearchDomains
                  - es:DescribeElasticsearchDomainConfig
                  - es:ESHttpPost
                  - es:ESHttpPut
                Resource: 
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/*"
              - Effect: Allow
                Action: 
                  - logs:PutLogEvents
                Resource: 
                  - !Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/*"]]
                  - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/kinesisfirehose/${AWS::StackName}-esdomain/*"
              - Effect: Allow
                Action: 
                  - kinesis:DescribeStream
                  - kinesis:GetShardIterator
                  - kinesis:GetRecords
                Resource: !Sub "arn:aws:kinesis:${AWS::Region}:${AWS::AccountId}:stream/${AWS::StackName}-esdomain"

  CustomResourceRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
        - "arn:aws:iam::aws:policy/AdministratorAccess"
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          -
            Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        -
          PolicyName: !Sub "${AWS::StackName}-custom-resource"
          PolicyDocument:
            Statement:
              -
                Effect: Allow
                Action:
                  - iam:PassRole
                Resource:
                  - !Join ["", ["arn:aws:iam::", Ref: "AWS::AccountId", ":role/*"]]
              -
                Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource:
                  - !Join ["", ["arn:aws:logs:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":log-group:/aws/lambda/*"]]
              -
                Effect: Allow
                Action:
                  - lambda:UpdateFunctionConfiguration
                Resource:
                  - !Join ["", ["arn:aws:lambda:", Ref: "AWS::Region", ":", Ref: "AWS::AccountId", ":", "function:*" ]]
              -
                Effect: Allow
                Action:
                  - es:*
                Resource: 
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain"
                  - !Sub "arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${AWS::StackName}-esdomain/*"

  EsConfigureLambda:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Sub ${AWS::StackName}-EsConfigure
      Description: Collect events, update status, make metrics 
      Handler: index-custom-resource.lambda_handler
      Role: !GetAtt CustomResourceRole.Arn
      Code:
        S3Bucket: !Join ["-", [!FindInMap ["SourceCode", "General", "S3Bucket"], Ref: "AWS::Region"]]
        S3Key: !Join ["/", [!FindInMap ["SourceCode", "General", "KeyPrefix"],  "lambda.zip"]]
      Environment:
        Variables:
          ElasticsearchEndpoint: !GetAtt ESDomain.DomainEndpoint
      Runtime:  python3.6
      Timeout: 120

  EsConfigure:
    Type: Custom::EsConfigure
    Properties:
      ServiceToken: !GetAtt EsConfigureLambda.Arn
      Region: !Ref "AWS::Region"

Outputs: 
  KibanaURL: 
    Description: "Kibana URL"
    Value: !Join ["", ["https://", !GetAtt ESDomain.DomainEndpoint, "/_plugin/kibana/"]]
  ElasticsearchEndpoint: 
    Description: "Elasticsearch domain endpoint"
    Value: !Join ["", ["https://", !GetAtt ESDomain.DomainEndpoint]]
  ElasticsearchDomainARN: 
    Description: "Elasticsearch domain ARN"
    Value: !GetAtt ESDomain.DomainArn
  ElasticsearchDomain: 
    Description: "Elasticsearch domain name"
    Value: !Ref ESDomain