Description: CloudFormation template to create Apache Ranger server with Solr
Parameters:
  S3Bucket:
    Description: S3Bucket where artifacts are stored
    Type: String
    Default: aws-bigdata-blog
  S3Key:
    Description: S3Key of the Lambda code
    Type: String
  S3ArtifactBucket:
      Description: S3Bucket where artifacts are stored
      Type: String
      Default: aws-bigdata-blog
      AllowedValues: ["aws-bigdata-blog"]
  S3ArtifactKey:
      Description: S3Key of the Lambda code
      Type: String
      Default: artifacts/aws-blog-emr-ranger
      AllowedValues: ["artifacts/aws-blog-emr-ranger"]
  VPC:
    Description: VPC ID for creating the EMR cluster
    Type: AWS::EC2::VPC::Id
  Subnet1:
    Description: Subnet ID for creating the EMR cluster
    Type: AWS::EC2::Subnet::Id
  Subnet2:
    Description: Subnet ID for creating the EMR cluster
    Type: AWS::EC2::Subnet::Id
  ProjectVersion:
    Default: 3.0
    Description: Project version
    Type: String
    AllowedValues:
      - 3.0
      - beta
      - test
  KeyPairName:
    Description: Name of an existing EC2 KeyPair to enable SSH to the instances
    Type: AWS::EC2::KeyPair::KeyName
  DBHostName:
    Description: HostName of the database
    Type: String
  DBRootPassword:
    Description: Root password of database
    NoEcho: true
    Type: String
  ADAdminUsername:
    Description: Admin username of AD
    Type: String
  ADAdminPassword:
    Description: Admin password for AD
    NoEcho: true
    Type: String
  InstanceType:
    Description: Instance type of the Ranger Server
    Type: String
    Default: r5.xlarge
    AllowedValues:
      - m5.xlarge
      - m5.2xlarge
      - m5.4xlarge
      - r5.xlarge
      - r5.2xlarge
      - r5.4xlarge
    ConstraintDescription: must be a valid EC2 instance type.
  LDAPHostPrivateIP:
    Description: IP Address of the SimpleAD server
    Type: String
  DomainDNSName:
    AllowedPattern: '[a-zA-Z0-9\-]+\..+'
    Default: awsemr.com
    Description: The Active Directory domain that you want to establish the cross-realm
      trust with e.g., awsemr.com
    MaxLength: '25'
    MinLength: '3'
    Type: String
  LDAPSearchBase:
    Description: Base DN SimpleAD server
    Type: String
    Default: dc=awsemr,dc=com
  LDAPBindUserName:
    Description: BindUser SimpleAD server
    Type: String
    Default: binduser
    AllowedValues:
      - binduser
  LDAPBindPassword:
    Description: BindPassword SimpleAD server
    Type: String
    NoEcho: true
    Default: Bind@User123
  rangerVersion:
    Description: RangerVersion
    Type: String
    Default: '2.0'
    AllowedValues:
      - '2.0'
  AttachAdditionalSourcePrefixToSG:
    Description: Attaches additional sources to EMR Master SG
    Default: false
    Type: String
    AllowedValues: [true, false]
  CIDRAccessToPrivateSubnetResources:
    Description: IP address range (in CIDR notation) of the client that will be allowed
      to connect to the cluster using SSH e.g., 203.0.113.5/32
    AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})
    Type: String
    MinLength: '9'
    MaxLength: '18'
    Default: 10.0.0.0/16
    ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x
  AdditionalSourcePrefixToSG:
    Description: Sources that are allowd to access the Ranger Instance. Should be a source prefix
      e.g., pl-xxx
    Type: String
  AmazonLinux2ImageId:
    Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2'
  CreateTLSCerts:
    Description: This flag controls creation of certs used for Ranger setup. This includes Ranger admin certs, plugin certs/keys used for mutual TLS.  Additionally, if a new regional bucket is created using 'CreateRegionalS3BucketAndCopyScripts', certificates required by Elastic MapReduce (EMR) for In-Transit Encryption will also get created.
    Default: false
    Type: String
    AllowedValues: [ true, false ]
  IsRegionalBucketCreated:
    Description: This flag indicates if a new regional bucket was created. Is set to 'true' new certs will be created for In-Transit Encryption and uploaded to the bucket
    Default: false
    Type: String
    AllowedValues: [ true, false ]
#Mappings:
#  AWSInstanceType2Arch:
#    t1.micro:
#      Arch: PV64
#    t2.nano:
#      Arch: HVM64
#    t2.micro:
#      Arch: HVM64
#    t2.small:
#      Arch: HVM64
#    t2.medium:
#      Arch: HVM64
#    t2.large:
#      Arch: HVM64
#    m3.medium:
#      Arch: HVM64
#    m3.large:
#      Arch: HVM64
#    m3.xlarge:
#      Arch: HVM64
#    m3.2xlarge:
#      Arch: HVM64
#    m4.large:
#      Arch: HVM64
#    m4.xlarge:
#      Arch: HVM64
#    m4.2xlarge:
#      Arch: HVM64
#    m4.4xlarge:
#      Arch: HVM64
#    m4.10xlarge:
#      Arch: HVM64
#    c1.medium:
#      Arch: PV64
#    c1.xlarge:
#      Arch: PV64
#    c3.large:
#      Arch: HVM64
#    c3.xlarge:
#      Arch: HVM64
#    c3.2xlarge:
#      Arch: HVM64
#    c3.4xlarge:
#      Arch: HVM64
#    c3.8xlarge:
#      Arch: HVM64
#    c4.large:
#      Arch: HVM64
#    c4.xlarge:
#      Arch: HVM64
#    c4.2xlarge:
#      Arch: HVM64
#    c4.4xlarge:
#      Arch: HVM64
#    c4.8xlarge:
#      Arch: HVM64
#    r3.large:
#      Arch: HVM64
#    r3.xlarge:
#      Arch: HVM64
#    r3.2xlarge:
#      Arch: HVM64
#    r3.4xlarge:
#      Arch: HVM64
#    r3.8xlarge:
#      Arch: HVM64
#    i2.xlarge:
#      Arch: HVM64
#    i2.2xlarge:
#      Arch: HVM64
#    i2.4xlarge:
#      Arch: HVM64
#    i2.8xlarge:
#      Arch: HVM64
#    d2.xlarge:
#      Arch: HVM64
#    d2.2xlarge:
#      Arch: HVM64
#    d2.4xlarge:
#      Arch: HVM64
#    d2.8xlarge:
#      Arch: HVM64
#    hi1.4xlarge:
#      Arch: HVM64
#    hs1.8xlarge:
#      Arch: HVM64
#    cr1.8xlarge:
#      Arch: HVM64
#    cc2.8xlarge:
#      Arch: HVM64
#    r5.xlarge:
#      Arch: HVM64
#    r5.2xlarge:
#      Arch: HVM64
#    r5.4xlarge:
#      Arch: HVM64
#  AWSRegionArch2AMI:
#    us-east-1:
#      PV64: ami-0023040df18933030
#      HVM64: ami-0915e09cc7ceee3ab
#    us-east-2:
#      PV64: ami-0ea2bc03f34d1ada4
#      HVM64: ami-097834fcb3081f51a
#    us-west-2:
#      PV64: ami-00cb981adfcebb519
#      HVM64: ami-01f08ef3e76b957e5
#    us-west-1:
#      PV64: ami-0027eed75be6f3bf4
#      HVM64: ami-014743cb7690ea737
#    eu-west-1:
#      PV64: ami-00b6370b096f24de2
#      HVM64: ami-00890f614e48ce866
#    eu-central-1:
#      PV64: ami-0001160eb97d88825
#      HVM64: ami-03ab4e8f1d88ce614
#    eu-north-1:
#      PV64: ami-0001160eb97d88825
#      HVM64: ami-0031cb7c28e14ea6f
#    ap-northeast-1:
#      PV64: ami-0095079896fca4cca
#      HVM64: ami-0318ecd6d05daa212
#    ap-northeast-2:
#      PV64: NOT_SUPPORTED
#      HVM64: ami-09391a0ad9f9243b6
#    ap-southeast-1:
#      PV64: ami-021f73ba029345fb1
#      HVM64: ami-0dff4318d85149d5d
#    ap-southeast-2:
#      PV64: ami-01c6bf0aeb3c63052
#      HVM64: ami-050e1ec030abb8dde
#    sa-east-1:
#      PV64: ami-0015527da78932f76
#      HVM64: ami-03e1e4abf50e14ded
#    cn-north-1:
#      PV64: ami-7f84361b
#      HVM64: ami-021321e9bc16d5186

Conditions:
  AttachAdditionalSourcePrefixToSG: !Equals [true, !Ref AttachAdditionalSourcePrefixToSG]
  CreateTLSCertsCondition: !Equals [true, !Ref CreateTLSCerts]

Resources:
  ManagedInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ssm.amazonaws.com
            - ec2.amazonaws.com
          Action: sts:AssumeRole
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM
      - arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess
      - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess
      Path: "/"
  AllowSecretsRetrievalPolicy:
    Type: 'AWS::IAM::Policy'
    DependsOn: ManagedInstanceRole
    Properties:
      PolicyName: AllowSecretsRetrievalPolicy
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - secretsmanager:DeleteSecret
              - secretsmanager:CreateSecret
              - secretsmanager:GetSecretValue
              - secretsmanager:ListSecrets
              - secretsmanager:DescribeSecret
            Resource:
              - !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:emr/ranger*']]
      Roles:
        - !Ref ManagedInstanceRole
  AllowSecretsCreationPolicy:
    Condition: CreateTLSCertsCondition
    DependsOn: ManagedInstanceRole
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyName: AllowSecretsCreationPolicy
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - secretsmanager:DeleteSecret
              - secretsmanager:CreateSecret
            Resource:
              - !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:emr/ranger*']]
      Roles:
        - !Ref ManagedInstanceRole
  ManagedInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
        - !Ref ManagedInstanceRole
  sgRangerAdminServer:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Open Up all Ranger Server ports
      VpcId: !Ref 'VPC'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: '6080'
          ToPort: '6080'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '6182'
          ToPort: '6182'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '8983'
          ToPort: '8983'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '8984'
          ToPort: '8984'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '22'
          ToPort: '22'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
  sgRangerAdminServerWithAdditions:
    Type: AWS::EC2::SecurityGroup
    Condition: AttachAdditionalSourcePrefixToSG
    Properties:
      GroupDescription: Open Up all Ranger Server ports
      VpcId: !Ref 'VPC'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: '6080'
          ToPort: '6080'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '6080'
          ToPort: '6080'
          SourcePrefixListId: !Ref AdditionalSourcePrefixToSG
        - IpProtocol: tcp
          FromPort: '6182'
          ToPort: '6182'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '6182'
          ToPort: '6182'
          SourcePrefixListId: !Ref AdditionalSourcePrefixToSG
        - IpProtocol: tcp
          FromPort: '8983'
          ToPort: '8983'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '8983'
          ToPort: '8983'
          SourcePrefixListId: !Ref AdditionalSourcePrefixToSG
        - IpProtocol: tcp
          FromPort: '8984'
          ToPort: '8984'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '8984'
          ToPort: '8984'
          SourcePrefixListId: !Ref AdditionalSourcePrefixToSG
        - IpProtocol: tcp
          FromPort: '22'
          ToPort: '22'
          CidrIp: !Ref CIDRAccessToPrivateSubnetResources
        - IpProtocol: tcp
          FromPort: '22'
          ToPort: '22'
          SourcePrefixListId: !Ref AdditionalSourcePrefixToSG
  RangerLogGroup:
    Type: "AWS::Logs::LogGroup"
    Properties:
      RetentionInDays: 14
      #LogGroupName: rangerlogs-${AWS::StackName}
  RangerLoadBalancer:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Scheme: internal
      Subnets:
      - Ref: Subnet1
      - Ref: Subnet2
      Type: network
  RangerTargetGroupOne:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Port: 6182
      Protocol: TCP
      TargetType: instance
      VpcId: !Ref VPC
  RangerListenerDefault:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref RangerTargetGroupOne
      LoadBalancerArn: !Ref RangerLoadBalancer
      Port: 6182
      Protocol: TCP
  RangerAutoscalingGroup:
    DependsOn: ManagedInstanceProfile
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      LaunchConfigurationName: !Ref 'RangerLaunchConfig'
      MinSize: '1'
      MaxSize: '1'
      VPCZoneIdentifier:
        - !Ref Subnet1
        - !Ref Subnet2
      TargetGroupARNs:
        - !Ref RangerTargetGroupOne
      HealthCheckType: ELB
      HealthCheckGracePeriod: 500
      Tags:
        - Key: Name
          Value: RangerServer
          PropagateAtLaunch: true
  RangerLaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Properties:
      ImageId: !Ref AmazonLinux2ImageId
      SecurityGroups:
        - !If [ AttachAdditionalSourcePrefixToSG, !Ref sgRangerAdminServerWithAdditions, !Ref sgRangerAdminServer ]
      InstanceType: !Ref 'InstanceType'
      IamInstanceProfile: !Ref ManagedInstanceProfile
      KeyName: !Ref 'KeyPairName'
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeType: gp2
            VolumeSize: '200'
            DeleteOnTermination: 'true'
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash
          set -euo pipefail
          set -x
          cd /tmp
          aws s3 cp s3://${S3ArtifactBucket}/${S3ArtifactKey}/${ProjectVersion}/scripts/install-ranger-admin-server.sh .
          aws s3 cp s3://${S3ArtifactBucket}/${S3ArtifactKey}/${ProjectVersion}/scripts/create-tls-certs.sh .

          sudo yum update -y
          sudo yum install -y awslogs
          sudo systemctl enable awslogsd.service
          sudo systemctl restart awslogsd.service
          yum update aws-cfn-bootstrap

          cat <<EOF > /etc/awslogs/awslogs.conf
          [general]
          # Path to the CloudWatch Logs agent's state file. The agent uses this file to maintain
          # client side state across its executions.
          state_file = /var/lib/awslogs/agent-state
          [ranger_admin_setup_log]
          datetime_format = %Y-%m-%d %H:%M:%S,%f
          file = /tmp/create-ranger-server-output.log
          buffer_duration = 500
          log_stream_name = adminsetuplog-{instance_id}
          initial_position = start_of_file
          log_group_name = ${RangerLogGroup}
          [ranger_admin_log]
          datetime_format = %Y-%m-%d %H:%M:%S,%f
          file = /usr/lib/ranger/logs/admin/logs/ranger_admin*
          buffer_duration = 500
          log_stream_name = adminlog-{instance_id}
          initial_position = start_of_file
          log_group_name = ${RangerLogGroup}
          EOF
          if [[ ${CreateTLSCerts} == "true" ]]; then
            echo "Creating TLS certs and uploading to secrets manager"
            chmod +x /tmp/create-tls-certs.sh
            /tmp/create-tls-certs.sh ${AWS::Region} ${IsRegionalBucketCreated} ${S3Bucket} ${S3Key} ${ProjectVersion} > create-tls-certs-output.log 2>&1 || true
          fi

          chmod +x /tmp/install-ranger-admin-server.sh
          /tmp/install-ranger-admin-server.sh "${LDAPHostPrivateIP}" "${LDAPSearchBase}" "${LDAPBindUserName}@${DomainDNSName}" "${LDAPBindPassword}" "${rangerVersion}" "s3://${S3ArtifactBucket}/${S3ArtifactKey}" "${ProjectVersion}" "${DBHostName}" "${DBRootPassword}" "${AWS::Region}" > create-ranger-server-output.log 2>&1
#  myEC2:
#    Type: AWS::EC2::Instance
#    Metadata:
#      AWS::CloudFormation::Init:
#        configSets:
#          ascending:
#            - CloudwatchLogs
#            - RangerServer
#        CloudwatchLogs:
#          files:
#            '/etc/awslogs/awslogs.conf':
#              content: !Sub |
#                [general]
#                # Path to the CloudWatch Logs agent's state file. The agent uses this file to maintain
#                # client side state across its executions.
#                state_file = /var/lib/awslogs/agent-state
#                [ranger_admin_setup_log]
#                datetime_format = %Y-%m-%d %H:%M:%S,%f
#                file = /tmp/create-ranger-server-output.log
#                buffer_duration = 500
#                log_stream_name = adminsetuplog-{instance_id}
#                initial_position = start_of_file
#                log_group_name = ${RangerLogGroup}
#                [ranger_admin_log]
#                datetime_format = %Y-%m-%d %H:%M:%S,%f
#                file = /usr/lib/ranger/logs/admin/logs/ranger_admin*
#                buffer_duration = 500
#                log_stream_name = adminlog-{instance_id}
#                initial_position = start_of_file
#                log_group_name = ${RangerLogGroup}
#          commands:
#            startawslogs:
#              command: !Sub |
#                sudo yum update -y
#                sudo yum install -y awslogs
#                sudo service awslogs restart
#                sudo chkconfig awslogs on
#        RangerServer:
#          commands:
#            installrangerserver:
#              command:
#                !Join ['', ['bash /tmp/install-ranger-admin-server.sh ',
#                            !Ref 'LDAPHostPrivateIP', ' ', !Ref 'LDAPSearchBase', ' ',
#                            !Ref 'LDAPBindUserName', '@',!Ref 'DomainDNSName' , ' ', !Ref 'LDAPBindPassword',
#                            ' ', !Ref 'rangerVersion', ' ', !Join ['', ['s3://', !Ref S3ArtifactBucket,
#                                                                        '/', !Ref S3ArtifactKey]],
#                            ' ', !Ref 'ProjectVersion',
#                            ' ', !Ref 'DBHostName',
#                            ' ', !Ref 'DBRootPassword',
#                            ' ', !Ref "AWS::Region",
#                            '  '," >\
#                                    \ create-ranger-server-output.log \n"]]
#    Properties:
#      SubnetId: !Ref 'Subnet'
#      IamInstanceProfile: !Ref ManagedInstanceProfile
#      SecurityGroupIds:
#        - !If [ AttachAdditionalSourcePrefixToSG, !Ref sgRangerAdminServerWithAdditions, !Ref sgRangerAdminServer ]
#       ImageId: !Ref RangerServerServerAMI
##      ImageId: !FindInMap [AWSRegionArch2AMI, !Ref 'AWS::Region', !FindInMap [AWSInstanceType2Arch,
##                                                                              !Ref 'InstanceType', Arch]]
#      !FindInMap [AWSAMIRegionMap, !Ref AWS::Region, !FindInMap [LinuxAMINameMap, !Ref BastionAMIOS, Code]]
#      InstanceType: !Ref 'InstanceType'
#      KeyName: !Ref 'KeyPairName'
#      BlockDeviceMappings:
#        - DeviceName: /dev/xvda
#          Ebs:
#            VolumeType: gp2
#            VolumeSize: '100'
#            DeleteOnTermination: 'true'
#      UserData: !Base64
#        Fn::Join:
#          - ''
#          - - '#!/bin/bash
#
#            '
#            - 'cd /tmp
#
#            '
#            - 'aws s3 cp '
#            - !Join ['', ['s3://', !Ref S3Bucket, '/', !Ref S3Key, '/', !Ref ProjectVersion]]
#            - '/scripts/install-ranger-admin-server.sh .
#
#            '
#            - 'yum update aws-cfn-bootstrap
#
#            '
#            - '# Install the files and packages from the metadata
#
#            '
#            - '/opt/aws/bin/cfn-init '
#            - '         --stack '
#            - !Ref 'AWS::StackName'
#            - '         --resource myEC2 '
#            - '         --configsets ascending '
#            - '         --region '
#            - !Ref 'AWS::Region'
#            - '
#
#            '
#      Tags:
#        - Key: Name
#          Value: RangerServer
Outputs:
  RangerAdminHost:
    Description: Ranger Admin Host
    Value: !GetAtt RangerLoadBalancer.DNSName
  RangerAdminUrl:
    Description: URL to connect to the Ranger Admin server
    Value: !Join ['', ['https://', !GetAtt RangerLoadBalancer.DNSName , ':', '6182']]