AWSTemplateFormatVersion: 2010-09-09
Description: Creates MSHPCPACK Auto Scaling group for HPC worker nodes (qs-1s4u7q650)
Metadata:
  cfn-lint:
    config:
      ignore_checks:
        - E9101
        - W9006
  'AWS::CloudFormation::Interface':
    ParameterGroups:
      - Label:
          default: Network Configuration
        Parameters:
          #- ThirdAZ
          #- VPCID
          - PrivateSubnet1ID
          #- PrivateSubnet2ID
          #- PrivateSubnet3ID
          - WorkerSGID
      - Label:
          default: Amazon EC2 Configuration
        Parameters:
          - WorkerInstanceType
          - WorkerAMI
          - KeyPairName
          - ASGName
          - HeadNodeNetBIOSName
          #- MasterServerName
      - Label:
          default: Active directory configuration
        Parameters:
          - DomainDNSName
          - DomainNetBIOSName
          - ADUserSecrets
          - DomainJoined
      - Label:
          default: HPC configuratiopn
        Parameters:
          - HTCoreCount
#          - ThreadsPerCore
      - Label:
          default: Certificate Configuration
        Parameters:
          - CertS3Bucket
          - CertS3Key
      - Label:
          default: Quick Start Configuration
        Parameters:
          - QSS3BucketName
          - QSS3BucketRegion
          - QSS3KeyPrefix
    ParameterLabels:
      DomainJoined:
        default: Join domain
      HTCoreCount:
        default: Core count
#      ThreadsPerCore:
#       default: Threads per core
      HeadNodeNetBIOSName:
        default: Head node NetBIOS name
      WorkerSGID:
        default: Worker node security group ID
      ASGName:
        default: Auto scaling group name
      WorkerAMI:
        default: Worker AMI
      WorkerInstanceType:
        default: Worker instance type
      CertS3Bucket:
        default: Certificate S3 bucket
      CertS3Key:
        default: Certificate S3 key
      ADUserSecrets:
        default: Domain Admin secret
      DomainDNSName:
        default: Domain DNS name
      DomainNetBIOSName:
        default: Domain NetBIOS name
      # DomainMemberSGID:
      #   default: Security Group ID for AD Domain Members
      # DomainNetBIOSName:
      #   default: Domain NetBIOS Name
      KeyPairName:
        default: Key Pair Name
      PrivateSubnet1ID:
        default: Private Subnet 1 ID
      # PrivateSubnet2ID:
      #   default: Private Subnet 2 ID
      QSS3BucketName:
        default: Quick Start S3 bucket name
      QSS3BucketRegion:
        default: Quick Start S3 bucket region
      QSS3KeyPrefix:
        default: Quick Start S3 key prefix
      #PrivateSubnet3ID:
      #  default: Private Subnet 3 ID
      #VPCID:
      #  default: VPC ID
Parameters:
  CertS3Bucket:
    AllowedPattern: '^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$'
    Description: S3 bucket containing the certificate
    Type: String
  CertS3Key:
    Description: S3 key for the certificate
    Type: String
  DomainJoined:
    Description: Join worker nodes to Active Directory
    Type: String
    AllowedValues:
      - 'True'
      - 'False'
    Default: 'False'
  WorkerSGID:
    Description: 'WORKER Security Group '
    Type: 'AWS::EC2::SecurityGroup::Id'
  HeadNodeNetBIOSName:
    AllowedPattern: '[a-zA-Z0-9\-]+'
    Default: GridNode
    Description: >-
      NetBIOS name of the domain (up to 15 characters) for users of earlier
      versions of Windows e.g. EXAMPLE
    MaxLength: '15'
    MinLength: '1'
    Type: String
  ADUserSecrets:
    Description: Domain Admin secret
    Type: String
  DomainDNSName:
    AllowedPattern: '[a-zA-Z0-9\-]+\..+'
    Default: example.com
    Description: Fully qualified domain name (FQDN).
    MaxLength: '255'
    MinLength: '2'
    Type: String
  DomainNetBIOSName:
    AllowedPattern: '[a-zA-Z0-9\-]+'
    Default: EXAMPLE
    Description: NetBIOS name of the domain (up to 15 characters) for users of earlier
      versions of Windows e.g. EXAMPLE
    MaxLength: '15'
    MinLength: '1'
    Type: String
  KeyPairName:
    Description: >-
      Public/private key pairs allow you to securely connect to your instance
      after it launches
    Type: 'AWS::EC2::KeyPair::KeyName'
  HTCoreCount:
    Default: '2'
    Description: This depends on Instance Type. For example To disable HT for c5d.18xlarge, specify 32.
    Type: String
#  ThreadsPerCore:
#    AllowedValues:
#      - '1'
#    Default: '1'
#    Description: This disables HYPERTHREADING
#    Type: String
  # MasterServerName:
  #   AllowedPattern: '[a-zA-Z0-9\-]+'
  #   Default: EC2AMAZ-Q5VLDH3
  #   Description: Specify the profile MasterServerName
  #   MaxLength: '15'
  #   MinLength: '1'
  #   Type: String
  ASGName:
    AllowedPattern: '[a-zA-Z0-9\-]+'
    Default: glink-processor
    Description: AutoScaling Group Name for Worker Node
    MaxLength: '15'
    MinLength: '1'
    Type: String
  PrivateSubnet1ID:
    Description: 'ID of the private subnet 1 in Availability Zone 1 (e.g., subnet-a0246dcd)'
    Type: 'AWS::EC2::Subnet::Id'
  # PrivateSubnet2ID:
  #   Description: 'ID of the private subnet 2 in Availability Zone 2 (e.g., subnet-a0246dcd)'
  #   Type: 'AWS::EC2::Subnet::Id'
  # PrivateSubnet3ID:
  #   Default: ''
  #   Description: >-
  #     (Optional) ID of the optional private subnet 3 in Availability Zone 3
  #     (e.g., subnet-a0246dcd)
  #   Type: String
  QSS3BucketName:
    AllowedPattern: '^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$'
    ConstraintDescription: >-
      Quick Start bucket name can include numbers, lowercase letters, uppercase
      letters, and hyphens (-). It cannot start or end with a hyphen (-).
    Default: aws-quickstart
    Description: >-
      Bucket name to store automation assets. Bucket name can
      include numbers, lowercase letters, uppercase letters, and hyphens (-). It
      cannot start or end with a hyphen (-).
    Type: String
  QSS3BucketRegion:
    Default: 'us-east-1'
    Description: 'AWS Region where the Quick Start S3 bucket (QSS3BucketName) is 
    hosted. Keep the default Region unless you are customizing the template. 
    Changing this Region updates code references to point to a new Quick Start location. 
    When using your own bucket, specify the Region. 
    See https://aws-quickstart.github.io/option1.html.'
    Type: String
  QSS3KeyPrefix:
    AllowedPattern: '^[0-9a-zA-Z-/]*$'
    ConstraintDescription:
      The Quick Start S3 key prefix can include numbers, lowercase letters,
      uppercase letters, hyphens (-), and forward slashes (/). The prefix should
      end with a forward slash (/).
    Default: quickstart-microsoft-hpc/
    Description:
      S3 key prefix that is used to simulate a directory for your copy of the 
      Quick Start assets. Keep the default prefix unless you are customizing 
      the template. Changing this prefix updates code references to point to 
      a new Quick Start location. This prefix can include numbers, lowercase 
      letters, uppercase letters, hyphens (-), and forward slashes (/). End with 
      a forward slash. See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 
      and https://aws-quickstart.github.io/option1.html.
    Type: String
  # VPCID:
  #   Description: 'ID of the VPC (e.g., vpc-0343606e)'
  #   Type: 'AWS::EC2::VPC::Id'
  WorkerAMI:
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: '/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base'
    Description: 'Enter an AMI Id. The default value is Windows Server 2019 Core:
      /aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base.'
  WorkerInstanceType:
    AllowedValues:
      - c5d.large
      - c5d.xlarge
      - c5d.2xlarge
      - c5d.4xlarge
      - c5d.9xlarge
      - c5d.18xlarge
    Default: c5d.xlarge
    Description: Amazon EC2 instance type for Worker NOde
    Type: String
Conditions:
  UsingDefaultBucket: !Equals [!Ref QSS3BucketName, 'aws-quickstart']
  JoinDomain: !Equals [!Ref DomainJoined, 'True']
Resources:
  EventBridgeSSMAutoRole:
    Type: AWS::IAM::Role
    Properties:
      Policies:
        - PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
                - ssm:StartAutomationExecution
              Resource:
                - !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/${RemoveConfigurationDoc}:$DEFAULT'
                - !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/${SetupConfigurationDoc}:$DEFAULT'
            - Effect: Allow
              Action:
                - iam:PassRole
              Resource:
                - !GetAtt ExecutionResourceRole.Arn
              Condition: {"StringLikeIfExists": {"iam:PassedToService": "ssm.amazonaws.com"}}
          PolicyName: "EventBridge_Invoke_SSM_Automation"
      Path: /service-role/
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: 
                - events.amazonaws.com
            Action: sts:AssumeRole
  ExecutionResourceRole:
    Type: AWS::IAM::Role
    Properties:
      Policies:
        - PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - autoscaling:CompleteLifecycleAction
                Resource: !Sub arn:aws:autoscaling:*:${AWS::AccountId}:autoScalingGroup:*:autoScalingGroupName/${ASGName}
          PolicyName: asg-lch-complete
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ssm.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonSSMAutomationRole
      Description: New IAM role to allow SSM access.
  WorkerRole:
    Type: AWS::IAM::Role
    Properties:
      ManagedPolicyArns:
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore'
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/CloudWatchAgentServerPolicy'
        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/AmazonSSMDirectoryServiceAccess'
      Path: /
      AssumeRolePolicyDocument:
        Statement:
          - Action: sts:AssumeRole
            Effect: Allow
            Principal:
              Service:
                  - ec2.amazonaws.com
                  - ssm.amazonaws.com

      Policies:
        - PolicyName: QS-Secrets
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - secretsmanager:GetSecretValue
                  - secretsmanager:DescribeSecret
                Resource: 
                  - !Ref 'ADUserSecrets'
        - PolicyName: WorkerTaggingPolicy
          PolicyDocument:
            Statement:
              - Effect: 'Allow'
                Action:
                  - 'ec2:CreateTags'
                  - 'ec2:DeleteTags'
                  - 'ec2:DescribeTags'
                  - 'ec2:Describe*'
                  - 'ssm:*'
                  - 's3:*'
                  - 'autoscaling:SetInstanceHealth'
                Resource: '*'
  WorkerInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - !Ref 'WorkerRole'
  LaunchConfig:
    Type: "AWS::EC2::LaunchTemplate"
    Properties:
      LaunchTemplateName: !Sub 'GridHelperLT-${AWS::StackName}'
      LaunchTemplateData:
        ImageId: !Ref WorkerAMI
        InstanceType: !Ref WorkerInstanceType
        KeyName: !Ref KeyPairName
        TagSpecifications:
          - ResourceType: "instance"
            Tags:
              - Key: "MasterServerName"
                Value: !Ref HeadNodeNetBIOSName
        SecurityGroupIds:
          -  !Ref WorkerSGID
        IamInstanceProfile:
          Arn: !GetAtt WorkerInstanceProfile.Arn
        CpuOptions:
          CoreCount: !Ref HTCoreCount
          ThreadsPerCore: '1'
        # UserData:
        #   'Fn::Base64': !Sub |
        #       <powershell>
        #       function ConvertTo-HexIP {
        #       <#
        #       .Synopsis
        #       Converts a dotted decimal IP address into a hexadecimal string.
        #       .Description
        #       ConvertTo-HexIP takes a dotted decimal IP and returns a single hexadecimal string value.
        #       .Parameter IPAddress
        #       An IP Address to convert.
        #       #>

        #       [CmdLetBinding()]
        #       param(
        #       [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true)]
        #       [Net.IPAddress]$IPAddress
        #       )

        #       process {
        #       "$($IPAddress.GetAddressBytes() | ForEach-Object { '{0:x2}' -f $_ })" -Replace 's'
        #       }
        #       }

        #       $instanceId = (Invoke-RestMethod -Method Get -Uri http://169.254.169.254/latest/meta-data/instance-id)
        #       $hostIP = (Invoke-RestMethod -Method Get -Uri http://169.254.169.254/latest/meta-data/local-ipv4)
        #       $hostName =  ConvertTo-HexIP($hostIP)
        #       $hostName = "ip-" + $hostName -replace('\s','')
        #       $tags = Get-EC2Tag | Where-Object {$_.ResourceId -eq $instanceId -and $_.Key -eq "MasterServerName"}
        #       $masterName = $tags.Value
        #       $retryCount = 4
        #       $retryDelay = 60
        #       $selfTestPassed = 1

        #       ######## Self Test 1 - Ping Headnode and HPC APP Service #########

        #       $test1Passed = $false
        #       $test1RetryCount = 0

        #       while($test1Passed -eq $false -and $test1RetryCount -lt $retryCount)
        #       {
        #       $test1Result = Test-NetConnection -ComputerName $masterName 

        #       if($test1Result.PingSucceeded -eq $true)
        #       {
        #       $test1Passed = $true
        #       }
        #       else
        #       {
        #       Start-Sleep -Seconds $retryDelay
        #       $test1RetryCount = $test1RetryCount + 1
        #       }
        #       }

        #       ####### Self Test 2 - Test Head Node Service Running #########
        #       $test2Passed = $false
        #       if($test1Passed -eq $true)
        #       {
        #       $GLService = Get-Service -Name Dhcp

        #       if($GLService.Status -ne 'Running')
        #       {

        #       Start-Service -Name Dhcp
        #       $GLService = Get-Service -Name Dhcp
        #       }
        #       if($GLService.Status -eq 'Running')
        #       {
        #       $test2Passed = $true
        #       }
        #       }

        #       ######### Action Based on Tests ####################
        #       New-EC2Tag -Resource $instanceId -Tag @{Key="Name"; Value="$HostName"}
        #       New-EC2Tag -Resource $instanceId -Tag @{Key="Octank_InFarm"; Value="pass"}

        #       if($test1Passed -eq $true -and $test2Passed -eq $true)
        #       {
        #       New-EC2Tag -Resource $instanceId -Tag @{Key="Octank_PassedTest"; Value="Pass"}
        #       }
        #       else
        #       {
        #       New-EC2Tag -Resource $instanceId -Tag @{Key="Octank_PassedTest"; Value="Fail"}
        #       Set-ASInstanceHealth -InstanceId $instanceId -HealthStatus "Unhealthy"
        #       }
        #       #####  MSHPCPACK Install #############
        #       Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
        #       Write-Host "Disabling Windows Firewall"
        #       Get-NetFirewallProfile | Set-NetFirewallProfile -Enabled False

        #       $Admin = ConvertFrom-Json -InputObject (Get-SECSecretValue -SecretId "${ADUserSecrets}").SecretString
        #       $DomainAdminPassword = $Admin.password
              
        #       Write-Host "Disabling IE Enhanced Security Configuration (ESC)..."
        #       $AdminKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A7-37EF-4b3f-8CFC-4F3A74704073}"
        #       $UserKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A8-37EF-4b3f-8CFC-4F3A74704073}"
        #       Set-ItemProperty -Path $AdminKey -Name "IsInstalled" -Value 0
        #       Set-ItemProperty -Path $UserKey -Name "IsInstalled" -Value 0
              
        #       Write-Host "Download HPCPACK Software"
        #       Write-Host "Creating Directory for HPC PACK 2019 Public Cert"
        #       New-Item -Path C:\ms-hpcpack2019\ -ItemType directory -Force
        #       $url = "https://download.microsoft.com/download/c/e/b/ceb6122c-1006-44f8-99b1-1792f928f155/HPC%20Pack%202019.zip"
        #       $location = "C:\ms-hpcpack2019\HPCPack2019.zip"
        #       $wc = New-Object net.webclient
        #       $wc.Downloadfile($url,$location)
        #       $installdir = "C:\install"
        #       Set-ExecutionPolicy Bypass -Scope Process -Force;
        #       Expand-Archive -Path $location -DestinationPath $installdir
        #       $cerFileName = "C:\ms-hpcpack2019\myhpc.pfx"
        #       $bucket = "${QSS3BucketName}"
        #       Copy-S3Object -BucketName "${CertS3Bucket}" -Key "${CertS3Key}" -LocalFile $cerFileName

        #       $HeadNode = "${HeadNodeNetBIOSName}" + "." + "${DomainDNSName}"
        #       $pw = $DomainAdminPassword
        #       $Admin=[adsi]("WinNT://$env:COMPUTERNAME/Administrator, user")
        #       $Admin.SetPassword($pw)

        #       $setupArg = "-unattend -ComputeNode:$HeadNode -SSLPfxFilePath:$cerFileName -SSLPfxFilePassword:$pw"

        #       $p = Start-Process -FilePath "$installdir\setup.exe" -ArgumentList $setupArg -PassThru -Wait
        #       $p
        #       ## Bring node online
        #       Start-Sleep -s 60  
        #       Add-PSSnapIn Microsoft.HPC -ErrorAction SilentlyContinue

        #       # This node will be a compute node
        #       Assign-HpcNodeTemplate -NodeName $env:COMPUTERNAME -Name "NonDomain ComputeNode Template" -Confirm:$false

        #       # Only use physical cores, and leave the OS assign the processes on the machine
        #       #$cores = (Get-WmiObject Win32_Processor | Measure-Object -Property NumberOfCores -Sum | Select-Object -ExpandProperty Sum)
        #       #Set-HpcNode -Name $env:COMPUTERNAME -SubscribedCores $cores -Affinity:$false

        #       # Bring the node online
        #       Set-HpcNodeState -Name $env:COMPUTERNAME -State online
        #       #######end ###########
        #       </powershell>
        #       <persist>true</persist>
        BlockDeviceMappings:
          - DeviceName: /dev/sdk
            Ebs:
              VolumeSize: 50
          - DeviceName: /dev/sdc
            VirtualName: ephemeral0
  PlacementGroup:
    Type: AWS::EC2::PlacementGroup
    Properties:
      Strategy: cluster
  AutoScalingGroup:
    Type: 'AWS::AutoScaling::AutoScalingGroup'
    DependsOn:
      - LaunchConfig
      - ScaleUpEventBridgeResource
    Properties:
      AutoScalingGroupName: !Ref ASGName
      VPCZoneIdentifier:
        - !Ref 'PrivateSubnet1ID'
      Tags:
        - Key: 'MasterServerName'
          Value: !Ref HeadNodeNetBIOSName
          PropagateAtLaunch: true
      LifecycleHookSpecificationList:
        - LifecycleTransition: autoscaling:EC2_INSTANCE_LAUNCHING
          LifecycleHookName: DomainJoinHook
          DefaultResult: ABANDON
          HeartbeatTimeout: 1200
        - LifecycleTransition: autoscaling:EC2_INSTANCE_TERMINATING
          LifecycleHookName: DomainUnjoinHook
          DefaultResult: ABANDON
          HeartbeatTimeout: 600
      LaunchTemplate:
        LaunchTemplateName: !Sub 'GridHelperLT-${AWS::StackName}'
        Version: '1'
      MinSize: '0'
      MaxSize: '0'
      PlacementGroup: !Ref PlacementGroup
  ScaleUpEventBridgeResource:
    Type: AWS::Events::Rule
    Properties:
      State: 'ENABLED'
      Description: Run configuration document that joins domain.
      EventPattern:
        source:
          - aws.autoscaling
        detail-type:
          - EC2 Instance-launch Lifecycle Action
        detail:
          AutoScalingGroupName:
            - !Ref ASGName
      Targets:
        - Arn: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/${SetupConfigurationDoc}:$DEFAULT'
          Id: Windows-Scale-Out
          RoleArn: !GetAtt EventBridgeSSMAutoRole.Arn
          InputTransformer:
            InputPathsMap:
              InstanceId: $.detail.EC2InstanceId
              ASGName: $.detail.AutoScalingGroupName
              LCHName: $.detail.LifecycleHookName
            InputTemplate: !Sub '{"AutomationAssumeRole":["${ExecutionResourceRole.Arn}"],"InstanceId":[<InstanceId>],"ASGName":[<ASGName>],"LCHName":[<LCHName>],"DomainJoined":["${DomainJoined}"],"ADUserSecrets":["${ADUserSecrets}"], "QSS3BucketName":["${QSS3BucketName}"],"QSS3KeyPrefix":["${QSS3KeyPrefix}"],"URLSuffix":["${AWS::URLSuffix}"], "DomainNetBIOSName":["${DomainNetBIOSName}"], "DomainDNSName":["${DomainDNSName}"], "CertS3Bucket":["${CertS3Bucket}"], "CertS3Key":["${CertS3Key}"], "HeadNodeNetBIOSName":["${HeadNodeNetBIOSName}"]}'
  ScaleDownEventBridgeResource:
    Type: AWS::Events::Rule
    Properties:
      State: !If [JoinDomain, 'ENABLED', 'DISABLED']
      Description: Run removal document that unjoins domain.
      EventPattern:
        source:
          - aws.autoscaling
        detail-type:
          - EC2 Instance-terminate Lifecycle Action
        detail:
          AutoScalingGroupName:
            - !Ref ASGName
      Targets:
        - Arn: !Sub 'arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/${RemoveConfigurationDoc}:$DEFAULT'
          Id: Windows-Scale-In
          RoleArn: !GetAtt EventBridgeSSMAutoRole.Arn
          InputTransformer:
            InputPathsMap:
              InstanceId: $.detail.EC2InstanceId
              ASGName: $.detail.AutoScalingGroupName
              LCHName: $.detail.LifecycleHookName
            InputTemplate: !Sub '{"AutomationAssumeRole":["${ExecutionResourceRole.Arn}"],"InstanceId":[<InstanceId>],"ASGName":[<ASGName>],"LCHName":[<LCHName>],"DomainJoined":["${DomainJoined}"],"ADUserSecrets":["${ADUserSecrets}"], "QSS3BucketName":["${QSS3BucketName}"],"QSS3KeyPrefix":["${QSS3KeyPrefix}"],"URLSuffix":["${AWS::URLSuffix}"], "DomainNetBIOSName":["${DomainNetBIOSName}"], "DomainDNSName":["${DomainDNSName}"]}'
  SetupConfigurationDoc:
    Type: AWS::SSM::Document
    Properties:
      DocumentType: Automation
      Content: 
        schemaVersion: "0.3"
        description: "Configure instances on launch."
        assumeRole: "{{AutomationAssumeRole}}"
        parameters:
          DomainJoined:
            description: "Domain joined"
            type: "String"
          InstanceId:
            description: "ID of the instance."
            type: "String" 
          ASGName:
            description: "Auto Scaling group name."
            type: "String"
          LCHName:
            description: "Life cycle hook name."
            type: "String"
          DomainDNSName: 
            default: "example.com"
            description: "Fully qualified domain name (FQDN) of the forest root domain (for example, example.com)."
            type: "String"
          DomainNetBIOSName:
            default: "example"
            description: "NetBIOS name of the domain (up to 15 characters) for users of earlier versions of Windows (for example, EXAMPLE)."
            type: "String"
          QSS3BucketName:
            default: "aws-quickstart"
            description: "Name of the S3 bucket for your copy of the Quick Start assets. 
      Keep the default name unless you are customizing the template. 
      Changing the name updates code references to point to a new Quick 
      Start location. This name can include numbers, lowercase letters, 
      uppercase letters, and hyphens, but do not start or end with a hyphen (-). 
      See https://aws-quickstart.github.io/option1.html."
            type: "String"
          QSS3KeyPrefix:
            default: "quickstart-microsoft-hpc/"
            description: "S3 key prefix that is used to simulate a directory for your copy of the 
      Quick Start assets. Keep the default prefix unless you are customizing 
      the template. Changing this prefix updates code references to point to 
      a new Quick Start location. This prefix can include numbers, lowercase 
      letters, uppercase letters, hyphens (-), and forward slashes (/). 
      See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 
      and https://aws-quickstart.github.io/option1.html."
            type: "String"
          URLSuffix:
            default: "amazonaws.com"
            description: "AWS URL suffix."
            type: "String"
          AutomationAssumeRole:
            default: ""
            description: "(Optional) The ARN of the role that allows automation to perform the actions on your behalf."
            type: "String" 
          ADUserSecrets:
            default: ""
            description: "AD user secrets"
            type: "String"
          CertS3Bucket:
            default: ""
            description: "Certificate S3 bucket"
            type: "String"
          CertS3Key:
            default: ""
            description: "Certificate S3 key"
            type: "String"
          HeadNodeNetBIOSName:
            default: ""
            description: "Head node NetBIOS name"
            type: "String"
        mainSteps:
          - name: waitUntilInstanceStateRunning
            action: aws:waitForAwsResourceProperty
            timeoutSeconds: 600
            inputs:
              Service: ec2
              Api: DescribeInstanceStatus
              InstanceIds:
                - "{{InstanceId}}"
              PropertySelector: "$.InstanceStatuses[0].InstanceState.Name"
              DesiredValues:
                - running
          - name: assertInstanceStateRunning
            action: aws:assertAwsResourceProperty
            inputs:
              Service: ec2
              Api: DescribeInstanceStatus
              InstanceIds:
                - "{{InstanceId}}"
              PropertySelector: "$.InstanceStatuses[0].InstanceState.Name"
              DesiredValues:
                - running
          - name: "EnableCredSSP"
            action: "aws:runCommand"
            inputs:
              DocumentName: "AWS-RunRemoteScript"
              InstanceIds:
                - "{{InstanceId}}"
              CloudWatchOutputConfig:
                CloudWatchOutputEnabled: "true"
                CloudWatchLogGroupName: !Sub '/aws/Quick_Start/HPC-${AWS::StackName}'
              Parameters:
                sourceType: "S3"
                sourceInfo: !Sub
                  - '{"path": "https://${S3Bucket}.s3.${S3Region}.${AWS::URLSuffix}/${QSS3KeyPrefix}scripts/Enable-CredSSP.ps1"}'
                  - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion]
                    S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName]
                commandLine: "./Enable-CredSSP.ps1"
          - name: DomainJoinBranch
            action: aws:branch
            inputs:
              Choices:
              - Variable: "{{DomainJoined}}"
                StringEquals: "True"
                NextStep: JoinDomain
              - Variable: "{{DomainJoined}}"
                StringEquals: "False"
                NextStep: HPCSetup
          - name: "JoinDomain"
            action: "aws:runCommand"
            inputs:
              DocumentName: "AWS-RunRemoteScript"
              InstanceIds:
                - "{{InstanceId}}"
              CloudWatchOutputConfig:
                CloudWatchOutputEnabled: "true"
                CloudWatchLogGroupName: !Sub '/aws/Quick_Start/HPC-${AWS::StackName}'
              Parameters:
                sourceType: "S3"
                sourceInfo: !Sub
                  - '{"path": "https://${S3Bucket}.s3.${S3Region}.{{URLSuffix}}/{{QSS3KeyPrefix}}scripts/Join-Domain.ps1"}'
                  - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion]
                    S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName]
                commandLine: "./Join-Domain.ps1 -DomainName {{DomainDNSName}} -ADUserSecrets {{ADUserSecrets}}"
          - name: "HPCSetup"
            action: "aws:runCommand"
            inputs:
              DocumentName: "AWS-RunRemoteScript"
              InstanceIds:
                - "{{InstanceId}}"
              CloudWatchOutputConfig:
                CloudWatchOutputEnabled: "true"
                CloudWatchLogGroupName: !Sub '/aws/Quick_Start/HPC-${AWS::StackName}'
              Parameters:
                sourceType: "S3"
                sourceInfo: !Sub
                  - '{"path": "https://${S3Bucket}.s3.${S3Region}.{{URLSuffix}}/{{QSS3KeyPrefix}}scripts/ConfigWorkerNode.ps1"}'
                  - S3Region: !If [UsingDefaultBucket, !Ref 'AWS::Region', !Ref QSS3BucketRegion]
                    S3Bucket: !If [UsingDefaultBucket, !Sub '${QSS3BucketName}-${AWS::Region}', !Ref QSS3BucketName]
                commandLine: "./ConfigWorkerNode.ps1 -ADUserSecrets {{ADUserSecrets}} -QSS3BucketName {{QSS3BucketName}} -CertS3Bucket {{CertS3Bucket}} -CertS3Key {{CertS3Key}} -HeadNodeNetBIOSName {{HeadNodeNetBIOSName}} -DomainDNSName {{DomainDNSName}}"
          - name: "completeHookAction"
            action: aws:executeAwsApi
            isEnd: true
            inputs:
              Service: autoscaling
              Api: CompleteLifecycleAction
              AutoScalingGroupName: "{{ASGName}}"
              InstanceId: "{{InstanceId}}"
              LifecycleActionResult: CONTINUE
              LifecycleHookName: "{{LCHName}}"
          - name: "abandonHookAction"
            action: aws:executeAwsApi
            isEnd: true
            inputs:
              Service: autoscaling
              Api: CompleteLifecycleAction
              AutoScalingGroupName: "{{ASGName}}"
              InstanceId: "{{InstanceId}}"
              LifecycleActionResult: ABANDON
              LifecycleHookName: "{{LCHName}}"
  RemoveConfigurationDoc:
    Type: AWS::SSM::Document
    Properties:
      DocumentType: Automation
      Content:
        schemaVersion: "0.3"
        description: "Remove EC2 Instances from AD domain."
        assumeRole: "{{AutomationAssumeRole}}"
        parameters:
          InstanceId:
            description: "ID of the instance."
            type: "String"
          DomainJoined:
            description: "Domain joined"
            type: "String" 
          ASGName:
            description: "Auto Scaling group name."
            type: "String"
          LCHName:
            description: "Life cycle hook name."
            type: "String"
          AutomationAssumeRole:
            default: ""
            description: "(Optional) The ARN of the role that allows automation to perform the actions on your behalf."
            type: "String"
          URLSuffix:
            default: "amazonaws.com"
            description: "AWS URL suffix."
            type: "String"
          DomainDNSName: 
            default: "example.com"
            description: "Fully qualified domain name (FQDN) of the forest root domain (for example, example.com)."
            type: "String"
          DomainNetBIOSName:
            default: "example"
            description: "NetBIOS name of the domain (up to 15 characters) for users of earlier versions of Windows (for example, EXAMPLE)."
            type: "String"
          QSS3BucketName:
            default: "aws-quickstart"
            description: "Name of the S3 bucket for your copy of the Quick Start assets. 
      Keep the default name unless you are customizing the template. 
      Changing the name updates code references to point to a new Quick 
      Start location. This name can include numbers, lowercase letters, 
      uppercase letters, and hyphens, but do not start or end with a hyphen (-). 
      See https://aws-quickstart.github.io/option1.html."
            type: "String"
          QSS3KeyPrefix:
            default: "quickstart-microsoft-hpc/"
            description: "S3 key prefix that is used to simulate a directory for your copy of the 
      Quick Start assets. Keep the default prefix unless you are customizing 
      the template. Changing this prefix updates code references to point to 
      a new Quick Start location. This prefix can include numbers, lowercase 
      letters, uppercase letters, hyphens (-), and forward slashes (/). 
      See https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html 
      and https://aws-quickstart.github.io/option1.html."
            type: "String"
          ADUserSecrets:
            default: ""
            description: "AD user secrets"
            type: "String"
        mainSteps:
          - name: "RemoveDomain"
            action: "aws:runCommand"
            inputs:
              DocumentName: "AWS-RunRemoteScript"
              InstanceIds:
                - "{{InstanceId}}"
              CloudWatchOutputConfig:
                CloudWatchOutputEnabled: "true"
                CloudWatchLogGroupName: !Sub '/aws/Quick_Start/HPC-${AWS::StackName}'
              Parameters:
                sourceType: "S3"
                sourceInfo:
                  !Sub
                    - '{"path": "https://${S3Bucket}.s3.${S3Region}.{{URLSuffix}}/{{QSS3KeyPrefix}}scripts/Remove-Domain.ps1"}'
                    - S3Bucket: !If
                        - UsingDefaultBucket
                        - !Sub '${QSS3BucketName}-${AWS::Region}'
                        - !Ref QSS3BucketName
                      S3Region: !If 
                        - UsingDefaultBucket
                        - !Ref AWS::Region
                        - !Ref QSS3BucketRegion
                commandLine: "./Remove-Domain.ps1 -DomainNetBIOSName {{DomainNetBIOSName}} -ADUserSecrets {{ADUserSecrets}}"
          - name: "completeHookAction"
            action: aws:executeAwsApi
            isEnd: true
            inputs:
              Service: autoscaling
              Api: CompleteLifecycleAction
              AutoScalingGroupName: "{{ASGName}}"
              InstanceId: "{{InstanceId}}"
              LifecycleActionResult: CONTINUE
              LifecycleHookName: "{{LCHName}}"
          - name: "abandonHookAction"
            action: aws:executeAwsApi
            isEnd: true
            inputs:
              Service: autoscaling
              Api: CompleteLifecycleAction
              AutoScalingGroupName: "{{ASGName}}"
              InstanceId: "{{InstanceId}}"
              LifecycleActionResult: ABANDON
              LifecycleHookName: "{{LCHName}}"