# Following example shows how to create AWS Gateway Load Balancer (GWLB)
# Appliance VPC for centralized architecture using AWS CloudFormation. 

# For architecture details refer to blog:
# https://aws.amazon.com/blogs/networking-and-content-delivery/centralized-inspection-architecture-with-aws-gateway-load-balancer-and-aws-transit-gateway/

# Template uses Amazon Linux 2 instances as appliances behind the GWLB.
# Configures iptables on instances for hairpin setup. The hairpin setup 
# allows traffic coming from GWLB on appliance to be sent back to GWLB.

# iptables configuration is for sample purpose only. It allows all the traffic! Use it for GWLB POC only**

AWSTemplateFormatVersion: "2010-09-09"

Description: >-
  AWS CloudFormation Sample Template For Appliance VPC Setup in Centralized
  Archite For Gateway Load Balancer (GWLB). It creates a hairpin setup using
  iptables. Appliance VPC is created in the same account as Spoke VPCs and
  Transit Gateay.
  
  This template creates:
    - 1 VPC
    - 1 IGW
    - 2 NAT gateways
    - 6 subnets, 3 in each Availability Zone (AZ)
    - 6 route tables, 3 in each AZ
    - 1 Security group
      - port 22 access
      - port 80 access
      - All TCP, UDP and ICMP from VPC CIDR
    - 2 Amazon Linux 2 instances acting as appliance in each AZ. 
      Registered as targets behind a GWLB. Configures iptables on 
      instances for hairpin setup
    - 1 Amazon Linux 2 acting as bastion host
    - 1 GWLB
    - 1 Target group for GWLB 
    - 1 Listner for GWLB
    - 1 VPC endpoint service
    - 2 GWLB endpoints, 1 in each AZ
  
  **WARNING** This template creates one or more Amazon EC2 instances,
  GWLB, GWLB endpints and NAT gateways. You will be billed for the AWS 
  resources used if you create a stack from this template.

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: Network Configuration
        Parameters:
          - VpcCidr
          - AvailabilityZone1
          - TgwAttachSubnet1Cidr
          - ApplianceSubnet1Cidr
          - NatgwSubnet1Cidr
          - AvailabilityZone2
          - TgwAttachSubnet2Cidr
          - ApplianceSubnet2Cidr
          - NatgwSubnet2Cidr
      - Label:
          default: Appliance Configuration
        Parameters:
          - ApplianceInstanceType
          - ApplianceInstanceAmiId          
          - ApplianceInstanceDiskSize
          - KeyPairName
          - AccessLocation          
      - Label:
          default: Gateway Load Balancer Configuration
        Parameters:
          - GwlbName
          - TargetGroupName
          - HealthPort
          - HealthProtocol          
      - Label:
          default: Appliance VPC Route Table Configuration
        Parameters:
          - Spoke1VpcCidr
          - Spoke2VpcCidr

    ParameterLabels:
      VpcCidr:
        default: Appliance VPC - Network CIDR for VPC   
      AvailabilityZone1:
        default: Appliance VPC - Availability Zone 1
      TgwAttachSubnet1Cidr:
        default: Appliance VPC - TGW Attachment Subnet 1 CIDR in AZ1
      ApplianceSubnet1Cidr:
        default: Appliance VPC - Appliance Subnet 1 CIDR in AZ1
      NatgwSubnet1Cidr:
        default: Appliance VPC - NAT GW Subnet 1 CIDR in AZ1
      AvailabilityZone2:
        default: Availability Zone 2        
      TgwAttachSubnet2Cidr:
        default: Appliance VPC - TGW Attachment Subnet 2 CIDR in AZ2
      ApplianceSubnet2Cidr:
        default: Appliance VPC - Appliance Subnet 2 CIDR in AZ2
      NatgwSubnet2Cidr:
        default: Appliance VPC - NAT GW Subnet 2 CIDR in AZ2
      ApplianceInstanceType:
        default: Appliance Instance Type
      ApplianceInstanceAmiId:
        default: Latest AMI ID for appliance (ec2 instance)        
      ApplianceInstanceDiskSize:
        default: Appliance Instance Size in GB
      KeyPairName:
        default: KeyPair required for accessing Appliance instance
      AccessLocation:
        default: Network CIDR to access Appliance instance
      Spoke1VpcCidr:
        default: Spoke1 VPC Network CIDR
      Spoke2VpcCidr:
        default: Spoke2 VPC Network CIDR       
      GwlbName:
        default: Appliance VPC - Gateway Load Balancer Name
      TargetGroupName:
        default: Appliance VPC - Target Group Name
      HealthPort:
        default: Appliance VPC - Health Check Port
      HealthProtocol:
        default: Appliance VPC - Health Check Protocol   

Parameters:
  VpcCidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.0/24
    Description: Appliance VPC - CIDR block for the VPC
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/y
  AvailabilityZone1:
    Description: Appliance VPC - Availability Zone 1
    Type: AWS::EC2::AvailabilityZone::Name
    ConstraintDescription: Valid Availability Zone Id
  TgwAttachSubnet1Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.0/28
    Description: Appliance VPC - TGW Attachment Subnet 1 CIDR in AZ1
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  ApplianceSubnet1Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.16/28
    Description: Appliance VPC - Appliance Subnet 1 CIDR in AZ1
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  NatgwSubnet1Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.32/28
    Description: Appliance VPC - NAT GW Subnet 1 CIDR in AZ1
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28    
  AvailabilityZone2:
    Description: Appliance VPC - Availability Zone 2
    Type: AWS::EC2::AvailabilityZone::Name
    ConstraintDescription: Valid Availability Zone Id
  TgwAttachSubnet2Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.48/28
    Description: Appliance VPC - TGW Attachment Subnet 2 CIDR in AZ2
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  ApplianceSubnet2Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.64/28
    Description: Appliance VPC - Appliance Subnet 2 CIDR in AZ2
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  NatgwSubnet2Cidr:
    AllowedPattern: "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/(1[6-9]|2[0-8]))$"
    Default: 192.168.1.80/28
    Description: Appliance VPC - NAT GW Subnet 2 CIDR in AZ2
    Type: String
    ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-28
  ApplianceInstanceType:
    Description: Select EC2 instance type for Appliance instance. Default is set to t2.micro
    Default: t2.micro
    AllowedValues:
      - t2.micro
    Type: String
  ApplianceInstanceAmiId:
    Description: EC2 Instance AMI ID retrieved using SSM
    Type: String
  ApplianceInstanceDiskSize:
    Description: Appliance instance disk size in GB. Default is set to 8GB
    Default: 8
    AllowedValues: [8]
    Type: Number
    ConstraintDescription: Should be a valid instance size in GB
  KeyPairName:
    Description: EC2 KeyPair required for accessing EC2 instance
    Type: AWS::EC2::KeyPair::KeyName
    ConstraintDescription: Must be the name of an existing EC2 KeyPair
  AccessLocation:
    Description: >-
      Enter desired Network CIDR to allow traffic to appliance. Default is set to
      access from anywhere and it is not recommended
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    MinLength: "9"
    MaxLength: "18"
    Default: 0.0.0.0/0
    Type: String
    ConstraintDescription: Must be a valid Network CIDR of the form x.x.x.x/y
  Spoke1VpcCidr:
    Description: Enter Spoke1 VPC Network CIDR required for routing desired traffic to Spoke1.
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    Default: 10.0.0.0/24
    Type: String
    ConstraintDescription: Must be a valid Network CIDR of the form x.x.x.x/y
  Spoke2VpcCidr:
    Description: Enter Spoke2 VPC Network CIDR required for routing desired traffic to Spoke2.
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    Default: 10.0.1.0/24
    Type: String
    ConstraintDescription: Must be a valid Network CIDR of the form x.x.x.x/y  
  GwlbName:
    Description: >-
      Gateway Load Balancer name. This name must be unique within your AWS 
      account and can have a maximum of 32 alphanumeric characters and 
      hyphens. A name cannot begin or end with a hyphen.
    Type: String
    Default: gwlb1
    ConstraintDescription: Must be a valid GWLB Name
  TargetGroupName:
    Description: Target Group Name
    Type: String
    Default: gwlb1-tg1
    ConstraintDescription: Must be a valid target group name
  HealthProtocol:
    Description: >-
      The protocol GWLB uses when performing health checks on targets.
      Default is HTTP.
    Type: String
    Default: HTTP
    AllowedValues: ['TCP', 'HTTP', 'HTTPS']
    ConstraintDescription: Must be a valid health check protocol
  HealthPort:
    Description: >- 
      The port the load balancer uses when performing health checks
      on targets. Default is 80.
    Type: String
    Default: '80'
    ConstraintDescription: Must be a valid health check port    

Resources:

# Create VPC, IGW and associate IGW with VPC:

  Vpc:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: !Ref VpcCidr
      EnableDnsSupport: "true"
      EnableDnsHostnames: "true"
      InstanceTenancy: default
      Tags:
        - Key: Name
          Value: !Join
            - ""
            - - !Ref AWS::StackName
              - "-vpc"
  
  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
        - Key: Name
          Value: !Join
            - ""
            - - !Ref AWS::StackName
              - "-igw"
  
  AttachInternetGateway:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref Vpc
      InternetGatewayId: !Ref InternetGateway

# Create Subnets:

  # AZ1:
  TgwAttachSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone1
      CidrBlock: !Ref TgwAttachSubnet1Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-tgw-attach-subnet-1"

  ApplianceSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone1
      CidrBlock: !Ref ApplianceSubnet1Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-applinace-subnet-1"

  NatgwSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone1
      CidrBlock: !Ref NatgwSubnet1Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-natgw-subnet-1"
  
  # AZ2:
  TgwAttachSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone2
      CidrBlock: !Ref TgwAttachSubnet2Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-tgw-attach-subnet-2"

  ApplianceSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone2
      CidrBlock: !Ref ApplianceSubnet2Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-subnet-2"

  NatgwSubnet2:
    Type: AWS::EC2::Subnet
    Properties:
      AvailabilityZone: !Ref AvailabilityZone2
      CidrBlock: !Ref NatgwSubnet2Cidr
      VpcId: !Ref Vpc
      MapPublicIpOnLaunch: "true"
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-natgw-subnet-2"

# Create NAT Gateways:

  # AZ1:
  Eip1:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc

  Natgw1:
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt
        - Eip1
        - AllocationId
      SubnetId: !Ref NatgwSubnet1
      Tags:
      - Key: Name
        Value: !Sub "${AWS::StackName}-natgw-1"

  # AZ2:
  Eip2:
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc

  Natgw2:
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt
        - Eip2
        - AllocationId
      SubnetId: !Ref NatgwSubnet2
      Tags:
      - Key: Name
        Value: !Sub "${AWS::StackName}-natgw-2"

# Create Route Tables: 

  # AZ1:
  TgwAttachRouteTable1:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-tgw-attach-rtb-1"

  ApplianceRouteTable1:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-rtb-1"

  NatGwRouteTable1:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-natgw-rtb-1"

  # AZ2:
  TgwAttachRouteTable2:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-tgw-attach-rtb-2"

  ApplianceRouteTable2:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-rtb-2"

  NatGwRouteTable2:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref Vpc
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-natgw-rtb-2"

# Associate Subnets with appropriate Route Tables:

  # AZ1:
  TgwAttachSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref TgwAttachSubnet1
      RouteTableId: !Ref TgwAttachRouteTable1
  
  ApplianceSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref ApplianceSubnet1
      RouteTableId: !Ref ApplianceRouteTable1

  NatgwSubnet1RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref NatgwSubnet1
      RouteTableId: !Ref NatGwRouteTable1

  # AZ2:
  TgwAttachSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref TgwAttachSubnet2
      RouteTableId: !Ref TgwAttachRouteTable2
  
  ApplianceSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref ApplianceSubnet2
      RouteTableId: !Ref ApplianceRouteTable2

  NatgwSubnet2RouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref NatgwSubnet2
      RouteTableId: !Ref NatGwRouteTable2

# Create Security Group:

  BastionSg:
    Type: AWS::EC2::SecurityGroup
    Properties:
      VpcId: !Ref Vpc
      GroupName: !Sub "${AWS::StackName}-bastion-sg"
      GroupDescription: >-
        Access to bastion instance: allow SSH and ICMP access from appropriate location.
        Allow all traffic from VPC CIDR
      SecurityGroupIngress:
        - CidrIp: !Ref AccessLocation
          IpProtocol: tcp
          FromPort: 22
          ToPort: 22
        - CidrIp: !Ref AccessLocation
          IpProtocol: ICMP
          FromPort: -1
          ToPort: -1
        - CidrIp: !Ref VpcCidr
          IpProtocol: "-1"
          FromPort: -1
          ToPort: -1          
      SecurityGroupEgress:
        - CidrIp: 0.0.0.0/0
          IpProtocol: "-1"
          FromPort: -1
          ToPort: -1
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-bastion-sg"

  ApplianceSg:
    Type: AWS::EC2::SecurityGroup
    Properties:
      VpcId: !Ref Vpc
      GroupName: !Sub "${AWS::StackName}-appliance-sg"
      GroupDescription: >-
        Access to Appliance instance: allow SSH and ICMP access from Bastion SG.
        Allow all traffic from VPC CIDR
      SecurityGroupIngress:
        - SourceSecurityGroupId: !GetAtt BastionSg.GroupId
          IpProtocol: tcp
          FromPort: 22
          ToPort: 22
        - SourceSecurityGroupId: !GetAtt BastionSg.GroupId
          IpProtocol: ICMP
          FromPort: -1
          ToPort: -1
        - SourceSecurityGroupId: !GetAtt BastionSg.GroupId
          IpProtocol: tcp
          FromPort: 80
          ToPort: 80
        - CidrIp: !Ref VpcCidr
          IpProtocol: "-1"
          FromPort: -1
          ToPort: -1          
      SecurityGroupEgress:
        - CidrIp: 0.0.0.0/0
          IpProtocol: "-1"
          FromPort: -1
          ToPort: -1
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-sg"

# Create Appliances and related resources:

  ApplianceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: /

  AppliancePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: ApplianceInstanceAccess
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - ec2:DescribeNetworkInterfaces
            Resource: '*'
      Roles:
        - !Ref ApplianceRole

  ApplianceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - !Ref ApplianceRole

  Appliance1:
    DependsOn: 
      - Gwlb
      - ApplianceRtb1Natgw1Route
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !Ref ApplianceInstanceAmiId
      KeyName: !Ref KeyPairName
      InstanceType: !Ref ApplianceInstanceType
      IamInstanceProfile: !Ref ApplianceProfile
      SecurityGroupIds:
        - !Ref ApplianceSg
      SubnetId: !Ref ApplianceSubnet1
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: !Ref ApplianceInstanceDiskSize
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-1"
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash -ex
          
          # Install packages:
          yum update -y;
          yum install jq -y;
          yum install httpd -y;
          yum install htop -y;
          yum install iptables-services -y;
          
          # Enable IP Forwarding and persist across reboot:
          # sysctl -w net.ipv4.ip_forward=1;
          echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/00-defaults.conf
          sysctl -p /etc/sysctl.d/00-defaults.conf          

          # Configure hostname:
          hostnamectl set-hostname ${AWS::StackName}-appliance1;
          
          # Configure SSH client alive interval for ssh session timeout:
          echo 'ClientAliveInterval 60' | sudo tee --append /etc/ssh/sshd_config;
          service sshd restart;
          
          # Set dark background for vim:
          touch /home/ec2-user/.vimrc;
          echo "set background=dark" >> /home/ec2-user/.vimrc;

          # Define variables:
          curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document > /home/ec2-user/iid;
          export instance_interface=$(curl --silent http://169.254.169.254/latest/meta-data/network/interfaces/macs/);
          export instance_vpcid=$(curl --silent http://169.254.169.254/latest/meta-data/network/interfaces/macs/$instance_interface/vpc-id);
          export instance_az=$(cat /home/ec2-user/iid |grep 'availability' | awk -F': ' '{print $2}' | awk -F',' '{print $1}');
          export instance_ip=$(cat /home/ec2-user/iid |grep 'privateIp' | awk -F': ' '{print $2}' | awk -F',' '{print $1}' | awk -F'"' '{print$2}');
          export instance_region=$(cat /home/ec2-user/iid |grep 'region' | awk -F': ' '{print $2}' | awk -F',' '{print $1}' | awk -F'"' '{print$2}');
          export gwlb_ip=$(aws --region $instance_region ec2 describe-network-interfaces --filters Name=vpc-id,Values=$instance_vpcid | jq ' .NetworkInterfaces[] | select(.AvailabilityZone=='$instance_az') | select(.InterfaceType=="gateway_load_balancer") |.PrivateIpAddress' -r);

          # Start http and configure index.html:
          systemctl start httpd;
          touch /var/www/html/index.html;
          echo "<html>" >> /var/www/html/index.html
          echo "  <head>" >> /var/www/html/index.html
          echo "    <title>Gateway Load Balancer POC</title>" >> /var/www/html/index.html
          echo "    <meta http-equiv='Content-Type' content='text/html; charset=ISO-8859-1'>" >> /var/www/html/index.html
          echo "  </head>" >> /var/www/html/index.html
          echo "  <body>" >> /var/www/html/index.html
          echo "    <h1>Welcome to Gateway Load Balancer POC:</h1>" >> /var/www/html/index.html
          echo "    <h2>This is appliance running in $instance_az. Happy testing!</h2>" >> /var/www/html/index.html
          echo "  </body>" >> /var/www/html/index.html
          echo "</html>" >> /var/www/html/index.html

          # Start and configure iptables:
          systemctl enable iptables;
          systemctl start iptables;
          
          # Configuration below allows allows all traffic:
          # Set the default policies for each of the built-in chains to ACCEPT:
          iptables -P INPUT ACCEPT;
          iptables -P FORWARD ACCEPT;
          iptables -P OUTPUT ACCEPT;

          # Flush the nat and mangle tables, flush all chains (-F), and delete all non-default chains (-X):
          iptables -t nat -F;
          iptables -t mangle -F;
          iptables -F;
          iptables -X;

          # Configure nat table to hairpin traffic back to GWLB:
          iptables -t nat -A PREROUTING -p udp -s $gwlb_ip -d $instance_ip -i eth0 -j DNAT --to-destination $gwlb_ip:6081;
          iptables -t nat -A POSTROUTING -p udp --dport 6081 -s $gwlb_ip -d $gwlb_ip -o eth0 -j MASQUERADE;

          # Save iptables:
          service iptables save;

  Appliance2:
    DependsOn: 
      - Gwlb
      - ApplianceRtb2Natgw2Route  
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !Ref ApplianceInstanceAmiId
      KeyName: !Ref KeyPairName
      InstanceType: !Ref ApplianceInstanceType
      IamInstanceProfile: !Ref ApplianceProfile
      SecurityGroupIds:
        - !Ref ApplianceSg
      SubnetId: !Ref ApplianceSubnet2
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: !Ref ApplianceInstanceDiskSize
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-appliance-2"
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash -ex
          
          # Install packages:
          yum update -y;
          yum install jq -y;
          yum install httpd -y;
          yum install htop -y;
          yum install iptables-services -y;
          
          # Enable IP Forwarding and persist across reboot:
          # sysctl -w net.ipv4.ip_forward=1;
          echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.d/00-defaults.conf
          sysctl -p /etc/sysctl.d/00-defaults.conf

          # Configure hostname:
          hostnamectl set-hostname ${AWS::StackName}-appliance2;
          
          # Configure SSH client alive interval for ssh session timeout:
          echo 'ClientAliveInterval 60' | sudo tee --append /etc/ssh/sshd_config;
          service sshd restart;
          
          # Set dark background for vim:
          touch /home/ec2-user/.vimrc;
          echo "set background=dark" >> /home/ec2-user/.vimrc;

          # Define variables:
          curl --silent http://169.254.169.254/latest/dynamic/instance-identity/document > /home/ec2-user/iid;
          export instance_interface=$(curl --silent http://169.254.169.254/latest/meta-data/network/interfaces/macs/);
          export instance_vpcid=$(curl --silent http://169.254.169.254/latest/meta-data/network/interfaces/macs/$instance_interface/vpc-id);
          export instance_az=$(cat /home/ec2-user/iid |grep 'availability' | awk -F': ' '{print $2}' | awk -F',' '{print $1}');
          export instance_ip=$(cat /home/ec2-user/iid |grep 'privateIp' | awk -F': ' '{print $2}' | awk -F',' '{print $1}' | awk -F'"' '{print$2}');
          export instance_region=$(cat /home/ec2-user/iid |grep 'region' | awk -F': ' '{print $2}' | awk -F',' '{print $1}' | awk -F'"' '{print$2}');
          export gwlb_ip=$(aws --region $instance_region ec2 describe-network-interfaces --filters Name=vpc-id,Values=$instance_vpcid | jq ' .NetworkInterfaces[] | select(.AvailabilityZone=='$instance_az') | select(.InterfaceType=="gateway_load_balancer") |.PrivateIpAddress' -r);

          # Start httpd and configure index.html:
          systemctl start httpd;
          touch /var/www/html/index.html;
          echo "<html>" >> /var/www/html/index.html
          echo "  <head>" >> /var/www/html/index.html
          echo "    <title>Gateway Load Balancer POC</title>" >> /var/www/html/index.html
          echo "    <meta http-equiv='Content-Type' content='text/html; charset=ISO-8859-1'>" >> /var/www/html/index.html
          echo "  </head>" >> /var/www/html/index.html
          echo "  <body>" >> /var/www/html/index.html
          echo "    <h1>Welcome to Gateway Load Balancer POC:</h1>" >> /var/www/html/index.html
          echo "    <h2>This is appliance running in $instance_az. Happy testing!</h2>" >> /var/www/html/index.html
          echo "  </body>" >> /var/www/html/index.html
          echo "</html>" >> /var/www/html/index.html

          # Start and configure iptables:
          systemctl enable iptables;
          systemctl start iptables;

          # Configuration below allows allows all traffic:
          # Set the default policies for each of the built-in chains to ACCEPT:
          iptables -P INPUT ACCEPT;
          iptables -P FORWARD ACCEPT;
          iptables -P OUTPUT ACCEPT;

          # Flush the nat and mangle tables, flush all chains (-F), and delete all non-default chains (-X):
          iptables -t nat -F;
          iptables -t mangle -F;
          iptables -F;
          iptables -X;

          # Configure nat table to hairpin traffic back to GWLB:
          iptables -t nat -A PREROUTING -p udp -s $gwlb_ip -d $instance_ip -i eth0 -j DNAT --to-destination $gwlb_ip:6081;
          iptables -t nat -A POSTROUTING -p udp --dport 6081 -s $gwlb_ip -d $gwlb_ip -o eth0 -j MASQUERADE;

          # Save iptables:
          service iptables save;

# Create Bastion Host:
# Bastion host is being created in one AZ (AZ1) only.
# As required, you can create 2nd bastion host in AZ2.

  BastionHost:
    DependsOn: NatgwRtb1IgwRoute
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !Ref ApplianceInstanceAmiId
      KeyName: !Ref KeyPairName
      InstanceType: !Ref ApplianceInstanceType
      SecurityGroupIds:
        - !Ref BastionSg
      SubnetId: !Ref NatgwSubnet1
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: !Ref ApplianceInstanceDiskSize
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-bastion-host"
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash -ex

          # Install packages:
          yum update -y;
          yum install htop -y;

          # Configure hostname:
          hostnamectl set-hostname ${AWS::StackName}-bastion-host;

          # Configure SSH client alive interval for ssh session timeout:
          echo 'ClientAliveInterval 60' | sudo tee --append /etc/ssh/sshd_config;
          service sshd restart;
          
          # Set dark background for vim:
          touch /home/ec2-user/.vimrc;
          echo "set background=dark" >> /home/ec2-user/.vimrc;

# Create Gateway Load Balancer and related resources:

  Gwlb:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Name: !Ref GwlbName
      Type: gateway
      Subnets:
        - !Ref ApplianceSubnet1
        - !Ref ApplianceSubnet2
      Tags:
      - Key: Name
        Value: !Sub "${AWS::StackName}-gwlb-1"

  TargetGroup:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      Name: !Ref TargetGroupName
      Port: 6081
      Protocol: GENEVE
      TargetGroupAttributes:
      - Key: deregistration_delay.timeout_seconds
        Value: 20
      VpcId: !Ref Vpc
      HealthCheckPort: !Ref HealthPort
      HealthCheckProtocol: !Ref HealthProtocol
      TargetType: instance
      Targets:
        - Id: !Ref Appliance1
        - Id: !Ref Appliance2
      Tags:
      - Key: Name
        Value: !Sub "${AWS::StackName}-gwlbtg-1"

  Listener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      DefaultActions:
      - Type: forward
        TargetGroupArn: !Ref TargetGroup
      LoadBalancerArn: !Ref Gwlb

# Create VPC Endpoint Service:

  VpcEndpointService:
    Type: AWS::EC2::VPCEndpointService
    Properties:
      GatewayLoadBalancerArns:
        - !Ref Gwlb
      AcceptanceRequired: false

# Create Lambda Custom Resource to retrieve VPC Endpoint Service Name:

  VpceServiceLambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      Policies:
        - PolicyName: root
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource: arn:aws:logs:*:*:*
              - Effect: Allow
                Action:
                  - ec2:DescribeVpcEndpointServiceConfigurations
                  - ec2:DescribeVpcEndpointServicePermissions
                  - ec2:DescribeVpcEndpointServices
                Resource: "*"

  # Lambda creates CloudWatch Log Group.
  # Since CF stack didn't explicitly create the Log Group, Log Group doesn't get deleted when stack is deleted.
  # Hence creating Log Group though the stack for Lambda specific funciton.
  # Their are few things to consider. For more details refer to: https://github.com/aws/serverless-application-model/issues/1216
  VpceServiceLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
        LogGroupName: !Sub /aws/lambda/${AWS::StackName}-service
        RetentionInDays: 1

  VpceServiceName:
    Type: AWS::Lambda::Function
    DependsOn: VpceServiceLogGroup
    Properties:
      FunctionName: !Sub ${AWS::StackName}-service
      Handler: "index.handler"
      Role: !GetAtt VpceServiceLambdaExecutionRole.Arn
      Code:
        ZipFile: |
          import json
          import logging
          import time

          import boto3
          import cfnresponse
          from botocore.exceptions import ClientError

          try:
              ec2 = boto3.client('ec2')
          except ClientError as e:
              logger.error(f"ERROR: failed to connect to EC2 client: {e}")
              sys.exit(1)              

          def handler(event, context):
              logger = logging.getLogger()
              logger.setLevel(logging.INFO)
              logger.info('Received event: {}'.format(json.dumps(event)))

              responseData = {}
              responseStatus = cfnresponse.FAILED

              try:
                  serviceid = event["ResourceProperties"]["VpceServiceId"]                
              except Exception as e:
                  logger.info('Attribute retrival failure: {}'.format(e))

              try:
                  if event["RequestType"] == "Delete":    
                      responseStatus = cfnresponse.SUCCESS
                      cfnresponse.send(event, context, responseStatus, responseData)
              except Exception:
                  logger.exception("Signaling failure to CloudFormation.")
                  cfnresponse.send(event, context, cfnresponse.FAILED, {})

              if event["RequestType"] == "Create":
                  logger.info("Retrieving VPC Endpoint Service Name:")
                  try:
                      response = ec2.describe_vpc_endpoint_service_configurations(
                          Filters=[
                              {
                                  'Name': 'service-id',
                                  'Values': [serviceid]
                              }
                          ]
                      )
                  except Exception as e:
                      logger.info('ec2.describe_vpc_endpoint_service_configurations failure: {}'.format(e))

                  service_name = response['ServiceConfigurations'][0]['ServiceName']

                  time.sleep(120)

                  responseData['ServiceName'] = service_name             
                  responseStatus = cfnresponse.SUCCESS
                  cfnresponse.send(event, context, responseStatus, responseData)
      Runtime: python3.7
      Timeout: 150

  RetrieveVpceServiceName:
    Type: Custom::RetrieveAttributes
    Properties:
      ServiceToken: !GetAtt VpceServiceName.Arn
      VpceServiceId: !Ref VpcEndpointService

# Create Gateway Load Balancer Endpoint:

  GwlbEndpoint1:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      VpcId: !Ref Vpc
      ServiceName: !GetAtt RetrieveVpceServiceName.ServiceName
      VpcEndpointType: GatewayLoadBalancer
      SubnetIds:
        - !Ref ApplianceSubnet1

  GwlbEndpoint2:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      VpcId: !Ref Vpc
      ServiceName: !GetAtt RetrieveVpceServiceName.ServiceName
      VpcEndpointType: GatewayLoadBalancer
      SubnetIds:
        - !Ref ApplianceSubnet2

# Create appropriate routes for appropriate route tables. Routes with TGW as target are added through TGW template.
  # AZ1:
  NatgwRtb1IgwRoute:
    Type: AWS::EC2::Route
    DependsOn: AttachInternetGateway
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      GatewayId: !Ref InternetGateway
      RouteTableId: !Ref NatGwRouteTable1

  NatgwRtb1Gwlbe1Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint1
    Properties:
      DestinationCidrBlock: !Ref Spoke1VpcCidr
      VpcEndpointId: !Ref GwlbEndpoint1
      RouteTableId: !Ref NatGwRouteTable1

  NatgwRtb1Gwlbe2Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint2
    Properties:
      DestinationCidrBlock: !Ref Spoke2VpcCidr
      VpcEndpointId: !Ref GwlbEndpoint1
      RouteTableId: !Ref NatGwRouteTable1

  ApplianceRtb1Natgw1Route:
    Type: AWS::EC2::Route
    DependsOn: Natgw1
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      NatGatewayId: !Ref Natgw1
      RouteTableId: !Ref ApplianceRouteTable1
  
  TgwAttachRtb1Gwlbe1Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint1
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      VpcEndpointId: !Ref GwlbEndpoint1
      RouteTableId: !Ref TgwAttachRouteTable1

  # AZ2:
  NatgwRtb2IgwRoute:
    Type: AWS::EC2::Route
    DependsOn: AttachInternetGateway
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      GatewayId: !Ref InternetGateway
      RouteTableId: !Ref NatGwRouteTable2

  NatgwRtb2Gwlbe1Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint1
    Properties:
      DestinationCidrBlock: !Ref Spoke1VpcCidr
      VpcEndpointId: !Ref GwlbEndpoint2
      RouteTableId: !Ref NatGwRouteTable2

  NatgwRtb2Gwlbe2Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint2
    Properties:
      DestinationCidrBlock: !Ref Spoke2VpcCidr
      VpcEndpointId: !Ref GwlbEndpoint2
      RouteTableId: !Ref NatGwRouteTable2

  ApplianceRtb2Natgw2Route:
    Type: AWS::EC2::Route
    DependsOn: Natgw2
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      NatGatewayId: !Ref Natgw2
      RouteTableId: !Ref ApplianceRouteTable2
  
  TgwAttachRtb2Gwlbe2Route:
    Type: AWS::EC2::Route
    DependsOn: GwlbEndpoint2
    Properties:
      DestinationCidrBlock: !Ref AccessLocation
      VpcEndpointId: !Ref GwlbEndpoint2
      RouteTableId: !Ref TgwAttachRouteTable2

Outputs:
  ApplianceVpcCidr:
    Description: Appliance VPC CIDR
    Value: !Ref VpcCidr
  ApplianceVpcId:
    Description: Appliance VPC ID
    Value: !Ref Vpc
  Appliance1PrivateIp:
    Description: Appliance VPC Appliance1's private IP
    Value: !GetAtt Appliance1.PrivateIp
  Appliance2PrivateIp:
    Description: Appliance VPC Appliance2's private IP
    Value: !GetAtt Appliance2.PrivateIp
  ApplianceBastionHostPublicIp:
    Description: Appliance VPC Bastion Instance Public IP
    Value: !GetAtt BastionHost.PublicIp
  ApplianceTgwAttachSubnet1Id:
    Description: Appliance VPC TgwAttachSubnet1 ID
    Value: !Ref TgwAttachSubnet1
  ApplianceTgwAttachSubnet2Id:
    Description: Appliance VPC TgwAttachSubnet2 ID
    Value: !Ref TgwAttachSubnet2       
  ApplianceRtb1Id:
    Description: Appliance VPC gwlb-rtb1's id
    Value: !Ref ApplianceRouteTable1
  ApplianceRtb2Id:
    Description: Appliance VPC gwlb-rtb2's id
    Value: !Ref ApplianceRouteTable2