AWSTemplateFormatVersion: '2010-09-09' Description: 'Cloudformation Template DAT312 ACK-RDS-GITOPS-Workshop' Parameters: EKSClusterName: Type: String Description: The desired name of your AWS EKS Cluster. Default: eksclu KubernetesVersion: Description: The Kubernetes version to install Type: String Default: 1.23 AllowedValues: - 1.25 - 1.24 - 1.23 NumWorkerNodes: Type: Number Description: Number of worker nodes to create Default: 2 LoabdBalancerVersion: Description: EKS Load Balancer Controller Version Type: String Default: v2.4.1 SubnetAPublic: Description: SubnetAPublic Type: String Default: subnet-0c40f93bba8c7a898 SubnetAPrivate: Description: SubnetAPrivate Type: String Default: subnet-0c40f93bba8c7a898 SubnetBPrivate: Description: SubnetBPrivate Type: String Default: subnet-0f890153b313f45b6 SubnetCPrivate: Description: SubnetCPrivate Type: String Default: subnet-0ad40ae4a0953a6e6 VPC: Description: VPC ID Type: String Default: vpc-0ebb65d1194794109 OwnerRole: Description: Owner of the EKS cluster Type: String Default: arn:aws:iam IsWorkshopStudioEnv: Type: String Default: "no" Description: Whether this stack is being deployed in a Workshop Studio environment or not. If not sure, leave as default of "no". C9InstanceType: Default: t3.medium Description: Amazon Cloud9 instance type Type: String Conditions: isWorkshopStudio: !Equals [ !Ref IsWorkshopStudioEnv, "yes"] Resources: #============================================================================# # Cloud9 Environment #============================================================================# C9Role: Type: AWS::IAM::Role Properties: Tags: - Key: Environment Value: DAT312 Event AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: - ec2.amazonaws.com - ssm.amazonaws.com Action: - sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/AdministratorAccess Path: "/" C9LambdaExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: "/" Policies: - PolicyName: Fn::Join: - '' - - C9LambdaPolicy- - Ref: AWS::Region PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: arn:aws:logs:*:*:* - Effect: Allow Action: - cloudformation:DescribeStacks - cloudformation:DescribeStackEvents - cloudformation:DescribeStackResource - cloudformation:DescribeStackResources - ec2:DescribeInstances - ec2:AssociateIamInstanceProfile - ec2:ModifyInstanceAttribute - ec2:ReplaceIamInstanceProfileAssociation - iam:ListInstanceProfiles Resource: "*" - Effect: Allow Action: - iam:ListInstanceProfiles - iam:PassRole Resource: "arn:aws:iam::*:role/*" C9BootstrapInstanceLambda: Type: Custom::C9BootstrapInstanceLambda DependsOn: - C9LambdaExecutionRole Properties: Tags: - Key: Environment Value: DAT312 Event ServiceToken: Fn::GetAtt: - C9BootstrapInstanceLambdaFunction - Arn REGION: Ref: AWS::Region StackName: Ref: AWS::StackName EnvironmentId: Ref: C9Instance LabIdeInstanceProfileName: Ref: C9InstanceProfile LabIdeInstanceProfileArn: Fn::GetAtt: - C9InstanceProfile - Arn C9BootstrapInstanceLambdaFunction: Type: AWS::Lambda::Function Properties: Tags: - Key: Environment Value: DAT312 Event Handler: index.lambda_handler Role: Fn::GetAtt: - C9LambdaExecutionRole - Arn Runtime: python3.9 MemorySize: 2056 Timeout: '600' Code: ZipFile: | from __future__ import print_function import boto3 import json import os import time import traceback import cfnresponse import logging def lambda_handler(event, context): print('event: {}'.format(event)) print('context: {}'.format(context)) responseData = {} if event['RequestType'] == 'Create': try: # Open AWS clients ec2 = boto3.client('ec2') # Get the InstanceId of the Cloud9 IDE instance = ec2.describe_instances(Filters=[{'Name': 'tag:Name','Values': ['aws-cloud9-'+event['ResourceProperties']['StackName']+'-'+event['ResourceProperties']['EnvironmentId']]}])['Reservations'][0]['Instances'][0] logging.info('instance: {}'.format(instance)) # Create the IamInstanceProfile request object iam_instance_profile = { 'Arn': event['ResourceProperties']['LabIdeInstanceProfileArn'], 'Name': event['ResourceProperties']['LabIdeInstanceProfileName'] } logging.info('iam_instance_profile: {}'.format(iam_instance_profile)) # Wait for Instance to become ready before adding Role instance_state = instance['State']['Name'] logging.info('instance_state: {}'.format(instance_state)) while instance_state != 'running': time.sleep(5) instance_state = ec2.describe_instances(InstanceIds=[instance['InstanceId']]) logging.info('instance_state: {}'.format(instance_state)) # attach instance profile response = ec2.associate_iam_instance_profile(IamInstanceProfile=iam_instance_profile, InstanceId=instance['InstanceId']) logging.info('response - associate_iam_instance_profile: {}'.format(response)) r_ec2 = boto3.resource('ec2') responseData = {'Success': 'Started bootstrapping for instance: '+instance['InstanceId']} cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, 'CustomResourcePhysicalID') except Exception as e: logging.error(e, exc_info=True) responseData = {'Error': str(traceback.format_exc(e))} cfnresponse.send(event, context, cfnresponse.FAILED, responseData, 'CustomResourcePhysicalID') C9OutputBucket: Type: AWS::S3::Bucket DeletionPolicy: Delete C9Bootstrap: Type: AWS::SSM::Document Properties: Tags: - Key: Environment Value: DAT312 Event DocumentType: Command Content: schemaVersion: '2.2' description: Bootstrap cloud9 for installing redisclient and other required libraries mainSteps: - action: aws:runShellScript name: BootstrapTools inputs: runCommand: - "#!/bin/bash" - su - ec2-user -c " - cd /home/ec2-user/environment - wget https://raw.githubusercontent.com/aws-samples/ack-rds-gitops-workshop/main/scripts/prereq.sh - chmod +x prereq.sh - ./prereq.sh > prereq.log 2>&1 " - echo "Bootstrap completed with return code $?" C9BootstrapAssociation: Type: AWS::SSM::Association Properties: Name: !Ref C9Bootstrap OutputLocation: S3Location: OutputS3BucketName: !Ref C9OutputBucket OutputS3KeyPrefix: bootstrapoutput Targets: - Key: tag:SSMBootstrap Values: - Active C9InstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: "/" Roles: - Ref: C9Role C9Instance: DependsOn: [C9BootstrapAssociation, EKSCluster, WorkerNodegroup] Type: AWS::Cloud9::EnvironmentEC2 Properties: Name: !Sub '${AWS::StackName}' AutomaticStopTimeMinutes: 60 Description: AWS Cloud9 instance for DAT312 event InstanceType: !Ref C9InstanceType ImageId: amazonlinux-2-x86_64 SubnetId: !Ref SubnetAPublic OwnerArn: Fn::If: - isWorkshopStudio - !Sub 'arn:aws:sts::${AWS::AccountId}:assumed-role/WSParticipantRole/Participant' - !Ref "AWS::NoValue" Tags: - Key: SSMBootstrap Value: Active - Key: Environment Value: DAT312 Event #============================================================================# # Creating IAM role for ACK #============================================================================# MemoryDBSLR: Type: 'AWS::IAM::ServiceLinkedRole' Properties: AWSServiceName: memorydb.amazonaws.com Description: Service Control Role for MemoryDB ACKGrants: Type: Custom::ACKGrants DependsOn: - EKSCluster - MemoryDBSLR Properties: ServiceToken: !GetAtt 'ACKGrantsLambda.Arn' eks_cluster_name: !Ref EKSClusterName ACKGrantsRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: lambda.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Path: / Policies: - PolicyName: lambda-createkeypair1 PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - eks:DescribeCluster - iam:CreateRole - iam:DescribeRole - iam:AttachRolePolicy - iam:GetRole - iam:CreatePolicy - ec2:DescribeSubnets - memorydb:CreateSubnetGroup Resource: - '*' ACKGrantsLambda: Type: AWS::Lambda::Function Properties: Description: Creates a keypair and stores private key in SSM parameter store. Handler: index.lambda_handler Runtime: python3.9 Environment: Variables: loadbalancerversion: !Ref LoabdBalancerVersion vpcid: !Ref VPC Role: !GetAtt 'ACKGrantsRole.Arn' Timeout: 300 Code: ZipFile: | import json import boto3 import sys import os import urllib.request import cfnresponse import traceback from botocore.exceptions import ClientError def get_doc(url): try: res = urllib.request.urlopen(urllib.request.Request( url=url, headers={'Accept': 'application/json'}, method='GET'), timeout=5) result = res.read().decode().strip() return result except: return {} def get_arns(url): try: res = urllib.request.urlopen(urllib.request.Request( url=url, headers={'Accept': 'application/json'}, method='GET'), timeout=5) result = res.read().decode().strip().split("\n") return result except: return [] def get_memorydb_info(): subnet_group_name = "memorydb-db-subnet" vpcid = os.environ.get('vpcid') memory_client = boto3.client("memorydb") vpc_client = boto3.client("ec2") mysubnets = [] az_list = [] failed = False subnet_response = vpc_client.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpcid]}]) print(subnet_response['Subnets']) for vpc in subnet_response['Subnets']: if not vpc['MapPublicIpOnLaunch'] : mysubnets.append(vpc['SubnetId']) az_list.append(vpc['AvailabilityZone']) try: reponse = memory_client.create_subnet_group( SubnetGroupName=subnet_group_name, SubnetIds=mysubnets) except Exception as e: if "SubnetGroupAlreadyExistsFault" in str(e): print("Subnet group already exists.. continuing") else: failed = True print("Creation of test memorydb subnet failed: {}".format(str(e))) if "SubnetNotAllowedFault" in str(e): az= str(e).split("[")[-1].replace(" ","").replace("].","") print("Supported region {}".format(az)) az_list = az.split(",") subnet_list=[] for vpc in subnet_response['Subnets']: if vpc['AvailabilityZone'] in az_list and not vpc['MapPublicIpOnLaunch'] : subnet_list.append(vpc['SubnetId']) if failed: try: reponse = memory_client.create_subnet_group( SubnetGroupName=subnet_group_name, SubnetIds=subnet_list) except Exception as e: print("Failed to create the subnet with the new subnetlist {}".format(str(e))) print(az_list) print(subnet_list) return (az_list, subnet_list) def ack_permissions(eks_cluster_name): service_names = ['rds','memorydb','dynamodb'] output = {} aws_account_id=boto3.client('sts').get_caller_identity().get('Account') print ("Account id {}".format(aws_account_id)) oidc_provider = boto3.client('eks').describe_cluster(name=eks_cluster_name)['cluster']['identity']['oidc']['issuer'].replace("https://","") ack_k8s_namespace="ack-system" for service in service_names: ack_k8s_service_account_name="ack-{}-controller".format(service) trust_json = """{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": "arn:aws:iam::"""+aws_account_id+""":oidc-provider/"""+oidc_provider+"""" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { \""""+ oidc_provider+""":sub": "system:serviceaccount:"""+ack_k8s_namespace+""":"""+ack_k8s_service_account_name+"""" } } } ] }""" ack_controller_iam_role = "ack-{}-controller".format(service) ack_controller_iam_role_description = "IRSA role for ACK {} controller deployment on EKS cluster using Helm charts".format(service) response = None try: response = boto3.client("iam").create_role(RoleName=ack_controller_iam_role, AssumeRolePolicyDocument=trust_json, Description = ack_controller_iam_role_description) except ClientError as e: if e.response['Error']['Code'] == 'EntityAlreadyExists': response = boto3.client("iam").get_role(RoleName=ack_controller_iam_role) print("Iam Role {} already exists".format(ack_controller_iam_role)) ack_controller_iam_role_arn = response['Role']['Arn'] output[ack_controller_iam_role] = ack_controller_iam_role_arn base_url="https://raw.githubusercontent.com/aws-controllers-k8s/{}-controller/main".format(service) policy_arn_url = "{}/config/iam/recommended-policy-arn".format(base_url) policy_arn_strings = get_arns(policy_arn_url) for policy_arn in policy_arn_strings: response = boto3.client("iam").attach_role_policy(RoleName=ack_controller_iam_role, PolicyArn = policy_arn) inline_policy_url="{}/config/iam/recommended-inline-policy".format(base_url) inline_policy=get_arns(inline_policy_url) loadbalancerversion = os.environ.get('loadbalancerversion') policyDoc = get_doc("https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/{}/docs/install/iam_policy.json".format(loadbalancerversion)) try: response = boto3.client("iam").create_policy(PolicyName="AWSLoadBalancerControllerIAMPolicy", PolicyDocument=policyDoc, Description="Load balancer controller policy") except ClientError as e: if e.response['Error']['Code'] == 'EntityAlreadyExists': pass output["AWSLoadBalancerControllerIAMPolicy"] = "arn:aws:iam::{}:policy/AWSLoadBalancerControllerIAMPolicy".format(aws_account_id) return str(json.dumps(output,indent=2)) def lambda_handler(event, context): status = cfnresponse.SUCCESS data = {} eks_cluster_name = event['ResourceProperties']['eks_cluster_name'] print ("My event {}".format(event)) print("Eks cluster name {}".format(eks_cluster_name)) key_name = eks_cluster_name try: data['Arns'] = ack_permissions(eks_cluster_name) az_list, subnet_list = get_memorydb_info() data['memorydb_azs'] = ",".join(az_list) data['memorydb_subnets'] = ",".join(subnet_list) except: traceback.print_exc() status = cfnresponse.FAILED cfnresponse.send(event, context, status, data, key_name, noEcho=True) DBSubnetGroup: Type: 'AWS::RDS::DBSubnetGroup' Properties: DBSubnetGroupName: rds-db-subnet DBSubnetGroupDescription: !Ref 'AWS::StackName' SubnetIds: - !Ref SubnetAPrivate - !Ref SubnetBPrivate - !Ref SubnetCPrivate ClusterSecurityGroup: Type: 'AWS::EC2::SecurityGroup' Properties: GroupDescription: !Ref 'AWS::StackName' SecurityGroupIngress: - IpProtocol: tcp FromPort: 5432 ToPort: 5432 CidrIp: '10.0.0.0/8' Description: 'Access to AppServer Host Security Group for PG' - IpProtocol: tcp FromPort: 6379 ToPort: 6379 CidrIp: '10.0.0.0/8' Description: 'Access to AppServer Host Security Group for Redis' SecurityGroupEgress: - IpProtocol: -1 FromPort: 0 ToPort: 65535 CidrIp: 0.0.0.0/0 VpcId: !Ref VPC Tags: - Key: Name Value: !Sub '${AWS::StackName}-AuroraClusterSecurityGroup' ClusterSecurityGroupIngress: Type: 'AWS::EC2::SecurityGroupIngress' Properties: GroupId: !GetAtt 'ClusterSecurityGroup.GroupId' IpProtocol: -1 SourceSecurityGroupId: !Ref ClusterSecurityGroup Description: 'Self Reference' #============================================================================# # Control plane security group #============================================================================# ControlPlaneSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Cluster communication with worker nodes VpcId: !Ref VPC ControlPlaneIngressFromWorkerNodesHttps: Type: AWS::EC2::SecurityGroupIngress Properties: Description: Allow incoming HTTPS traffic (TCP/443) from worker nodes (for API server) GroupId: !Ref ControlPlaneSecurityGroup SourceSecurityGroupId: !Ref WorkerNodesSecurityGroup IpProtocol: tcp ToPort: 443 FromPort: 443 ControlPlaneEgressToWorkerNodesKubelet: Type: AWS::EC2::SecurityGroupEgress Properties: Description: Allow outgoing kubelet traffic (TCP/10250) to worker nodes GroupId: !Ref ControlPlaneSecurityGroup DestinationSecurityGroupId: !Ref WorkerNodesSecurityGroup IpProtocol: tcp FromPort: 10250 ToPort: 10250 ControlPlaneEgressToWorkerNodesHttps: Type: AWS::EC2::SecurityGroupEgress Properties: Description: Allow outgoing HTTPS traffic (TCP/442) to worker nodes (for pods running extension API servers) GroupId: !Ref ControlPlaneSecurityGroup DestinationSecurityGroupId: !Ref WorkerNodesSecurityGroup IpProtocol: tcp FromPort: 443 ToPort: 443 #============================================================================# # EKS configuration #============================================================================# EKSCluster: Type: AWSQS::EKS::Cluster Properties: Name: !Ref EKSClusterName RoleArn: !GetAtt EKSIAMRole.Arn Version: !Ref KubernetesVersion ResourcesVpcConfig: SecurityGroupIds: - !Ref ControlPlaneSecurityGroup SubnetIds: - !Ref SubnetAPrivate - !Ref SubnetBPrivate - !Ref SubnetCPrivate KubernetesApiAccess: Roles: - Arn: !GetAtt C9Role.Arn Username: "AdminRole" Groups: ["system:masters"] EKSIAMRole: Type: AWS::IAM::Role Properties: Path: / AssumeRolePolicyDocument: Version: '2012-10-17' Statement: Effect: Allow Principal: Service: - eks.amazonaws.com - ec2.amazonaws.com - eks-connector.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy - arn:aws:iam::aws:policy/AmazonEKSServicePolicy - arn:aws:iam::aws:policy/AdministratorAccess #============================================================================# # WorkerNode security group #============================================================================# WorkerNodesSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Security group for all the worker nodes VpcId: !Ref VPC Tags: - Key: Name Value: !Sub "${AWS::StackName}-WorkerNodesSecurityGroup" - Key: !Sub "kubernetes.io/cluster/${EKSCluster}" Value: "owned" SecurityGroupEgress: - IpProtocol: -1 FromPort: 0 ToPort: 65535 CidrIp: 0.0.0.0/0 WorkerNodesIngressFromWorkerNodes: Type: AWS::EC2::SecurityGroupIngress Properties: Description: Allow all incoming traffic from other worker nodes GroupId: !Ref WorkerNodesSecurityGroup SourceSecurityGroupId: !Ref WorkerNodesSecurityGroup IpProtocol: "-1" WorkerNodesIngressFromControlPlaneKubelet: Type: AWS::EC2::SecurityGroupIngress Properties: Description: Allow incoming kubelet traffic (TCP/10250) from control plane GroupId: !Ref WorkerNodesSecurityGroup SourceSecurityGroupId: !Ref ControlPlaneSecurityGroup IpProtocol: tcp FromPort: 10250 ToPort: 10250 WorkerNodesIngressFromControlPlaneHttps: Type: AWS::EC2::SecurityGroupIngress Properties: Description: Allow incoming HTTPS traffic (TCP/443) from control plane (for pods running extension API servers) GroupId: !Ref WorkerNodesSecurityGroup SourceSecurityGroupId: !Ref ControlPlaneSecurityGroup IpProtocol: tcp FromPort: 443 ToPort: 443 #============================================================================# # Worker Nodes Group #============================================================================# WorkerNodesRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: Effect: Allow Principal: Service: - ec2.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly - arn:aws:iam::aws:policy/AmazonRDSFullAccess - arn:aws:iam::aws:policy/AmazonElastiCacheFullAccess - arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess # IMPORTANT NOTE: We have to define NodeGroup (type: AWS::EKS::Nodegroup), without this no woker nodes will be attach to cluster WorkerNodegroup: Type: AWS::EKS::Nodegroup DependsOn: EKSCluster Properties: ClusterName: !Ref EKSClusterName NodeRole: !GetAtt WorkerNodesRole.Arn ScalingConfig: MinSize: Ref: NumWorkerNodes DesiredSize: Ref: NumWorkerNodes MaxSize: Ref: NumWorkerNodes Subnets: - !Ref SubnetAPrivate - !Ref SubnetBPrivate - !Ref SubnetCPrivate Outputs: TemplateID: Description: 'Template ID' Value: 'DAT312' Region: Description: 'Region' Value: !Sub '${AWS::Region}' StackName: Description: 'Stack name' Value: !Sub '${AWS::StackName}' EKSRole: Value: !Ref EKSIAMRole Export: Name: 'Fn::Sub': '${AWS::StackName}-EKSRole' EKSClusterName: Value: !Ref EKSClusterName Export: Name: 'Fn::Sub': '${AWS::StackName}-EKSClusterName' DBSubnetGroup: Value: !Ref DBSubnetGroup Export: Name: 'Fn::Sub': '${AWS::StackName}-DBSubnetGroup' ClusterSecurityGroup: Value: !Ref ClusterSecurityGroup Export: Name: 'Fn::Sub': '${AWS::StackName}-ClusterSecurityGroup' ACKIamArns: Value: !GetAtt ACKGrants.Arns MemoryDBAllowedAZ: Value: !GetAtt ACKGrants.memorydb_azs MemoryDBAllowedSubnets: Value: !GetAtt ACKGrants.memorydb_subnets Cloud9IDEURL: Export: Name: 'Fn::Sub': '${AWS::StackName}-Cloud9IDEURL' Value: !Join - '' - - !Sub https://${AWS::Region}.console.aws.amazon.com/cloud9/ide/ - !Ref C9Instance