--- AWSTemplateFormatVersion: "2010-09-09" Parameters: LatestAmiId: Type: 'AWS::SSM::Parameter::Value' Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' SourceBucket: Description: S3 bucket which contains the source object Default: aws-dataengineering-day.workshop.aws Type: String SourceKey: Description: S3 Key which contains the source object Default: dmslambda_v1.zip Type: String Resources: DMSLabCFNS3Bucket: Type: 'AWS::S3::Bucket' LambdaCopySourceRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: LambdaCopySourceRolePolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: s3:GetObject Resource: !Sub arn:aws:s3:::${SourceBucket}/* - Effect: Allow Action: - s3:PutObject - s3:DeleteObject Resource: !Sub arn:aws:s3:::${DMSLabCFNS3Bucket}/* - Effect: Allow Action: logs:CreateLogGroup Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:* - Effect: Allow Action: - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:* LambdaCopySource: Type: AWS::Lambda::Function Properties: Handler: index.handler Role: !GetAtt LambdaCopySourceRole.Arn Code: ZipFile: !Sub | import os import json import cfnresponse import boto3 from botocore.exceptions import ClientError s3 = boto3.client('s3') import logging logger = logging.getLogger() logger.setLevel(logging.INFO) def handler(event, context): logger.info("Received event: %s" % json.dumps(event)) source_bucket = '${SourceBucket}' source_prefix = '${SourceKey}' bucket = '${DMSLabCFNS3Bucket}' prefix = '${SourceKey}' result = cfnresponse.SUCCESS try: if event['RequestType'] == 'Create' or event['RequestType'] == 'Update': copy_source = {'Bucket': source_bucket, 'Key': source_prefix} s3.copy(copy_source, bucket, prefix) elif event['RequestType'] == 'Delete': s3.delete_object(Bucket=bucket, Key=prefix) except ClientError as e: logger.error('Error: %s', e) result = cfnresponse.FAILED cfnresponse.send(event, context, result, {}) Runtime: python3.8 Timeout: 60 CopySourceObject: Type: "Custom::CopySourceObject" Properties: ServiceToken: !GetAtt LambdaCopySource.Arn DMSInstructorVPC: Type: AWS::EC2::VPC Properties: CidrBlock: 10.0.0.0/16 EnableDnsSupport: true EnableDnsHostnames: true Tags: - Key: Name Value: DMSLabSourceDB RDSSubNet: Type: AWS::EC2::Subnet Properties: CidrBlock: 10.0.1.0/24 AvailabilityZone: !Select - 0 - Fn::GetAZs: !Ref 'AWS::Region' VpcId: !Ref DMSInstructorVPC Tags: - Key: Name Value: DMSLabRDS1 EC2SubNet: Type: AWS::EC2::Subnet Properties: CidrBlock: 10.0.0.0/24 AvailabilityZone: !Select - 0 - Fn::GetAZs: !Ref 'AWS::Region' VpcId: !Ref DMSInstructorVPC Tags: - Key: Name Value: DMSLabEC2 RDSSubNet2: Type: AWS::EC2::Subnet Properties: CidrBlock: 10.0.2.0/24 AvailabilityZone: !Select - 1 - Fn::GetAZs: !Ref 'AWS::Region' VpcId: !Ref DMSInstructorVPC Tags: - Key: Name Value: DMSLabRDS2 DMSLabIGW: Type: AWS::EC2::InternetGateway Properties: Tags: - Key: Name Value: DMSLabIGW DMSLabDHCPOptions: Type: AWS::EC2::DHCPOptions Properties: DomainName: ec2.internal DomainNameServers: - AmazonProvidedDNS DMSLabRouteTable: Type: AWS::EC2::RouteTable Properties: VpcId: !Ref DMSInstructorVPC Tags: - Key: Name Value: DMSLabRT DMSLabInstance: Type: AWS::EC2::Instance Properties: DisableApiTermination: false InstanceInitiatedShutdownBehavior: stop EbsOptimized: true ImageId: !Ref LatestAmiId InstanceType: t3.2xlarge UserData: Fn::Base64: !Sub | Content-Type: multipart/mixed; boundary="//" MIME-Version: 1.0 --// Content-Type: text/cloud-config; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="cloud-config.txt" #cloud-config cloud_final_modules: - [scripts-user, always] --// Content-Type: text/x-shellscript; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="userdata.txt" #!/bin/bash -xe yum install -y postgresql yum install -y git yum update -y cd /home/ec2-user DIR="aws-database-migration-samples" if [ ! -d "$DIR" ]; then git clone https://github.com/aws-samples/aws-database-migration-samples.git fi cd aws-database-migration-samples/PostgreSQL/sampledb/v1/ export PGPASSWORD=admin123 export ENDPOINT=${RDSDMSLabDB.Endpoint.Address} nohup psql --host=${!ENDPOINT} --port=5432 --dbname=sportstickets --username=adminuser -f install-postgresql.sql Monitoring: false Tags: - Key: Name Value: DMSLabEC2 NetworkInterfaces: - DeleteOnTermination: true Description: 'Primary network interface' DeviceIndex: '0' SubnetId: !Ref EC2SubNet PrivateIpAddresses: - PrivateIpAddress: 10.0.0.40 Primary: true GroupSet: - !Ref DMSLabSG AssociatePublicIpAddress: true RDSDMSLabDB: Type: AWS::RDS::DBInstance DependsOn: InternetRoute Properties: AllocatedStorage: '500' AllowMajorVersionUpgrade: false AutoMinorVersionUpgrade: true DBInstanceClass: db.t2.2xlarge DBInstanceIdentifier: dmslabinstance Port: '5432' PubliclyAccessible: true StorageType: gp2 BackupRetentionPeriod: 7 MasterUsername: adminuser MasterUserPassword: admin123 PreferredBackupWindow: 04:00-04:30 PreferredMaintenanceWindow: sun:05:20-sun:05:50 DBName: sportstickets Engine: postgres EngineVersion: '11.20' LicenseModel: postgresql-license DBSubnetGroupName: !Ref DMSLabDefaultDBSubnet DBParameterGroupName: !Ref DMSLabDBPG VPCSecurityGroups: - !Ref DMSLabDBSG Tags: - Key: workload-type Value: other DMSLabDefaultDBSubnet: Type: AWS::RDS::DBSubnetGroup Properties: DBSubnetGroupDescription: Created from the RDS Management Console SubnetIds: - !Ref RDSSubNet - !Ref EC2SubNet - !Ref RDSSubNet2 DMSLabDBPG: Type: AWS::RDS::DBParameterGroup Properties: Description: Parameter for Ticket RedshiftDatabaseName Family: postgres11 Parameters: rds.logical_replication: '1' wal_sender_timeout: '0' max_wal_senders: '20' max_replication_slots: '50' DMSLabSG: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: EC2 Security Group VpcId: !Ref DMSInstructorVPC DMSLabDBSG: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: RDS Security Group VpcId: !Ref DMSInstructorVPC Tags: - Key: Name Value: DMSLabRDS-SG DMSLabGWAttachement: Type: AWS::EC2::VPCGatewayAttachment Properties: VpcId: !Ref DMSInstructorVPC InternetGatewayId: !Ref DMSLabIGW SubnetRoute1: Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref DMSLabRouteTable SubnetId: !Ref RDSSubNet2 SubnetRoute2: Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref DMSLabRouteTable SubnetId: !Ref RDSSubNet SubnetRoute3: Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref DMSLabRouteTable SubnetId: !Ref EC2SubNet InternetRoute: Type: AWS::EC2::Route Properties: DestinationCidrBlock: 0.0.0.0/0 RouteTableId: !Ref DMSLabRouteTable GatewayId: !Ref DMSLabIGW DependsOn: DMSLabGWAttachement DHCPAssociation: Type: AWS::EC2::VPCDHCPOptionsAssociation Properties: VpcId: !Ref DMSInstructorVPC DhcpOptionsId: !Ref DMSLabDHCPOptions SGIngress1: Type: AWS::EC2::SecurityGroupIngress Properties: GroupId: !Ref DMSLabSG IpProtocol: tcp FromPort: 22 ToPort: 22 CidrIp: 0.0.0.0/0 SGIngress2: Type: AWS::EC2::SecurityGroupIngress Properties: GroupId: !Ref DMSLabDBSG IpProtocol: tcp FromPort: 5432 ToPort: 5432 SourceSecurityGroupId: !Ref DMSLabSG SGIngress3: Type: AWS::EC2::SecurityGroupIngress Properties: GroupId: !Ref DMSLabDBSG IpProtocol: tcp FromPort: 5432 ToPort: 5432 CidrIp: 10.0.0.5/32 SGEgress1: Type: AWS::EC2::SecurityGroupEgress Properties: GroupId: !Ref DMSLabSG IpProtocol: '-1' CidrIp: 0.0.0.0/0 SGEgress2: Type: AWS::EC2::SecurityGroupEgress Properties: GroupId: !Ref DMSLabDBSG IpProtocol: '-1' CidrIp: 0.0.0.0/0 GenerateCDCData: Type: AWS::Lambda::Function DependsOn: CopySourceObject Properties: Code: S3Bucket: !Ref DMSLabCFNS3Bucket S3Key: !Ref SourceKey Description: Function to generate CDC data FunctionName: GenerateCDCData Handler: index.handler MemorySize: 128 Runtime: nodejs18.x Timeout: 300 Environment: Variables: HOST: !GetAtt RDSDMSLabDB.Endpoint.Address Role: !GetAtt LambdaExecutionRole.Arn VpcConfig: SecurityGroupIds: - !GetAtt DMSLabSG.GroupId SubnetIds: - !Ref EC2SubNet LambdaExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Policies: - PolicyName: DMSlabRDSAccess PolicyDocument: Version: 2012-10-17 Statement: - Sid: RDSAccess Effect: Allow Action: - rds:* - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents - ec2:CreateNetworkInterface - ec2:DescribeNetworkInterfaces - ec2:DeleteNetworkInterface Resource: "*" LambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: LambdaRolePolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: s3:DeleteObject Resource: !Sub arn:aws:s3:::${DMSLabCFNS3Bucket}/* - Effect: Allow Action: s3:ListBucket Resource: !Sub arn:aws:s3:::${DMSLabCFNS3Bucket} - Effect: Allow Action: logs:CreateLogGroup Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:* - Effect: Allow Action: - logs:CreateLogStream - logs:PutLogEvents Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:* S3BucketHandler: Type: AWS::Lambda::Function Properties: Handler: index.handler Role: !GetAtt LambdaRole.Arn Code: ZipFile: | import os import json import cfnresponse import boto3 from botocore.exceptions import ClientError s3 = boto3.resource('s3') def handler(event, context): print("Received event: %s" % json.dumps(event)) s3_bucket = s3.Bucket(event['ResourceProperties']['Bucket']) try: if event['RequestType'] == 'Create' or event['RequestType'] == 'Update': result = cfnresponse.SUCCESS elif event['RequestType'] == 'Delete': s3_bucket.objects.delete() result = cfnresponse.SUCCESS except ClientError as e: print('Error: %s', e) result = cfnresponse.FAILED cfnresponse.send(event, context, result, {}) Runtime: python3.8 Timeout: 300 EmptyDMSLabCFNS3Bucket: Type: "Custom::EmptyDMSLabCFNS3Bucket" Properties: ServiceToken: !GetAtt S3BucketHandler.Arn Bucket: !Ref DMSLabCFNS3Bucket Outputs: DMSInstanceEndpoint: Description: DMS Instance Endpoint Value: !GetAtt RDSDMSLabDB.Endpoint.Address CDCFuntion: Description: CDC Function Value: !GetAtt GenerateCDCData.Arn Description: DMS Lab Instructor account