# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 # # Permission is hereby granted, free of charge, to any person obtaining a copy of this # software and associated documentation files (the "Software"), to deal in the Software # without restriction, including without limitation the rights to use, copy, modify, # merge, publish, distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A # PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. AWSTemplateFormatVersion: '2010-09-09' Description: "CloudFormation Template to create resources for the blog describing integration of Aurora MySQL with S3, SageMaker endpoint and Comprehend. Set up resources: Networking; IAM Roles; ML services: SageMaker and Comprehend; S3; Aurora MySQL DB; Secret Manager" Parameters: AuroraDatabaseAdministratorUsername: Description: "Administrator name for Aurora DB. Password is randomly generated and used as a secret per best practices." Type: String Default: admin SageMakerInstanceType: Description: "Instance type for SageMaker notebook." Type: String Default: ml.m4.xlarge AllowedValues: - ml.t2.medium - ml.t2.large - ml.t2.xlarge - ml.t2.2xlarge - ml.m4.xlarge - ml.m4.2xlarge - ml.m4.4xlarge - ml.m4.10xlarge - ml.m4.16xlarge - ml.p2.xlarge - ml.p2.8xlarge - ml.p2.16xlarge - ml.p3.2xlarge - ml.p3.8xlarge - ml.p3.16xlarge AuroraDatabaseInstanceType: Description: "Instance type for Aurora DB." Type: String Default: db.r5.large AllowedValues: - db.m5.medium - db.m5.large - db.m5.xlarge - db.m4.medium - db.m3.medium - db.m1.medium - db.z1d.medium - db.x1.medium - db.r5.medium - db.r5.large - db.r4.medium - db.r3.medium - db.m2.medium - db.t3.medium - db.t2.medium # Documentation for regions and corresponging containers is available # here: https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html # We use XGBoost version 0.90 Mappings: RegionMap: "us-west-1": "XGBoost": "746614075791.dkr.ecr.us-west-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "us-west-2": "XGBoost": "246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "us-east-1": "XGBoost": "683313688378.dkr.ecr.us-east-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "us-east-2": "XGBoost": "257758044811.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "us-gov-west-1": "XGBoost": "414596584902.dkr.ecr.us-gov-west-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-northeast-1": "XGBoost": "354813040037.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-northeast-2": "XGBoost": "366743142698.dkr.ecr.ap-northeast-2.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-southeast-1": "XGBoost": "121021644041.dkr.ecr.ap-southeast-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-southeast-2": "XGBoost": "783357654285.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-south-1": "XGBoost": "720646828776.dkr.ecr.ap-south-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ap-east-1": "XGBoost": "651117190479.dkr.ecr.ap-east-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "ca-central-1": "XGBoost": "341280168497.dkr.ecr.ca-central-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "eu-central-1": "XGBoost": "492215442770.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "eu-north-1": "XGBoost": "662702820516.dkr.ecr.eu-north-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "eu-west-1": "XGBoost": "141502667606.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "eu-west-2": "XGBoost": "764974769150.dkr.ecr.eu-west-2.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "eu-west-3": "XGBoost": "659782779980.dkr.ecr.eu-west-3.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "me-south-1": "XGBoost": "801668240914.dkr.ecr.me-south-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" "sa-east-1": "XGBoost": "737474898029.dkr.ecr.sa-east-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3" Resources: # Networking section # VPC: Description: "Provision VPC for resources." Type: AWS::EC2::VPC Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName EnableDnsHostnames: true EnableDnsSupport: true CidrBlock: 10.192.0.0/16 InternetGateway: Description: "Create Internet Gateway." Type: AWS::EC2::InternetGateway Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName InternetGatewayAttachment: Description: "Attach Internet Gateway." Type: AWS::EC2::VPCGatewayAttachment Properties: VpcId: !Ref VPC InternetGatewayId: !Ref InternetGateway NatGatewayEIP: Type: AWS::EC2::EIP DependsOn: InternetGatewayAttachment Properties: Domain: vpc NatGateway: Type: AWS::EC2::NatGateway Properties: AllocationId: !GetAtt NatGatewayEIP.AllocationId SubnetId: !Ref PublicSubnet PublicSubnet: Description: "Create public subnet." Type: AWS::EC2::Subnet Properties: AvailabilityZone: Fn::Select: - 0 - Fn::GetAZs: "" VpcId: !Ref VPC CidrBlock: 10.192.10.0/24 MapPublicIpOnLaunch: true Tags: - Key: 'StackName' Value: !Ref AWS::StackName PublicRouteTable: Description: "Create public route table." Type: AWS::EC2::RouteTable Properties: VpcId: !Ref VPC Tags: - Key: 'StackName' Value: !Ref AWS::StackName InternetRoute: Description: "Add internet route to the table." Type: AWS::EC2::Route Properties: RouteTableId: !Ref PublicRouteTable DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway PublicSubnetRouteTableAssociation: Description: "Define association for the public route table." Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref PublicRouteTable SubnetId: !Ref PublicSubnet PrivateSubnet1: Description: "Create private subnet 1." Type: AWS::EC2::Subnet Properties: AvailabilityZone: Fn::Select: - 0 - Fn::GetAZs: "" VpcId: !Ref VPC CidrBlock: 10.192.20.0/24 MapPublicIpOnLaunch: false Tags: - Key: 'StackName' Value: !Ref AWS::StackName PrivateRouteTable1: Description: "Create private subnet 1 route table." Type: AWS::EC2::RouteTable Properties: VpcId: !Ref VPC Tags: - Key: 'StackName' Value: !Ref AWS::StackName PrivateRoute1: Type: AWS::EC2::Route Properties: RouteTableId: !Ref PrivateRouteTable1 DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: !Ref NatGateway PrivateSubnetRouteTableAssociation1: Description: "Define association for the private subnet 1 route table." Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref PrivateRouteTable1 SubnetId: !Ref PrivateSubnet1 PrivateSubnet2: Description: "Create private subnet 2." Type: AWS::EC2::Subnet Properties: AvailabilityZone: Fn::Select: - 1 - Fn::GetAZs: "" VpcId: !Ref VPC CidrBlock: 10.192.21.0/24 MapPublicIpOnLaunch: false Tags: - Key: 'StackName' Value: !Ref AWS::StackName PrivateRouteTable2: Description: "Create private subnet 2 route table." Type: AWS::EC2::RouteTable Properties: VpcId: !Ref VPC Tags: - Key: 'StackName' Value: !Ref AWS::StackName PrivateRoute2: Type: AWS::EC2::Route Properties: RouteTableId: !Ref PrivateRouteTable2 DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: !Ref NatGateway PrivateSubnetRouteTableAssociation2: Description: "Define association for the private subnet 2 route table." Type: AWS::EC2::SubnetRouteTableAssociation Properties: RouteTableId: !Ref PrivateRouteTable2 SubnetId: !Ref PrivateSubnet2 AuroraDatabaseSubnetGroup: Description: "Define subnet group." Type: AWS::RDS::DBSubnetGroup Properties: DBSubnetGroupDescription: "Aurora Database Subnet Group" SubnetIds: - !Ref PrivateSubnet1 - !Ref PrivateSubnet2 Tags: - Key: 'StackName' Value: !Ref AWS::StackName # S3 Bucket # S3BucketForModelTraining: Description: "Create S3 bucket." DependsOn: S3EncryptionKMSKey Type: AWS::S3::Bucket DeletionPolicy: Retain Properties: AccessControl: "BucketOwnerFullControl" Tags: - Key: 'StackName' Value: !Ref AWS::StackName BucketName: !Join - "-" - - !Ref "AWS::AccountId" - "s3-bucket-for-model-training" - !Select - 0 - !Split - "-" - !Select - 2 - !Split - "/" - !Ref "AWS::StackId" BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !GetAtt S3EncryptionKMSKey.Arn PublicAccessBlockConfiguration: BlockPublicAcls: True BlockPublicPolicy: True IgnorePublicAcls: True RestrictPublicBuckets: True # Security Groups # SageMakerSecurityGroup: Description: "Create a security group for SageMaker." Type: AWS::EC2::SecurityGroup Properties: GroupDescription: "Allow SageMaker to access Aurora Database" VpcId: !Ref VPC Tags: - Key: 'StackName' Value: !Ref AWS::StackName AuroraDatabaseSecurityGroup: Description: "Create a securty group for Aurora Database." Type: AWS::EC2::SecurityGroup Properties: GroupDescription: "Allow DB access to SageMaker" VpcId: !Ref VPC Tags: - Key: 'StackName' Value: !Ref AWS::StackName AuroraDatabaseSecurityGroupIngress: Description: "Allow traffic from SageMaker to Aurora Database." Type: AWS::EC2::SecurityGroupIngress Properties: IpProtocol: tcp ToPort: 3306 FromPort: 3306 GroupId: !Ref AuroraDatabaseSecurityGroup SourceSecurityGroupId: !Ref SageMakerSecurityGroup # Secret Manager # # Implemented based on the blog: # https://aws.amazon.com/blogs/security/how-to-create-and-retrieve-secrets-managed-in-aws-secrets-manager-using-aws-cloudformation-template/ # Password rotation is not implemented. SecretsManager: Description: "Create a secret: username and a random password." Type: AWS::SecretsManager::Secret Properties: Description: 'Credentials secret for the Aurora database' GenerateSecretString: SecretStringTemplate: !Sub '{"username": "${AuroraDatabaseAdministratorUsername}"}' GenerateStringKey: 'password' PasswordLength: 16 ExcludeCharacters: '"@/\' Tags: - Key: 'StackName' Value: !Ref AWS::StackName SecretManagerAuroraAttachment: Type: "AWS::SecretsManager::SecretTargetAttachment" Properties: SecretId: !Ref SecretsManager TargetId: !Ref AuroraDatabaseInstance TargetType: AWS::RDS::DBInstance # KMS Key S3EncryptionKMSKey: Type: AWS::KMS::Key Properties: Description: "CMK for encryption of the S3 bucket" KeyPolicy: Version: 2012-10-17 Id: key-s3 Statement: - Sid: Enable IAM User Permissions Effect: Allow Principal: AWS: !Join - '' - - 'arn:aws:iam::' - !Ref 'AWS::AccountId' - ':root' Action: 'kms:*' Resource: '*' Tags: - Key: 'StackName' Value: !Ref AWS::StackName S3EncryptionKMSKeyAlias: Type: AWS::KMS::Alias Properties: AliasName: alias/blog TargetKeyId: Ref: S3EncryptionKMSKey # IAM Roles # # SageMaker role to access S3, secret; AmazonSageMakerFullAccess managed role # SageMakerExecutionRole: Description: "Create SageMaker IAM role." Type: "AWS::IAM::Role" Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - "sagemaker.amazonaws.com" Action: - "sts:AssumeRole" Path: "/" Policies: - PolicyName: Fn::Sub: "${AWS::StackName}-s3bucket" PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Resource: - Fn::Join: - '' - - 'arn:aws:s3:::' - Ref: S3BucketForModelTraining Action: - "s3:*" - Effect: Allow Resource: - Ref: SecretsManager Action: - "secretsmanager:GetSecretValue" - Effect: Allow Resource: - !GetAtt S3EncryptionKMSKey.Arn Action: - "kms:*" ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess # Role for Aurora to access S3 # AuroraDatabaseS3AccessRole: Description: "Create Aurora Database IAM role to access S3." Type: "AWS::IAM::Role" Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - "rds.amazonaws.com" Action: - "sts:AssumeRole" Path: "/" Policies: - PolicyName: Fn::Sub: "${AWS::StackName}-Aurora-Connect-to-S3-RolePolicy" PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Resource: - Fn::Join: - '' - - 'arn:aws:s3:::' - Ref: S3BucketForModelTraining - Fn::Join: - '' - - 'arn:aws:s3:::' - Ref: S3BucketForModelTraining - '/*' Action: - "s3:*" - Effect: Allow Resource: - !GetAtt S3EncryptionKMSKey.Arn Action: - "kms:*" # Role for Aurora to access Comprehend # AuroraDatabaseComprehendAccessRole: Description: "Create Aurora Database IAM role to access Amazon Comprehend." Type: "AWS::IAM::Role" Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - "rds.amazonaws.com" Action: - "sts:AssumeRole" Path: "/" Policies: - PolicyName: Fn::Sub: "${AWS::StackName}-Aurora-Connect-to-Comprehend-RolePolicy" PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Resource: "*" Action: - "comprehend:DetectSentiment" - "comprehend:BatchDetectSentiment" # Role for Aurora to access SageMaker Endpoint # AuroraDatabaseSageMakerEndpointAccessRole: Description: "Create Aurora Database IAM role for SageMaker Endpoint." Type: "AWS::IAM::Role" Properties: Tags: - Key: 'StackName' Value: !Ref AWS::StackName AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - "rds.amazonaws.com" Action: - "sts:AssumeRole" Path: "/" Policies: - PolicyName: Fn::Sub: "${AWS::StackName}-Aurora-Connect-to-SageMakerEndpoint-RolePolicy" PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Resource: !Ref SageMakerEndpoint Action: - "sagemaker:InvokeEndpoint" # SageMaker section # # SagMaker Notebook instance # SageMakerNotebookInstance: Description: "Create SageMaker Notebook Instance." Type: "AWS::SageMaker::NotebookInstance" Properties: InstanceType: !Ref SageMakerInstanceType NotebookInstanceName: !Sub "${AWS::StackName}-SageMakerNotebookInstance" RoleArn: !GetAtt SageMakerExecutionRole.Arn SecurityGroupIds: [!Ref SageMakerSecurityGroup] SubnetId: !Ref PublicSubnet LifecycleConfigName: !GetAtt SageMakerNotebookInstanceLifecycleConfig.NotebookInstanceLifecycleConfigName Tags: - Key: 'StackName' Value: !Ref AWS::StackName SageMakerNotebookInstanceLifecycleConfig: Description: "Create SageMaker Notebook LifeCycle configuration." Type: "AWS::SageMaker::NotebookInstanceLifecycleConfig" DependsOn: AuroraDatabaseInstance Properties: OnStart: - Content: Fn::Base64: !Sub | wget -O /home/ec2-user/SageMaker/utilities.py https://aws-ml-blog.s3.amazonaws.com/artifacts/prevent-customer-churn/utilities.py wget -O /home/ec2-user/SageMaker/part_1_preventing_customer_churn_Amazon_Aurora_setup.ipynb https://aws-ml-blog.s3.amazonaws.com/artifacts/prevent-customer-churn/part_1_preventing_customer_churn_Amazon_Aurora_setup.ipynb wget -O /home/ec2-user/SageMaker/part_2_preventing_customer_churn_XGBoost.ipynb https://aws-ml-blog.s3.amazonaws.com/artifacts/prevent-customer-churn/part_2_preventing_customer_churn_XGBoost.ipynb wget -O /home/ec2-user/SageMaker/part_3_preventing_customer_churn_inferences_from_Amazon_Aurora.ipynb https://aws-ml-blog.s3.amazonaws.com/artifacts/prevent-customer-churn/part_3_preventing_customer_churn_inferences_from_Amazon_Aurora.ipynb printf "ENDPOINT=\"${SageMakerEndpoint.EndpointName}\"\nREGION=\"${AWS::Region}\"\nDBSECRET=\"${SecretsManager}\"\nS3BUCKET=\"${S3BucketForModelTraining}\"" >> /home/ec2-user/SageMaker/cloudformation_values.py chmod +666 /home/ec2-user/SageMaker/*.py # SageMaker Model # SageMakerModel: Type: "AWS::SageMaker::Model" Properties: PrimaryContainer: Image: !FindInMap [RegionMap, !Ref "AWS::Region", "XGBoost"] ExecutionRoleArn: !GetAtt SageMakerExecutionRole.Arn ModelName: !Sub "${AWS::StackName}-SageMaker-Model" Tags: - Key: 'StackName' Value: !Ref AWS::StackName # SageMaker Endpoint Config # SageMakerEndpointConfig: Type: "AWS::SageMaker::EndpointConfig" Properties: EndpointConfigName: !Sub "${AWS::StackName}-SageMaker-EndpointConfig" ProductionVariants: - InitialInstanceCount: 1 InitialVariantWeight: 1.0 InstanceType: ml.m4.xlarge ModelName: !GetAtt SageMakerModel.ModelName VariantName: "AllTraffic" Tags: - Key: "StackName" Value: !Ref AWS::StackName # SageMaker Endpoint # SageMakerEndpoint: Type: "AWS::SageMaker::Endpoint" Properties: EndpointName: !Sub "${AWS::StackName}-SageMaker-Endpoint" EndpointConfigName: !GetAtt SageMakerEndpointConfig.EndpointConfigName Tags: - Key: "StackName" Value: !Ref AWS::StackName # Database section # # Aurora DB cluster # AuroraDatabaseCluster: Description: "Create Aurora Database cluster." Type: 'AWS::RDS::DBCluster' DeletionPolicy: Delete Properties: AssociatedRoles: - RoleArn: !GetAtt AuroraDatabaseS3AccessRole.Arn - RoleArn: !GetAtt AuroraDatabaseComprehendAccessRole.Arn - RoleArn: !GetAtt AuroraDatabaseSageMakerEndpointAccessRole.Arn DBSubnetGroupName: !Ref AuroraDatabaseSubnetGroup DBClusterIdentifier: !Sub "${AWS::StackName}-AuroraDatabaseCluster" DBClusterParameterGroupName: !Ref AuroraDatabaseClusterParameterGroup Engine: aurora-mysql EngineVersion: 5.7.mysql_aurora.2.07.1 MasterUsername: !Join ['', ['{{resolve:secretsmanager:', !Ref SecretsManager, ':SecretString:username}}' ]] MasterUserPassword: !Join ['', ['{{resolve:secretsmanager:', !Ref SecretsManager, ':SecretString:password}}' ]] Tags: - Key: 'StackName' Value: !Ref AWS::StackName VpcSecurityGroupIds: [!Ref AuroraDatabaseSecurityGroup] # Aurora DB instance # AuroraDatabaseInstance: Description: "Create Aurora Database instance." Type: AWS::RDS::DBInstance DeletionPolicy: Delete Properties: Engine: aurora-mysql EngineVersion: 5.7.mysql_aurora.2.07.1 DBClusterIdentifier: !Ref AuroraDatabaseCluster DBInstanceIdentifier: !Sub "${AWS::StackName}-AuroraDatabaseInstance" DBInstanceClass: !Ref AuroraDatabaseInstanceType Tags: - Key: 'StackName' Value: !Ref AWS::StackName PubliclyAccessible: false # Aurora DB Parameter Group # AuroraDatabaseClusterParameterGroup: Description: "Aurora Database Cluster Parameter Group." Type: "AWS::RDS::DBClusterParameterGroup" Properties: Description: "Custom parameter group" Family: aurora-mysql5.7 Parameters: character_set_database: utf32 aws_default_s3_role: !GetAtt AuroraDatabaseS3AccessRole.Arn aws_default_comprehend_role: !GetAtt AuroraDatabaseComprehendAccessRole.Arn aws_default_sagemaker_role: !GetAtt AuroraDatabaseSageMakerEndpointAccessRole.Arn Outputs: AuroraDatabaseClusterName: Value: !Ref AuroraDatabaseCluster SageMakerNotebookInstanceName: Value: !GetAtt SageMakerNotebookInstance.NotebookInstanceName SageMakerEndpointName: Value: !GetAtt SageMakerEndpoint.EndpointName S3BucketName: Value: !Ref S3BucketForModelTraining