AWSTemplateFormatVersion: "2010-09-09" Description: 'Amazon Redshift demos Nov 17, 2021' Parameters: Session: Type: String Description: Session name must satisfy regular expression pattern ^[a-zA-Z0-9](-*[a-zA-Z0-9])* Default: "demo" NumOfNodes: Description: How many compute nodes in your Redshift cluster Type: Number Default: '2' NodeType: Description: The type of node to be provisioned Type: String Default: ra3.xlplus AllowedValues: - ra3.xlplus - ra3.4xlarge - ra3.16xlarge - dc2.large - dc2.8xlarge Mappings: RegionMap: us-east-1: # N.Virginia "EC2AMI" : "ami-07ae2661f90f4be31" "OracleSnapshotId" : "arn:aws:rds:us-east-1:413094830157:snapshot:oradb-dms-sample-nov2021" Resources: SimpleStackUidFunction: Type: AWS::Lambda::Function Properties: Code: ZipFile: | import hashlib import cfnresponse def get_md5(account: str, region: str, stack: str, length: int = 8) -> str: md5 = hashlib.new('md5') md5.update(account.encode('utf-8')) md5.update(region.encode('utf-8')) md5.update(stack.encode('utf-8')) return md5.hexdigest()[:length] def lambda_handler(event: dict, context) -> None: responseData = {} if event['RequestType'] == 'Create': account = event['ResourceProperties']['AccountId'] region = event['ResourceProperties']['Region'] stack = event['StackId'] md5 = get_md5(account=account, region=region, stack=stack) responseData['upper'] = md5.upper() responseData['lower'] = md5.lower() else: # delete / update rs = event['PhysicalResourceId'] responseData['upper'] = rs.upper() responseData['lower'] = rs.lower() cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, responseData['lower'], noEcho=True) Handler: "index.lambda_handler" Timeout: 30 Role: Fn::GetAtt: - SimpleStackUidFunctionRole - Arn Runtime: python3.7 SimpleStackUid: Type: Custom::RandomString Properties: ServiceToken: Fn::GetAtt: - SimpleStackUidFunction - Arn Region: AWS::Region AccountId: AWS::AccountId SimpleStackUidFunctionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - lambda.amazonaws.com Action: - sts:AssumeRole Path: / Policies: - PolicyName: "lambda-logs" PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - logs:CreateLogGroup - logs:CreateLogStream - logs:PutLogEvents Resource: - "arn:aws:logs:*:*:*" LambdaCreateEmptyBucket: Type: "AWS::Lambda::Function" Properties: FunctionName: !Sub "CreateEmptyBucket-${SimpleStackUid}" Role: !GetAtt 'LambdaRole.Arn' Timeout: 300 Code: ZipFile: | import json import boto3 import cfnresponse import logging logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) def handler(event, context): logger.info(json.dumps(event)) s3Bucket = event['ResourceProperties']['s3Bucket'] if event['RequestType'] == 'Delete': try: s3 = boto3.resource('s3') bucket = s3.Bucket(s3Bucket) bucket.objects.delete() bucket.delete() except Exception as e: logger.info(e) logger.info('Delete Complete') cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Data': 'Delete complete'}) else: try: s3 = boto3.client('s3') s3.create_bucket(Bucket=s3Bucket) logger.info('Create Complete') cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Data': 'Create complete'}) except Exception as e: logger.error(e) cfnresponse.send(event, context, cfnresponse.FAILED, {'Data': 'Create failed'}) Handler: index.handler Runtime: python3.7 TaskDataBucket: Type: Custom::InitCreateEmptyBucket DependsOn: - LambdaCreateEmptyBucket - SimpleStackUid Properties: ServiceToken: !GetAtt 'LambdaCreateEmptyBucket.Arn' s3Bucket: !Sub "rgs-task-data-${SimpleStackUid}" InternetGateway: Type: 'AWS::EC2::InternetGateway' VirtualPrivateCloud: Type: 'AWS::EC2::VPC' Properties: CidrBlock: '10.0.0.0/16' EnableDnsSupport: true EnableDnsHostnames: true InstanceTenancy: default SecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: 'Redshift VPC security group' SecurityGroupIngress: - CidrIp: '10.0.0.0/16' IpProtocol: '-1' - CidrIp: '52.23.63.224/27' Description : CIDR Block for Quicksight in us-east-1 IpProtocol: tcp FromPort: 5439 ToPort: 5439 - CidrIp: '52.70.63.192/27' Description : CIDR Block for Firehose in us-east-1 IpProtocol: tcp FromPort: 5439 ToPort: 5439 VpcId: Ref: VirtualPrivateCloud SecurityGroupSelfIngress: Type: AWS::EC2::SecurityGroupIngress DependsOn: SecurityGroup Properties: FromPort: -1 IpProtocol: -1 GroupId: !GetAtt [SecurityGroup, GroupId] SourceSecurityGroupId: !GetAtt [SecurityGroup, GroupId] ToPort: -1 SecurityGroupPlIngress: Type: AWS::EC2::SecurityGroupIngress DependsOn: SecurityGroup Properties: FromPort: 5439 Description : PrefixList for AWS IP addresses connected through VPN. SourcePrefixListId: pl-4e2ece27 IpProtocol: tcp GroupId: !Ref SecurityGroup ToPort: 5439 VPCGatewayAttachment: Type: 'AWS::EC2::VPCGatewayAttachment' Properties: VpcId: Ref: VirtualPrivateCloud InternetGatewayId: Ref: InternetGateway SubnetAPublic: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: Fn::Select: - 0 - Fn::GetAZs: "" CidrBlock: '10.0.0.0/20' MapPublicIpOnLaunch: true VpcId: Ref: VirtualPrivateCloud SubnetAPrivate: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: Fn::Select: - 0 - Fn::GetAZs: "" CidrBlock: '10.0.16.0/20' VpcId: Ref: VirtualPrivateCloud SubnetBPublic: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: Fn::Select: - 1 - Fn::GetAZs: "" CidrBlock: '10.0.32.0/20' MapPublicIpOnLaunch: true VpcId: Ref: VirtualPrivateCloud SubnetBPrivate: Type: 'AWS::EC2::Subnet' Properties: AvailabilityZone: Fn::Select: - 1 - Fn::GetAZs: "" CidrBlock: '10.0.48.0/20' VpcId: Ref: VirtualPrivateCloud RouteTablePublic: Type: 'AWS::EC2::RouteTable' Properties: VpcId: Ref: VirtualPrivateCloud RouteTablePrivate: Type: 'AWS::EC2::RouteTable' Properties: VpcId: Ref: VirtualPrivateCloud RouteTableBPublic: Type: 'AWS::EC2::RouteTable' Properties: VpcId: Ref: VirtualPrivateCloud RouteTableBPrivate: Type: 'AWS::EC2::RouteTable' Properties: VpcId: Ref: VirtualPrivateCloud RouteTableAssociationAPublic: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: SubnetId: Ref: SubnetAPublic RouteTableId: Ref: RouteTablePublic RouteTableAssociationAPrivate: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: SubnetId: Ref: SubnetAPrivate RouteTableId: Ref: RouteTablePrivate RouteTableAssociationBPublic: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: SubnetId: Ref: SubnetBPublic RouteTableId: Ref: RouteTableBPublic RouteTableAssociationBPrivate: Type: 'AWS::EC2::SubnetRouteTableAssociation' Properties: SubnetId: Ref: SubnetBPrivate RouteTableId: Ref: RouteTableBPrivate RouteTablePublicInternetRoute: Type: 'AWS::EC2::Route' DependsOn: VPCGatewayAttachment Properties: RouteTableId: Ref: RouteTablePublic DestinationCidrBlock: '0.0.0.0/0' GatewayId: Ref: InternetGateway RouteTablePublicBInternetRoute: Type: 'AWS::EC2::Route' DependsOn: VPCGatewayAttachment Properties: RouteTableId: Ref: RouteTableBPublic DestinationCidrBlock: '0.0.0.0/0' GatewayId: Ref: InternetGateway NetworkAclPublic: Type: 'AWS::EC2::NetworkAcl' Properties: VpcId: Ref: VirtualPrivateCloud NetworkAclPrivate: Type: 'AWS::EC2::NetworkAcl' Properties: VpcId: Ref: VirtualPrivateCloud SubnetNetworkAclAssociationAPublic: Type: 'AWS::EC2::SubnetNetworkAclAssociation' Properties: SubnetId: Ref: SubnetAPublic NetworkAclId: Ref: NetworkAclPublic SubnetNetworkAclAssociationAPrivate: Type: 'AWS::EC2::SubnetNetworkAclAssociation' Properties: SubnetId: Ref: SubnetAPrivate NetworkAclId: Ref: NetworkAclPrivate SubnetNetworkAclAssociationBPublic: Type: 'AWS::EC2::SubnetNetworkAclAssociation' Properties: SubnetId: Ref: SubnetBPublic NetworkAclId: Ref: NetworkAclPublic SubnetNetworkAclAssociationBPrivate: Type: 'AWS::EC2::SubnetNetworkAclAssociation' Properties: SubnetId: Ref: SubnetBPrivate NetworkAclId: Ref: NetworkAclPrivate NetworkAclEntryInPublicAllowAll: Type: 'AWS::EC2::NetworkAclEntry' Properties: NetworkAclId: Ref: NetworkAclPublic RuleNumber: 99 Protocol: -1 RuleAction: allow Egress: false CidrBlock: '0.0.0.0/0' NetworkAclEntryOutPublicAllowAll: Type: 'AWS::EC2::NetworkAclEntry' Properties: NetworkAclId: Ref: NetworkAclPublic RuleNumber: 99 Protocol: -1 RuleAction: allow Egress: true CidrBlock: '0.0.0.0/0' NetworkAclEntryInPrivateAllowVPC: Type: 'AWS::EC2::NetworkAclEntry' Properties: NetworkAclId: Ref: NetworkAclPrivate RuleNumber: 99 Protocol: -1 RuleAction: allow Egress: false CidrBlock: '0.0.0.0/0' NetworkAclEntryOutPrivateAllowVPC: Type: 'AWS::EC2::NetworkAclEntry' Properties: NetworkAclId: Ref: NetworkAclPrivate RuleNumber: 99 Protocol: -1 RuleAction: allow Egress: true CidrBlock: '0.0.0.0/0' RedshiftClusterParameterGroup: Properties: Description: Fn::Sub: "${Session} cluster parameter group" ParameterGroupFamily: "redshift-1.0" Parameters: - ParameterName: "wlm_json_configuration" ParameterValue: "[{\"query_group\":[\"defaulton\",\"lab4on\"],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"concurrency_scaling\":\"off\",\"priority\":\"normal\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"query_group\":[\"lab1\",\"lab2\",\"lab3normal\",\"lab4off\",\"lab5\",\"lab6reads\",\"defaultoff\"],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"concurrency_scaling\":\"off\",\"priority\":\"normal\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"query_group\":[\"lab3highest\"],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"concurrency_scaling\":\"off\",\"priority\":\"highest\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"query_group\":[\"lab3lowest\"],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"concurrency_scaling\":\"off\",\"priority\":\"lowest\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"query_group\":[\"lab6writes\"],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"concurrency_scaling\":\"off\",\"priority\":\"highest\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"query_group\":[],\"query_group_wild_card\":0,\"user_group\":[],\"user_group_wild_card\":0,\"priority\":\"normal\",\"queue_type\":\"auto\",\"auto_wlm\":true},{\"short_query_queue\":false}]" Type: AWS::Redshift::ClusterParameterGroup RedshiftClusterSubnetGroup: Properties: Description: "GPS306 cluster subnet group" SubnetIds: - Ref: SubnetAPublic - Ref: SubnetBPublic Type: AWS::Redshift::ClusterSubnetGroup RedshiftClusterRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Action: sts:AssumeRole Effect: Allow Principal: Service: redshift.amazonaws.com Version: '2012-10-17' ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole - arn:aws:iam::aws:policy/AWSLakeFormationDataAdmin RedshiftRoleLFAdmin: Type: AWS::LakeFormation::DataLakeSettings Properties: Admins: - DataLakePrincipalIdentifier: !GetAtt 'RedshiftClusterRole.Arn' LambdaRole: Type: AWS::IAM::Role Properties : RoleName: !Sub "LambdaRole-${SimpleStackUid}" AssumeRolePolicyDocument: Version : 2012-10-17 Statement : - Effect : Allow Principal : Service : - lambda.amazonaws.com Action : - sts:AssumeRole Path : / Policies: - PolicyName: LambdaCloudFormationPolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - s3:* Resource: - !Sub "arn:aws:s3:::cloudformation-custom-resource-response-${AWS::Region}" - !Sub "arn:aws:s3:::cloudformation-waitcondition-${AWS::Region}" - !Sub "arn:aws:s3:::cloudformation-custom-resource-response-${AWS::Region}/*" - !Sub "arn:aws:s3:::cloudformation-waitcondition-${AWS::Region}/*" ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess - arn:aws:iam::aws:policy/AmazonRDSDataFullAccess - arn:aws:iam::aws:policy/IAMFullAccess RedshiftCluster: Type: AWS::Redshift::Cluster DependsOn: - VPCGatewayAttachment Properties: ClusterIdentifier: Fn::Sub: "democluster-${SimpleStackUid}" ClusterSubnetGroupName: Ref: RedshiftClusterSubnetGroup ClusterType: "multi-node" DBName: "dev" MasterUserPassword: Fn::Sub: "Pp-${SimpleStackUid}" MasterUsername: Ref: Session NodeType: Ref: NodeType NumberOfNodes: Ref: NumOfNodes VpcSecurityGroupIds: - Ref: SecurityGroup PubliclyAccessible: 'true' ClusterParameterGroupName: Ref: RedshiftClusterParameterGroup IamRoles: - Fn::GetAtt: - RedshiftClusterRole - Arn Tags: - Key: stack_name Value: Ref: AWS::StackName GlueExternalDatabase: Type: AWS::Glue::Database Properties: CatalogId: Ref: AWS::AccountId DatabaseInput: Name: Fn::Sub: "${Session}-${SimpleStackUid}" Description: Fn::Sub: "Database for the ${Session} deployment to enable Spectrum" EC2Instance: Type: AWS::EC2::Instance DependsOn: - SecurityGroup Properties: InstanceType: 'm5a.2xlarge' Tags: - Key: Name Value: Fn::Join: - "-" - - Ref: AWS::StackName - EC2Instance BlockDeviceMappings: - DeviceName: "/dev/sda1" Ebs: DeleteOnTermination: 'true' Iops: '2000' VolumeSize: '250' VolumeType: io1 ImageId: Fn::FindInMap: - RegionMap - !Ref AWS::Region - EC2AMI NetworkInterfaces: - AssociatePublicIpAddress: 'true' DeleteOnTermination: 'true' DeviceIndex: 0 SubnetId: !Ref SubnetAPublic GroupSet: - Ref: SecurityGroup UserData: Fn::Base64: !Sub | new-item 'c:\users\developer\initialize.log' $log = 'c:\users\developer\initialize.log' $now = Get-Date try { Add-Content $log -value "[$now] - setting developer password and granting remote login" net.exe user developer Pp-${SimpleStackUid} net.exe LOCALGROUP "Remote Desktop Users" developer /ADD Add-Content $log -value "[$now] - successfully set password to Pp-${SimpleStackUid}" } catch { Add-Content $log -value "[$now] - error changing password $_" } SecurityGroupPlIngressRDP: Type: AWS::EC2::SecurityGroupIngress DependsOn: SecurityGroup Properties: FromPort: 3389 Description : RDP access to EC2 instance from anywhere CidrIp: 0.0.0.0/0 IpProtocol: tcp GroupId: !Ref SecurityGroup ToPort: 3389 SecurityGroupPlIngressOracle: Type: AWS::EC2::SecurityGroupIngress DependsOn: SecurityGroup Properties: FromPort: 1521 Description : PrefixList for AWS IP addresses connected through VPN. SourcePrefixListId: pl-4e2ece27 IpProtocol: tcp GroupId: !Ref SecurityGroup ToPort: 1521 OracleRDSSubnetGroup: Type: AWS::RDS::DBSubnetGroup Properties: DBSubnetGroupDescription: Subnets available for the DMS Lab SubnetIds: - Ref: SubnetAPublic - Ref: SubnetBPublic SourceOracleDB: Type: AWS::RDS::DBInstance DependsOn: - OracleRDSSubnetGroup - SecurityGroup Properties: DBName: "OracleDB" AllocatedStorage: "100" StorageType: gp2 MasterUsername: Ref: Session MasterUserPassword: 'Password1' DBInstanceClass: 'db.r5.xlarge' Engine: oracle-ee EngineVersion: '12.1.0.2.v6' LicenseModel: bring-your-own-license PubliclyAccessible: true AvailabilityZone: Fn::GetAtt: - SubnetBPublic - AvailabilityZone MultiAZ: false DBSubnetGroupName: Ref: OracleRDSSubnetGroup VPCSecurityGroups: - Fn::GetAtt: - SecurityGroup - GroupId DBSnapshotIdentifier: Fn::FindInMap: - RegionMap - !Ref AWS::Region - OracleSnapshotId DBInstanceIdentifier: Fn::Sub: "oracle-${SimpleStackUid}" SCTUser: Type: 'AWS::IAM::User' Properties: UserName: Fn::Sub: "sct-migration-user-${SimpleStackUid}" ManagedPolicyArns: - arn:aws:iam::aws:policy/AmazonRedshiftFullAccess - arn:aws:iam::aws:policy/AmazonEC2FullAccess - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/IAMReadOnlyAccess SCTAccessKey: Type: 'AWS::IAM::AccessKey' Properties: Status: 'Active' UserName: !Ref SCTUser StagingS3Bucket: Type: 'AWS::S3::Bucket' Properties: AccessControl: Private Outputs: BucketName: Value: !Ref S3Bucket Description: Name of the sample Amazon S3 bucket with a notification configuration. Outputs: # RedshiftClusterIdentifier: # Description: "The identifier of the Amazon Redshift cluster, like: examplecluster" # Value: # Ref: RedshiftCluster RedshiftJDBCEndpoint: Description: "The jdbc endpoint to connect with JDBC tools like workbench like jdbc:redshift://examplecluster.cg034hpkmmjt.us-east-1.redshift.amazonaws.com:5439/dev" Value: !Sub "jdbc:redshift://${RedshiftCluster.Endpoint.Address}:5439/dev" RedshiftEndpoint: Description: "The connection endpoint for the Amazon Redshift cluster, like: examplecluster.cg034hpkmmjt.us-east-1.redshift.amazonaws.com" Value: Fn::GetAtt: - RedshiftCluster - Endpoint.Address # RedshiftClusterPort: # Description: "The port number on which the Amazon Redshift cluster accepts connections, like: 5439" # Value: # Fn::GetAtt: # - RedshiftCluster # - Endpoint.Port RedshiftClusterPassword: Description: "The master user password" Value: Fn::Sub: "Pp-${SimpleStackUid}" # RedshiftClusterDatabase: # Description: "Redshift cluster database name" # Value: # Fn::Sub: "dev" EC2HostName: Description: Public DNS endpoint for the EC2 instance Value: Fn::GetAtt: - EC2Instance - PublicDnsName # EC2PrivateDNS: # Description: Private DNS endpoint for the EC2 instance # Value: # Fn::GetAtt: # - EC2Instance # - PrivateDnsName EC2DeveloperPassword: Description: "EC2 developer user password" Value: Fn::Sub: "Pp-${SimpleStackUid}" # EC2LoginUserName: # Description: "EC2 login user" # Value: 'developer' OracleEndpoint: Description: Source Oracle RDS Endpoint Value: Fn::GetAtt: - SourceOracleDB - Endpoint.Address # SourceOracleUser: # Description: "The user name for Oracle cluster." # Value: 'dbmaster' OraclePassword: Description: "The master user password" Value: 'Password1' VPC: Description: "Virtual Private Cloud for the lab" Value: Ref: VirtualPrivateCloud # SourceOracleSID: # Description: "Oracle SID" # Value: 'ORACLEDB' # SourceOraclePort: # Description: "Oracle Port number" # Value: '1521' SCTUserAccessKey: Description: "SCT extractor user Access Key" Value: !Ref SCTAccessKey SCTUserSecretAccessKey: Description: "SCT extractor user Secret Access Key" Value: !GetAtt SCTAccessKey.SecretAccessKey SCTS3BucketName: Value: !Ref StagingS3Bucket Description: Name of the Amazon S3 bucket