AWSTemplateFormatVersion: 2010-09-09 Description: Create DMS repliaction Instance and task for data replication from Source S3 bucket to Target Dynamo tables. Parameters: ReplicationInstance: Default: dms.t2.large Description: The instance type to use for the replication instance. Type: String DMSRoleName: Default: dms-vpc-role Type: String Description: role name for DMS tasks Resources: DMSReplicationSubnetGroup: Type: AWS::DMS::ReplicationSubnetGroup Properties: ReplicationSubnetGroupIdentifier: "dms-subnet-grp" ReplicationSubnetGroupDescription: "dms subnet group" SubnetIds: - !ImportValue SubnetIdExport1 - !ImportValue SubnetIdExport2 Tags: - Key: "Name" Value: "importFromS3" DMSReplicationInstance: Type: AWS::DMS::ReplicationInstance Properties: AllocatedStorage: 100 MultiAZ: false PubliclyAccessible: false ReplicationInstanceClass: Ref: ReplicationInstance ReplicationSubnetGroupIdentifier: Ref: DMSReplicationSubnetGroup Tags: - Key: "Name" Value: "importFromS3" - Key: "project" Value: "exampleImport" VpcSecurityGroupIds: - !ImportValue NoIngressSecurityGroupExport DMSEndpointS3: Type: "AWS::DMS::Endpoint" Properties: EndpointType: "source" EngineName: "S3" S3Settings: CsvDelimiter: "," BucketName: !ImportValue DMSSourceBucketExport ExternalTableDefinition: "{\"Tables\": [{\"TablePath\": \"data/tags/\", \"TableColumnsTotal\": \"4\", \"TableOwner\": \"data\", \"TableName\": \"tags\", \"TableColumns\": [{\"ColumnName\": \"userId\", \"ColumnIsPk\": \"true\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"movieId\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"tag\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}, {\"ColumnName\": \"timestamp\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}]}, {\"TablePath\": \"data/ratings/\", \"TableColumnsTotal\": \"4\", \"TableOwner\": \"data\", \"TableName\": \"ratings\", \"TableColumns\": [{\"ColumnName\": \"userId\", \"ColumnIsPk\": \"true\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"movieId\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"rating\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"5\"}, {\"ColumnName\": \"timestamp\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}]}, {\"TablePath\": \"data/links/\", \"TableColumnsTotal\": \"3\", \"TableOwner\": \"data\", \"TableName\": \"links\", \"TableColumns\": [{\"ColumnName\": \"movieId\", \"ColumnIsPk\": \"true\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"imdbId\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"tmdbId\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}]}, {\"TablePath\": \"data/movies/\", \"TableColumnsTotal\": \"3\", \"TableOwner\": \"data\", \"TableName\": \"movies\", \"TableColumns\": [{\"ColumnName\": \"movieId\", \"ColumnIsPk\": \"true\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"title\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}, {\"ColumnName\": \"genres\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}]}], \"TableCount\": \"4\"}" ServiceAccessRoleArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/${DMSRoleName}" CompressionType: "NONE" Tags: - Key: "Name" Value: "importFromS3" - Key: "project" Value: "exampleImport" DMSEndpointDynamoDB: Type: "AWS::DMS::Endpoint" Properties: DynamoDbSettings: ServiceAccessRoleArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/${DMSRoleName}" EndpointType: "target" EngineName: "DYNAMODB" Tags: - Key: "Name" Value: "importFromS3" - Key: "project" Value: "exampleImport" DMSTaskMigration: Type: "AWS::DMS::ReplicationTask" Properties: MigrationType: "full-load" ReplicationInstanceArn: Ref: DMSReplicationInstance ReplicationTaskSettings: "{\"Logging\": {\"CloudWatchLogGroup\": \"dms-tasks-aurora-redshift-migration\", \"CloudWatchLogStream\": \"dms-task-DBCHWX72MDJRP2MT4MZN4RGIH4\", \"EnableLogging\": true, \"LogComponents\": [{\"Id\": \"SOURCE_UNLOAD\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"SOURCE_CAPTURE\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TARGET_LOAD\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TARGET_APPLY\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TASK_MANAGER\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}]}, \"ControlTablesSettings\": {\"SuspendedTablesTableEnabled\": false, \"HistoryTimeslotInMinutes\": 5, \"historyTimeslotInMinutes\": 5, \"StatusTableEnabled\": false, \"HistoryTableEnabled\": false, \"ControlSchema\": \"\"}, \"ChangeProcessingTuning\": {\"BatchApplyTimeoutMax\": 30, \"BatchApplyPreserveTransaction\": true, \"MemoryKeepTime\": 60, \"BatchApplyMemoryLimit\": 500, \"BatchSplitSize\": 0, \"StatementCacheSize\": 50, \"MinTransactionSize\": 1000, \"MemoryLimitTotal\": 1024, \"BatchApplyTimeoutMin\": 1, \"CommitTimeout\": 1}, \"TargetMetadata\": {\"SupportLobs\": false, \"LoadMaxFileSize\": 0, \"LimitedSizeLobMode\": true, \"FullLobMode\": false, \"TargetSchema\": \"\", \"LobMaxSize\": 32, \"ParallelLoadThreads\": 0, \"LobChunkSize\": 64, \"ParallelLoadBufferSize\": 0, \"BatchApplyEnabled\": false}, \"ChangeProcessingDdlHandlingPolicy\": {\"HandleSourceTableTruncated\": true, \"HandleSourceTableAltered\": true, \"HandleSourceTableDropped\": true}, \"FullLoadSettings\": {\"CreatePkAfterFullLoad\": false, \"TargetTablePrepMode\": \"DROP_AND_CREATE\", \"StopTaskCachedChangesApplied\": false, \"TransactionConsistencyTimeout\": 600, \"StopTaskCachedChangesNotApplied\": false, \"MaxFullLoadSubTasks\": 8, \"CommitRate\": 10000}, \"ErrorBehavior\": {\"TableErrorEscalationPolicy\": \"STOP_TASK\", \"RecoverableErrorThrottlingMax\": 1800, \"ApplyErrorDeletePolicy\": \"IGNORE_RECORD\", \"ApplyErrorEscalationPolicy\": \"LOG_ERROR\", \"ApplyErrorUpdatePolicy\": \"LOG_ERROR\", \"DataTruncationErrorPolicy\": \"LOG_ERROR\", \"RecoverableErrorInterval\": 5, \"RecoverableErrorCount\": -1, \"FullLoadIgnoreConflicts\": true, \"DataErrorEscalationPolicy\": \"SUSPEND_TABLE\", \"ApplyErrorInsertPolicy\": \"LOG_ERROR\", \"ApplyErrorEscalationCount\": 0, \"RecoverableErrorThrottling\": true, \"TableErrorEscalationCount\": 0, \"TableErrorPolicy\": \"SUSPEND_TABLE\", \"DataErrorEscalationCount\": 0, \"DataErrorPolicy\": \"LOG_ERROR\"}, \"StreamBufferSettings\": {\"StreamBufferSizeInMB\": 8, \"StreamBufferCount\": 3, \"CtrlStreamBufferSizeInMB\": 5}}" SourceEndpointArn: Ref: DMSEndpointS3 TableMappings: "{\"rules\": [{\"rule-type\": \"selection\", \"rule-id\": \"1\", \"rule-action\": \"include\", \"object-locator\": {\"schema-name\": \"data\", \"table-name\": \"%\"}, \"rule-name\": \"1\"}, {\"rule-action\": \"map-record-to-record\", \"object-locator\": {\"schema-name\": \"data\", \"table-name\": \"tags\"}, \"target-table-name\": \"tags_t\", \"rule-name\": \"2\", \"rule-type\": \"object-mapping\", \"rule-id\": \"2\", \"mapping-parameters\": {\"attribute-mappings\": [{\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"Id\", \"value\": \"${userId}X${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"userId\", \"value\": \"${userId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"movieId\", \"value\": \"${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"tags\", \"value\": \"${tags}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"timestamp_c\", \"value\": \"${timestamp}\"}], \"exclude-columns\": [\"userId\", \"movieId\", \"tags\", \"timestamp\"], \"partition-key-name\": \"Id\", \"sort-key-name\": \"movieId\"}}, {\"rule-action\": \"map-record-to-record\", \"object-locator\": {\"schema-name\": \"data\", \"table-name\": \"ratings\"}, \"target-table-name\": \"ratings_t\", \"rule-name\": \"3\", \"rule-type\": \"object-mapping\", \"rule-id\": \"3\", \"mapping-parameters\": {\"attribute-mappings\": [{\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"Id\", \"value\": \"${userId}X${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"userId\", \"value\": \"${userId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"movieId\", \"value\": \"${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"rating\", \"value\": \"${rating}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"timestamp_c\", \"value\": \"${timestamp}\"}], \"exclude-columns\": [\"userId\", \"movieId\", \"rating\", \"timestamp\"], \"partition-key-name\": \"Id\", \"sort-key-name\": \"movieId\"}}, {\"rule-action\": \"map-record-to-record\", \"object-locator\": {\"schema-name\": \"data\", \"table-name\": \"movies\"}, \"target-table-name\": \"movies_t\", \"rule-name\": \"4\", \"rule-type\": \"object-mapping\", \"rule-id\": \"4\", \"mapping-parameters\": {\"attribute-mappings\": [{\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"movieId\", \"value\": \"${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"title\", \"value\": \"${title}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"genres\", \"value\": \"${genres}\"}], \"exclude-columns\": [\"movieId\", \"title\", \"genres\"], \"partition-key-name\": \"movieId\", \"sort-key-name\": \"genres\"}}, {\"rule-action\": \"map-record-to-record\", \"object-locator\": {\"schema-name\": \"data\", \"table-name\": \"links\"}, \"target-table-name\": \"links_t\", \"rule-name\": \"5\", \"rule-type\": \"object-mapping\", \"rule-id\": \"5\", \"mapping-parameters\": {\"attribute-mappings\": [{\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"movieId\", \"value\": \"${movieId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"imdbId\", \"value\": \"${imdbId}\"}, {\"attribute-type\": \"scalar\", \"attribute-sub-type\": \"string\", \"target-attribute-name\": \"tmdbId\", \"value\": \"${tmdbId}\"}], \"exclude-columns\": [\"movieId\", \"imdbId\", \"tmdbId\"], \"partition-key-name\": \"movieId\", \"sort-key-name\": \"imdbId\"}}]}" Tags: - Key: "Name" Value: "importFromS3" - Key: "project" Value: "exampleImport" TargetEndpointArn: Ref: DMSEndpointDynamoDB Outputs: DMSTaskMigration: Description: A reference to the created VPC Value: !Ref DMSTaskMigration Export : Name: DMSTaskExport