AWSTemplateFormatVersion: 2010-09-09
Description: CloudFormation template for importing files from S3.
Parameters:
    ReplicationInstance:
        Default: dms.c4.large
        AllowedValues:
            - dms.t2.micro
            - dms.t2.small
            - dms.t2.medium
            - dms.t2.large
            - dms.c4.large
            - dms.c4.xlarge
            - dms.c4.2xlarge
            - dms.c4.4xlarge
        Description: The instance type to use for the replication instance.
        Type: String
    ReplicationSecurityGroups:
        Default: "sg-12aa1234"
        Type: String
        Description: Security group in your VPC with ingress to your data & this security group.
    S3BucketName:
        Default: "mybucketname"
        Type: String
        Description: Bucket name where your data is stored
    ServiceRoleArn:
        Default: "arn:aws:iam::123456789012:role/dms-vpc-role"
        Type: String
        Description: Service role ARN with permissions to DMS, S3 & DynamoDB
Metadata:
    AWS::CloudFormation::Interface:
        ParameterGroups:
            - Label:
                default: Replication Instance Information
              Parameters:
                - ReplicationInstance
                - ReplicationSecurityGroups
                - S3BucketName
                - ServiceRoleArn
        ParameterLabels:
            ReplicationInstance:
                default: Instance class
            ReplicationSecurityGroups:
                default: Security group
            S3BucketName:
                default: S3 bucket name
            ServiceRoleArn:
                default: Service role arn
Resources:
  DMSReplicationInstance:
    Type: "AWS::DMS::ReplicationInstance"
    Properties:
      AllocatedStorage: 100
      MultiAZ: false
      PubliclyAccessible: false
      ReplicationInstanceClass:
        Ref: ReplicationInstance
      Tags:
        -
          Key: "Name"
          Value: "importFromS3"
        -
          Key: "project"
          Value: "exampleImport"
      VpcSecurityGroupIds:
        - Ref: ReplicationSecurityGroups
  DMSEndpointS3:
    Type: "AWS::DMS::Endpoint"
    Properties:
      EndpointType: "source"
      EngineName: "S3"
      S3Settings:
        CsvDelimiter: "/t"
        BucketName:
          Ref: S3BucketName
        ExternalTableDefinition: "{\"Tables\": [{\"TablePath\": \"title_basics/\", \"TableColumnsTotal\": \"9\", \"TableOwner\": \"imdb\", \"TableName\": \"title_basics\", \"TableColumns\": [{\"ColumnName\": \"title_const\", \"ColumnIsPk\": \"true\", \"ColumnNullable\": \"false\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"title_type\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"10\"}, {\"ColumnName\": \"primary_title\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}, {\"ColumnName\": \"original_title\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}, {\"ColumnName\": \"is_adult\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"10\"}, {\"ColumnName\": \"start_year\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"10\"}, {\"ColumnName\": \"end_year\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"10\"}, {\"ColumnName\": \"runtime_mins\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"20\"}, {\"ColumnName\": \"genres\", \"ColumnType\": \"STRING\", \"ColumnLength\": \"255\"}]}], \"TableCount\": \"1\"}"
        ServiceAccessRoleArn:
          Ref: ServiceRoleArn
        CompressionType: "GZIP"
      Tags:
        -
          Key: "Name"
          Value: "importFromS3"
        -
          Key: "project"
          Value: "exampleImport"
  DMSEndpointDynamoDB:
    Type: "AWS::DMS::Endpoint"
    Properties:
      DynamoDbSettings:
        ServiceAccessRoleArn:
          Ref: ServiceRoleArn
      EndpointType: "target"
      EngineName: "DYNAMODB"
      Tags:
        -
          Key: "Name"
          Value: "importFromS3"
        -
          Key: "project"
          Value: "exampleImport"
  DMSTaskMigration:
    Type: "AWS::DMS::ReplicationTask"
    Properties:
      MigrationType: "full-load"
      ReplicationInstanceArn:
        Ref: DMSReplicationInstance
      ReplicationTaskSettings: "{\"Logging\": {\"CloudWatchLogGroup\": \"dms-tasks-aurora-redshift-migration\", \"CloudWatchLogStream\": \"dms-task-DBCHWX72MDJRP2MT4MZN4RGIH4\", \"EnableLogging\": true, \"LogComponents\": [{\"Id\": \"SOURCE_UNLOAD\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"SOURCE_CAPTURE\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TARGET_LOAD\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TARGET_APPLY\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}, {\"Id\": \"TASK_MANAGER\", \"Severity\": \"LOGGER_SEVERITY_DEFAULT\"}]}, \"ControlTablesSettings\": {\"SuspendedTablesTableEnabled\": false, \"HistoryTimeslotInMinutes\": 5, \"historyTimeslotInMinutes\": 5, \"StatusTableEnabled\": false, \"HistoryTableEnabled\": false, \"ControlSchema\": \"\"}, \"ChangeProcessingTuning\": {\"BatchApplyTimeoutMax\": 30, \"BatchApplyPreserveTransaction\": true, \"MemoryKeepTime\": 60, \"BatchApplyMemoryLimit\": 500, \"BatchSplitSize\": 0, \"StatementCacheSize\": 50, \"MinTransactionSize\": 1000, \"MemoryLimitTotal\": 1024, \"BatchApplyTimeoutMin\": 1, \"CommitTimeout\": 1}, \"TargetMetadata\": {\"SupportLobs\": false, \"LoadMaxFileSize\": 0, \"LimitedSizeLobMode\": true, \"FullLobMode\": false, \"TargetSchema\": \"\", \"LobMaxSize\": 32, \"ParallelLoadThreads\": 0, \"LobChunkSize\": 64, \"ParallelLoadBufferSize\": 0, \"BatchApplyEnabled\": false}, \"ChangeProcessingDdlHandlingPolicy\": {\"HandleSourceTableTruncated\": true, \"HandleSourceTableAltered\": true, \"HandleSourceTableDropped\": true}, \"FullLoadSettings\": {\"CreatePkAfterFullLoad\": false, \"TargetTablePrepMode\": \"DROP_AND_CREATE\", \"StopTaskCachedChangesApplied\": false, \"TransactionConsistencyTimeout\": 600, \"StopTaskCachedChangesNotApplied\": false, \"MaxFullLoadSubTasks\": 8, \"CommitRate\": 10000}, \"ErrorBehavior\": {\"TableErrorEscalationPolicy\": \"STOP_TASK\", \"RecoverableErrorThrottlingMax\": 1800, \"ApplyErrorDeletePolicy\": \"IGNORE_RECORD\", \"ApplyErrorEscalationPolicy\": \"LOG_ERROR\", \"ApplyErrorUpdatePolicy\": \"LOG_ERROR\", \"DataTruncationErrorPolicy\": \"LOG_ERROR\", \"RecoverableErrorInterval\": 5, \"RecoverableErrorCount\": -1, \"FullLoadIgnoreConflicts\": true, \"DataErrorEscalationPolicy\": \"SUSPEND_TABLE\", \"ApplyErrorInsertPolicy\": \"LOG_ERROR\", \"ApplyErrorEscalationCount\": 0, \"RecoverableErrorThrottling\": true, \"TableErrorEscalationCount\": 0, \"TableErrorPolicy\": \"SUSPEND_TABLE\", \"DataErrorEscalationCount\": 0, \"DataErrorPolicy\": \"LOG_ERROR\"}, \"StreamBufferSettings\": {\"StreamBufferSizeInMB\": 8, \"StreamBufferCount\": 3, \"CtrlStreamBufferSizeInMB\": 5}}"
      SourceEndpointArn:
        Ref: DMSEndpointS3
      TableMappings: "{\"rules\": [{\"rule-type\": \"selection\", \"rule-id\": \"1\", \"rule-action\": \"include\", \"object-locator\": {\"schema-name\": \"%\", \"table-name\": \"%\"}, \"rule-name\": \"1\"}]}"
      Tags:
        -
          Key: "Name"
          Value: "importFromS3"
        -
          Key: "project"
          Value: "exampleImport"
      TargetEndpointArn:
        Ref: DMSEndpointDynamoDB