--- AWSTemplateFormatVersion: '2010-09-09' Description: 'Malware Detection Orchestration with AWS Batch using Fargate using CloudFormation' Parameters : AwsRegion: Type: String Default: "eu-west-1" BenignInputBucket: Type: String BenignMetadatadbBucket: Type: String BenignMetadataKey: Type: String BenignCatalogPrefix: Type: String MalwareBucketName: Type: String MalwareMetadataDbObject: Type: String MalwareMetadataDbObjectLocal: Type: String MalwareBinaryPrefix: Type: String MalwareCatalodObjectPrefix: Type: String MalwareMetadataDbObjectS3: Type: String Multiplier: Type: String ConvertMalwareBinaryToImagesArraySize: Type: Number ConvertBenignBinaryToImagesArraySize: Type: Number TrainingBucketPopulateForMalwarePrefixList: Type: String TrainingBucketPopulateForMalwareArraySize: Type: Number TrainingBucketPopulateForMalwareImageToTrain: Type: String TrainingBucketPopulateForBenignImageToTrain: Type: String TrainingBucketPopulateForBenignPrefixList: Type: String Resources: VPC: Type: AWS::EC2::VPC Properties: CidrBlock: 10.0.0.0/16 EnableDnsSupport: true EnableDnsHostnames: true VPCFlowLogRole: Type: 'AWS::IAM::Role' Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: 'vpc-flow-logs.amazonaws.com' Action: 'sts:AssumeRole' Policies: - PolicyName: 'flowlogs-policy' PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: - 'logs:CreateLogStream' - 'logs:PutLogEvents' - 'logs:DescribeLogGroups' - 'logs:DescribeLogStreams' Resource: "arn:aws:logs:*:*:log-group:/aws/*/*" VPCFlowLog: Type: AWS::EC2::FlowLog Properties: DeliverLogsPermissionArn: !GetAtt 'VPCFlowLogRole.Arn' LogGroupName: MalwareVPCFlowLog ResourceId: !Ref VPC ResourceType: VPC TrafficType: ALL InternetGateway: Type: AWS::EC2::InternetGateway PublicSubnetRouteTable: Type: AWS::EC2::RouteTable Properties: VpcId: !Ref VPC Tags: - Key: Name Value: !Sub ${AWS::StackName} Public Route - Key: Project Value: AWS Batch in Fargate VPCGatewayAttachment: Type: AWS::EC2::VPCGatewayAttachment Properties: VpcId: !Ref VPC InternetGatewayId: !Ref InternetGateway NATEIP: DependsOn: VPCGatewayAttachment Type: AWS::EC2::EIP Properties: Domain: vpc NATGateway: DependsOn: VPCGatewayAttachment Type: AWS::EC2::NatGateway Properties: AllocationId: !GetAtt 'NATEIP.AllocationId' SubnetId: !Ref PublicSubnet SecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: EC2 Security Group for instances launched in the VPC by Batch VpcId: !Ref VPC InboundRule: Type: AWS::EC2::SecurityGroupIngress Properties: GroupId: !GetAtt 'SecurityGroup.GroupId' Description: SecurityGroupIngress IpProtocol: "-1" FromPort: -1 ToPort: -1 CidrIp: !GetAtt 'VPC.CidrBlock' OutboundRule: Type: AWS::EC2::SecurityGroupEgress Properties: GroupId: !GetAtt 'SecurityGroup.GroupId' Description: SecurityGroupEgress IpProtocol: "-1" FromPort: -1 ToPort: -1 CidrIp: 0.0.0.0/0 PublicSubnet: Type: AWS::EC2::Subnet Properties: CidrBlock: 10.0.0.0/20 VpcId: !Ref VPC MapPublicIpOnLaunch: False Tags: - Key: Name Value: !Sub ${AWS::StackName} Public Subnet PrivateSubnet: Type: AWS::EC2::Subnet Properties: CidrBlock: 10.0.128.0/20 VpcId: !Ref VPC Tags: - Key: Name Value: !Sub ${AWS::StackName} Private Subnet PublicSubnetRoute: Type: AWS::EC2::Route DependsOn: VPCGatewayAttachment Properties: RouteTableId: !Ref PublicSubnetRouteTable DestinationCidrBlock: 0.0.0.0/0 GatewayId: !Ref InternetGateway PublicSubnetRouteTableAssociation: Type: AWS::EC2::SubnetRouteTableAssociation Properties: SubnetId: !Ref PublicSubnet RouteTableId: !Ref PublicSubnetRouteTable PrivateSubnetRouteTable: Type: AWS::EC2::RouteTable Properties: VpcId: !Ref VPC Tags: - Key: Name Value: !Sub ${AWS::StackName} Private Route - Key: Project Value: AWS Batch in Fargate PrivateSubnetRoute: Type: AWS::EC2::Route Properties: RouteTableId: !Ref 'PrivateSubnetRouteTable' DestinationCidrBlock: 0.0.0.0/0 NatGatewayId: !Ref NATGateway PrivateSubnetRouteTableAssociation: Type: AWS::EC2::SubnetRouteTableAssociation Properties: SubnetId: !Ref PrivateSubnet RouteTableId: !Ref PrivateSubnetRouteTable BatchServiceRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: batch.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole KmsKeyId: Type: AWS::KMS::Key Properties: Enabled: true EnableKeyRotation: true KeyPolicy: Version: 2012-10-17 Id: default Statement: - Sid: Enable IAM User Permissions Effect: Allow Principal: AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root' Action: 'kms:*' Resource: '*' KeySpec: SYMMETRIC_DEFAULT BatchLogGroupKmsKeyId: Type: AWS::KMS::Key Properties: Enabled: true EnableKeyRotation: true KeyPolicy: Version: 2012-10-17 Id: default Statement: - Sid: Enable IAM User Permissions Effect: Allow Principal: AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root' Action: 'kms:*' Resource: '*' KeySpec: SYMMETRIC_DEFAULT MalwareCatalogS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "malware-catalog-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId MalwareCatalogS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref MalwareCatalogS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${MalwareCatalogS3Bucket}' - !Sub 'arn:aws:s3:::${MalwareCatalogS3Bucket}/*' BenignCatalogS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "benign-catalog-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId BenignCatalogS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref BenignCatalogS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${BenignCatalogS3Bucket}' - !Sub 'arn:aws:s3:::${BenignCatalogS3Bucket}/*' MetadataDbS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "metadatadb-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId MetadataDbS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref MetadataDbS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${MetadataDbS3Bucket}' - !Sub 'arn:aws:s3:::${MetadataDbS3Bucket}/*' ObjectImageS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "objectimage-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId ObjectImageS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref ObjectImageS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${ObjectImageS3Bucket}' - !Sub 'arn:aws:s3:::${ObjectImageS3Bucket}/*' MalwareDetectionTrainingS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "malware-detection-training-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId MalwareDetectionTrainingS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref MalwareDetectionTrainingS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${MalwareDetectionTrainingS3Bucket}' - !Sub 'arn:aws:s3:::${MalwareDetectionTrainingS3Bucket}/*' MalwareClassificationTrainingS3Bucket: Type: 'AWS::S3::Bucket' Properties: BucketName: !Sub "malware-classification-training-${AWS::AccountId}-${AWS::Region}" PublicAccessBlockConfiguration: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true RestrictPublicBuckets: true BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: SSEAlgorithm: aws:kms KMSMasterKeyID: !Ref KmsKeyId MalwareClassificationTrainingS3BucketPolicy: Type: AWS::S3::BucketPolicy Properties: Bucket: !Ref MalwareClassificationTrainingS3Bucket PolicyDocument: Version: 2012-10-17 Statement: - Action: - s3:PutObject - s3:GetObject - s3:ListBucket Effect: Allow Principal: AWS: !GetAtt 'BatchTaskExecutionRole.Arn' Resource: - !Sub 'arn:aws:s3:::${MalwareClassificationTrainingS3Bucket}' - !Sub 'arn:aws:s3:::${MalwareClassificationTrainingS3Bucket}/*' ECRRepository: Type: AWS::ECR::Repository Properties: ImageScanningConfiguration: ScanOnPush: True ImageTagMutability: 'MUTABLE' Tags: - Key: "stackname" Value: !Sub ${AWS::StackName} MalwareMetadataJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-malware-metadatadb" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Sub "${MalwareBucketName}" Name: "BUCKET_NAME" - Value: !Sub "${MalwareMetadataDbObject}" Name: "OBJECT_NAME" - Value: !Sub "${MalwareMetadataDbObjectLocal}" Name: "FILE_NAME" - Value: !Ref MetadataDbS3Bucket Name: "METADATADB_BUCKET" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "4" Type: VCPU - Value: "30720" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-malware-metadata.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} RawDataCatalogJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-malware-data-catalog" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Sub "${MalwareBucketName}" Name: "INPUT_BUCKET" - Value: !Sub ${MalwareBinaryPrefix} Name: "INPUT_BUCKET_PREFIX" - Value: !Ref MalwareCatalogS3Bucket Name: "OUTPUT_BUCKET" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "2" Type: VCPU - Value: "16384" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-raw-data-catalog.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} ObjectImageConvertionJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-malware-object-image-convertion" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Ref MalwareCatalogS3Bucket Name: "INPUT_BUCKET_CSV" - Value: !Sub ${MalwareCatalodObjectPrefix} Name: "INPUT_BUCKET_CSV_PREFIX" - Value: !Sub "${MalwareBucketName}" Name: "INPUT_BUCKET" - Value: !Ref ObjectImageS3Bucket Name: "IMAGE_BUCKET" - Value: !Ref MetadataDbS3Bucket Name: "METADATADB_BUCKET" - Value: !Sub ${MalwareMetadataDbObjectS3} Name: "METADATA_KEY" - Value: "${Multiplier}" Name: "MULTIPLIER" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "2" Type: VCPU - Value: "16384" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-object-image-converter.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} BenignDataCatalogJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-benign-data-catalog" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Ref BenignInputBucket Name: "INPUT_BUCKET" - Value: !Ref BenignCatalogS3Bucket Name: "OUTPUT_BUCKET" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "2" Type: VCPU - Value: "8192" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-benign-data-catalog.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} BenignObjectImageConvertionJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-benign-object-image-convertion" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Ref BenignCatalogS3Bucket Name: "INPUT_BUCKET_CSV" - Value: !Sub ${BenignCatalogPrefix} Name: "INPUT_BUCKET_CSV_PREFIX" - Value: !Sub ${BenignInputBucket} Name: "INPUT_BUCKET" - Value: !Ref ObjectImageS3Bucket Name: "IMAGE_BUCKET" - Value: !Ref MetadataDbS3Bucket Name: "METADATADB_BUCKET" - Value: !Sub ${BenignMetadataKey} Name: "METADATA_KEY" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "2" Type: VCPU - Value: "8192" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-benign-image-converter.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} TrainBucketPopulatorJobDefinition: Type: AWS::Batch::JobDefinition Properties: Type: container PropagateTags: true JobDefinitionName: !Sub "${AWS::StackName}-training-bucket-populator" ContainerProperties: Environment: - Value: !Sub ${AWS::Region} Name: "AWS_REGION" - Value: !Ref ObjectImageS3Bucket Name: "IMAGE_BUCKET" - Value: !Ref MalwareDetectionTrainingS3Bucket Name: "TRAINING_BUCKET" - Value: !Sub ${TrainingBucketPopulateForMalwareImageToTrain} Name: "IMAGES_TO_TRAIN" - Value: "[]" Name: "PREFIX_LIST" Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}:latest" FargatePlatformConfiguration: PlatformVersion: LATEST ResourceRequirements: - Value: "2" Type: VCPU - Value: "8192" Type: MEMORY JobRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' ExecutionRoleArn: !GetAtt 'BatchTaskExecutionRole.Arn' LogConfiguration: LogDriver: awslogs Options: awslogs-group: !Ref 'BatchLogGroup' awslogs-region: !Ref AWS::Region awslogs-stream-prefix: !Sub ${AWS::StackName}-logs Command: - python3 - /malware_detection_scripts/malware-detection-training-bucket-populator.py PlatformCapabilities: - FARGATE Tags: Service: Batch Name: !Sub ${AWS::StackName} BatchLogGroup: Type: AWS::Logs::LogGroup Properties: LogGroupName: !Sub ${AWS::StackName}-awslogs RetentionInDays: 7 BatchTaskExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: [ecs-tasks.amazonaws.com] Action: ['sts:AssumeRole'] Path: / Policies: - PolicyName: AmazonECSECRPolicy PolicyDocument: Statement: - Effect: Allow Action: - 'ecr:GetAuthorizationToken' - 'ecr:BatchCheckLayerAvailability' - 'ecr:GetDownloadUrlForLayer' - 'ecr:BatchGetImage' Resource: !Sub "arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ECRRepository}" - PolicyName: GetAuthorizationTokenPolicy PolicyDocument: Statement: - Effect: Allow Action: - 'ecr:GetAuthorizationToken' Resource: "*" - PolicyName: AmazonECSLogsPolicy PolicyDocument: Statement: - Effect: Allow Action: - 'logs:CreateLogStream' - 'logs:PutLogEvents' Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*" - PolicyName: !Sub ${AWS::StackName}-ecs-task-s3-get-policy PolicyDocument: Statement: - Effect: Allow Action: - s3:PutObject - s3:GetObject - s3:ListBucket - s3:HeadObject Resource: [ !Sub 'arn:aws:s3:::${MalwareCatalogS3Bucket}', !Sub 'arn:aws:s3:::${MalwareCatalogS3Bucket}/*', !Sub 'arn:aws:s3:::${BenignCatalogS3Bucket}', !Sub 'arn:aws:s3:::${BenignCatalogS3Bucket}/*', !Sub 'arn:aws:s3:::${MetadataDbS3Bucket}', !Sub 'arn:aws:s3:::${MetadataDbS3Bucket}/*', !Sub 'arn:aws:s3:::${ObjectImageS3Bucket}', !Sub 'arn:aws:s3:::${ObjectImageS3Bucket}/*', !Sub 'arn:aws:s3:::${MalwareDetectionTrainingS3Bucket}', !Sub 'arn:aws:s3:::${MalwareDetectionTrainingS3Bucket}/*', !Sub 'arn:aws:s3:::${MalwareClassificationTrainingS3Bucket}', !Sub 'arn:aws:s3:::${MalwareClassificationTrainingS3Bucket}/*', !Sub 'arn:aws:s3:::${BenignInputBucket}', !Sub 'arn:aws:s3:::${BenignInputBucket}/*', !Sub 'arn:aws:s3:::${BenignMetadatadbBucket}', !Sub 'arn:aws:s3:::${BenignMetadatadbBucket}/*', !Sub 'arn:aws:s3:::${MalwareBucketName}', !Sub 'arn:aws:s3:::${MalwareBucketName}/*', !Sub 'arn:aws:s3:::${MalwareCatalodObjectPrefix}', !Sub 'arn:aws:s3:::${MalwareCatalodObjectPrefix}/*' ] - PolicyName: !Sub ${AWS::StackName}-kms-policy PolicyDocument: Statement: - Effect: Allow Action: - 'kms:DescribeKey' - 'kms:Decrypt' - 'kms:Encrypt' - 'kms:GenerateDataKey' - 'kms:CreateGrant' - 'kms:ListGrants' Resource: - !GetAtt 'KmsKeyId.Arn' - !GetAtt 'BatchLogGroupKmsKeyId.Arn' BatchProcessingJobQueue: Type: AWS::Batch::JobQueue Properties: JobQueueName: !Sub "${AWS::StackName}-queue" State: ENABLED Priority: 1 ComputeEnvironmentOrder: - Order: 1 ComputeEnvironment: !Ref ComputeEnvironment ComputeEnvironment: Type: AWS::Batch::ComputeEnvironment Properties: ComputeEnvironmentName: !Sub "${AWS::StackName}" Type: MANAGED State: ENABLED ComputeResources: Type: FARGATE MaxvCpus: 1000 Subnets: - !Ref PrivateSubnet SecurityGroupIds: - !Ref SecurityGroup ServiceRole: !Ref BatchServiceRole MalwarePreprocessingStatesExecutionRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: "Allow" Principal: Service: - !Sub states.${AWS::Region}.amazonaws.com Action: "sts:AssumeRole" Path: "/" Policies: - PolicyName: CloudWatchLogsDeliveryFullAccessPolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "logs:CreateLogDelivery" - "logs:GetLogDelivery" - "logs:UpdateLogDelivery" - "logs:DeleteLogDelivery" - "logs:ListLogDeliveries" - "logs:PutResourcePolicy" - "logs:DescribeResourcePolicies" - "logs:DescribeLogGroups" Resource: "arn:aws:logs:*:*:log-group:/aws/*/*" - PolicyName: BatchJobManagementFullAccessPolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "batch:SubmitJob" - "batch:DescribeJobs" - "batch:TerminateJob" Resource: [ !Sub "${MalwareMetadataJobDefinition}", !Sub "${RawDataCatalogJobDefinition}", !Sub "${ObjectImageConvertionJobDefinition}", !Sub "${TrainBucketPopulatorJobDefinition}", !Sub "${BenignObjectImageConvertionJobDefinition}", !Sub "${BenignDataCatalogJobDefinition}", !Sub "${BatchProcessingJobQueue}" ] - PolicyName: EventRuleAccessPolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - "events:PutTargets" - "events:PutRule" - "events:DescribeRule" Resource: [ !Sub "arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/*" ] MalwarePreprocessingStateMachine: Type: "AWS::StepFunctions::StateMachine" Properties: DefinitionString: !Sub | { "Comment": "A description of my state machine", "StartAt": "ChoiceSkipPreprocessing", "States": { "ChoiceSkipPreprocessing": { "Type": "Choice", "Choices": [ { "Variable": "$.ByPassPreprocessing", "IsPresent": true, "Next": "Parallel-1" } ], "Default": "Parallel" }, "Parallel": { "Type": "Parallel", "Branches": [ { "StartAt": "Choice", "States": { "Choice": { "Type": "Choice", "Choices": [ { "Variable": "$.ByPassMetaDb", "IsPresent": true, "Next": "Convert Malware Binary To Images" } ], "Default": "Malware Metadata Db" }, "Malware Metadata Db": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "malware-metadata-db", "JobDefinition": "${MalwareMetadataJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}" }, "Next": "Malware Catalog", "ResultPath": null }, "Malware Catalog": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "malware-catalog", "JobDefinition": "${RawDataCatalogJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}" }, "Next": "Convert Malware Binary To Images", "ResultPath": null }, "Convert Malware Binary To Images": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "convert-malware-binary-to-images", "JobDefinition": "${ObjectImageConvertionJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}", "ContainerOverrides": { "Environment": [ { "Name": "AWS_REGION", "Value": "${AwsRegion}" }, { "Name": "INPUT_BUCKET_CSV", "Value": "${MalwareCatalogS3Bucket}" }, { "Name": "INPUT_BUCKET_CSV_PREFIX", "Value": "${MalwareCatalodObjectPrefix}" }, { "Name": "INPUT_BUCKET", "Value": "${MalwareBucketName}" }, { "Name": "IMAGE_BUCKET", "Value": "${ObjectImageS3Bucket}" }, { "Name": "METADATADB_BUCKET", "Value": "${MetadataDbS3Bucket}" }, { "Name": "METADATA_KEY", "Value": "${MalwareMetadataDbObjectS3}" }, { "Name": "MULTIPLIER", "Value": "${Multiplier}" } ] }, "ArrayProperties": { "Size": 500 } }, "End": true } } }, { "StartAt": "Benign Catalog", "States": { "Benign Catalog": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "benign-catalog", "JobDefinition": "${BenignDataCatalogJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}", "ContainerOverrides": { "Environment": [ { "Name": "AWS_REGION", "Value": "${AwsRegion}" }, { "Name": "INPUT_BUCKET", "Value": "${BenignInputBucket}" }, { "Name": "OUTPUT_BUCKET", "Value": "${BenignCatalogS3Bucket}" } ] } }, "Next": "Convert Benign Binary To Images", "ResultPath": null }, "Convert Benign Binary To Images": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "convert-benign-binary-to-images", "JobDefinition": "${BenignObjectImageConvertionJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}", "ContainerOverrides": { "Environment": [ { "Name": "AWS_REGION", "Value": "${AwsRegion}" }, { "Name": "INPUT_BUCKET_CSV", "Value": "${BenignCatalogS3Bucket}" }, { "Name": "INPUT_BUCKET_CSV_PREFIX", "Value": "${BenignCatalogPrefix}" }, { "Name": "INPUT_BUCKET", "Value": "${BenignInputBucket}" }, { "Name": "IMAGE_BUCKET", "Value": "${ObjectImageS3Bucket}" }, { "Name": "METADATADB_BUCKET", "Value": "${BenignMetadatadbBucket}" }, { "Name": "METADATA_KEY", "Value": "${BenignMetadataKey}" } ] }, "ArrayProperties": { "Size": 160 } }, "End": true } } } ], "Next": "Parallel-1", "ResultPath": null }, "Parallel-1": { "Type": "Parallel", "Branches": [ { "StartAt": "Training Bucket Populate For Malware", "States": { "Training Bucket Populate For Malware": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "training-bucket-populate-for-malware", "JobDefinition": "${TrainBucketPopulatorJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}", "ArrayProperties": { "Size": 11 }, "ContainerOverrides": { "Environment": [ { "Name": "AWS_REGION", "Value": "${AwsRegion}" }, { "Name": "IMAGE_BUCKET", "Value": "${ObjectImageS3Bucket}" }, { "Name": "TRAINING_BUCKET", "Value": "${MalwareDetectionTrainingS3Bucket}" }, { "Name": "IMAGES_TO_TRAIN", "Value": "${TrainingBucketPopulateForMalwareImageToTrain}" }, { "Name": "PREFIX_LIST", "Value": "${TrainingBucketPopulateForMalwarePrefixList}" } ] } }, "End": true, "ResultPath": null } } }, { "StartAt": "Training Bucket Populate For Benign", "States": { "Training Bucket Populate For Benign": { "Type": "Task", "Resource": "arn:aws:states:::batch:submitJob.sync", "Parameters": { "JobName": "training-bucket-populate-for-benign", "JobDefinition": "${TrainBucketPopulatorJobDefinition}", "JobQueue": "${BatchProcessingJobQueue}", "ContainerOverrides": { "Environment": [ { "Name": "AWS_REGION", "Value": "${AwsRegion}" }, { "Name": "IMAGE_BUCKET", "Value": "${ObjectImageS3Bucket}" }, { "Name": "TRAINING_BUCKET", "Value": "${MalwareDetectionTrainingS3Bucket}" }, { "Name": "IMAGES_TO_TRAIN", "Value": "${TrainingBucketPopulateForBenignImageToTrain}" }, { "Name": "PREFIX_LIST", "Value": "${TrainingBucketPopulateForBenignPrefixList}" } ] } }, "End": true } } } ], "End": true } } } RoleArn: !GetAtt 'MalwarePreprocessingStatesExecutionRole.Arn' #### VPC Endpoints S3VPCEndpoint: Type: "AWS::EC2::VPCEndpoint" Properties: VpcEndpointType: Interface ServiceName: !Sub "com.amazonaws.${AWS::Region}.s3" VpcId: !Ref VPC SubnetIds: - !Ref PrivateSubnet SecurityGroupIds: - !Ref SecurityGroup PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: '*' Action: - 's3:*' Resource: - !Join - '' - - 'arn:aws:s3:::' - !Sub "${AWS::StackName}-${AWS::AccountId}" EcrApiVPCEndpoint: Type: "AWS::EC2::VPCEndpoint" Properties: VpcEndpointType: Interface ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.api" VpcId: !Ref VPC SubnetIds: - !Ref PrivateSubnet SecurityGroupIds: - !Ref SecurityGroup PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: '*' Action: - 'ecr:*' Resource: - !Join - '' - - 'arn:aws:ecr:' - !Sub "${AWS::Region}:${AWS::AccountId}:repository/${AWS::StackName}" EcrDkrVPCEndpoint: Type: "AWS::EC2::VPCEndpoint" Properties: VpcEndpointType: Interface ServiceName: !Sub "com.amazonaws.${AWS::Region}.ecr.dkr" VpcId: !Ref VPC SubnetIds: - !Ref PrivateSubnet SecurityGroupIds: - !Ref SecurityGroup PolicyDocument: Version: 2012-10-17 Id: default Statement: - Effect: Allow Principal: '*' Action: - 'ecr:*' Resource: - !Join - '' - - 'arn:aws:ecr:' - !Sub "${AWS::Region}:${AWS::AccountId}:repository/${AWS::StackName}" SecretManagerVPCEndpoint: Type: "AWS::EC2::VPCEndpoint" Properties: VpcEndpointType: Interface ServiceName: !Sub "com.amazonaws.${AWS::Region}.secretsmanager" VpcId: !Ref VPC SubnetIds: - !Ref PrivateSubnet SecurityGroupIds: - !Ref SecurityGroup PolicyDocument: Version: 2012-10-17 Id: default Statement: - Effect: Allow Principal: '*' Action: - 'secretsmanager:*' Resource: - '*' Outputs: StackName: Description: 'Stack name.' Value: !Sub '${AWS::StackName}' IPAddress: Description: 'The public IP address of the NAT gateway.' Value: !Ref NATEIP Export: Name: !Sub '${AWS::StackName}-IPAddress' ComputeEnvironmentArn: Value: !Ref ComputeEnvironment BatchProcessingJobQueueArn: Value: !Ref BatchProcessingJobQueue RawDataCatalogJobDefinitionArn: Value: !Ref RawDataCatalogJobDefinition ObjectImageConvertionJobDefinitionArn: Value: !Ref ObjectImageConvertionJobDefinition BenignDataCatalogJobDefinitionArn: Value: !Ref BenignDataCatalogJobDefinition BenignObjectImageConvertionJobDefinitionArn: Value: !Ref BenignObjectImageConvertionJobDefinition TrainBucketPopulatorJobDefinitionArn: Value: !Ref TrainBucketPopulatorJobDefinition ECRRepositoryArn: Value: !Ref ECRRepository