AWSTemplateFormatVersion: "2010-09-09" Description: "Template to create resources for Security Group rules analysis" Parameters: scriptsLocation: Type: String Default: "" librariesLocation: Type: String Default: "" quicksightUserArn: Type: String Default: "" Resources: IAMRole: Type: "AWS::IAM::Role" Properties: Path: "/service-role/" RoleName: "sg-analysis-step-function-role" AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" MaxSessionDuration: 3600 ManagedPolicyArns: - !Sub "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" - !Sub "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" - !Sub "arn:aws:iam::aws:policy/AWSXrayFullAccess" IAMManagedPolicy: Type: "AWS::IAM::ManagedPolicy" Properties: ManagedPolicyName: "SG_Analysis_Amazon_EventBridge_Invoke_Step_Functions" Path: "/service-role/" PolicyDocument: !Sub | { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "states:StartExecution" ], "Resource": [ "${StepFunctionsStateMachine}" ] } ] } IAMRole2: Type: "AWS::IAM::Role" Properties: Path: "/service-role/" RoleName: "SG_Analysis_Glue_Access_Role" AssumeRolePolicyDocument: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"glue.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" MaxSessionDuration: 3600 ManagedPolicyArns: - "arn:aws:iam::aws:policy/AmazonEC2FullAccess" - "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole" - "arn:aws:iam::aws:policy/AmazonSESFullAccess" - "arn:aws:iam::aws:policy/CloudWatchFullAccess" - "arn:aws:iam::aws:policy/AmazonAthenaFullAccess" - "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" - "arn:aws:iam::aws:policy/AmazonS3FullAccess" StepFuncLogGroup: Type: "AWS::Logs::LogGroup" Properties: LogGroupName: "/aws/vendedlogs/states/sg-analysis-step-function-logs" StepFunctionsStateMachine: Type: "AWS::StepFunctions::StateMachine" Properties: StateMachineName: !Ref EventsRule DefinitionString: | { "Comment": "A description of my state machine", "StartAt": "Glue CreateDynamoDBTables", "States": { "Glue CreateDynamoDBTables": { "Type": "Task", "Resource": "arn:aws:states:::glue:startJobRun.sync", "Parameters": { "JobName": "sg-analysis-dynamodb-tables" }, "Next": "Glue GetSGRulesData" }, "Glue GetSGRulesData": { "Type": "Task", "Resource": "arn:aws:states:::glue:startJobRun.sync", "Parameters": { "JobName": "sg-analysis-get-rules-data" }, "Next": "Glue QueryVPCFlowLogsSaveToS3" }, "Glue QueryVPCFlowLogsSaveToS3": { "Type": "Task", "Resource": "arn:aws:states:::glue:startJobRun.sync", "Parameters": { "JobName": "sg-analysis-run-athena-query" }, "Next": "Glue ParseVPCFLowLogsSaveUsageCount" }, "Glue ParseVPCFLowLogsSaveUsageCount": { "Type": "Task", "Resource": "arn:aws:states:::glue:startJobRun.sync", "Parameters": { "JobName": "sg-analysis-flow-logs-parser" }, "Next": "Glue EmailAnalysisReportOn1st" }, "Glue EmailAnalysisReportOn1st": { "Type": "Task", "Resource": "arn:aws:states:::glue:startJobRun.sync", "Parameters": { "JobName": "sg-analysis-email-usage-report" }, "End": true } } } RoleArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/service-role/sg-analysis-step-function-role" StateMachineType: "STANDARD" LoggingConfiguration: Destinations: - CloudWatchLogsLogGroup: LogGroupArn: !GetAtt StepFuncLogGroup.Arn IncludeExecutionData: true Level: "ALL" EventsRule: Type: "AWS::Events::Rule" Properties: Name: "sg-analysis-step-function" ScheduleExpression: "cron(0 9 * * ? *)" State: "DISABLED" Targets: - Arn: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:sg-analysis-step-function" Id: "Id70bcbcb6-9bda-4f24-8f45-8f3dbb65692c" RoleArn: !Sub "arn:aws:iam::${AWS::AccountId}:role/service-role/SG_Analysis_Amazon_EventBridge_Invoke_Step_Functions" EventBusName: "default" GlueJob1: Type: "AWS::Glue::Job" Properties: Name: "sg-analysis-run-athena-query" Description: "Job to run Athena query on VPC Flow logs and save to S3" Role: !GetAtt IAMRole2.Arn ExecutionProperty: MaxConcurrentRuns: 1 Command: Name: "pythonshell" ScriptLocation: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref scriptsLocation - 'run_athena_query.py' PythonVersion: "3" DefaultArguments: --TempDir: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref scriptsLocation - 'temporary/' --class: "GlueApp" --enable-glue-datacatalog: "true" --enable-job-insights: "false" --job-language: "python" --prebuilt-library-option: "prebuilt-library-enable" MaxRetries: 1 Timeout: 2880 GlueVersion: "1.0" MaxCapacity: 0.0625 GlueJob2: Type: "AWS::Glue::Job" Properties: Name: "sg-analysis-get-rules-data" Description: "Job to get Security Group rules information" Role: !GetAtt IAMRole2.Arn ExecutionProperty: MaxConcurrentRuns: 1 Command: Name: "pythonshell" ScriptLocation: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref scriptsLocation - 'get_sg_rules_data.py' PythonVersion: "3" DefaultArguments: --class: "GlueApp" --enable-job-insights: "false" --extra-py-files: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref librariesLocation - 'boto3-1.22.4-py3-none-any.whl' --job-language: "python" MaxRetries: 1 Timeout: 2880 GlueVersion: "1.0" MaxCapacity: 0.0625 GlueJob3: Type: "AWS::Glue::Job" Properties: Name: "sg-analysis-flow-logs-parser" Description: "Job to parse flow logs and calculate usage" Role: !GetAtt IAMRole2.Arn ExecutionProperty: MaxConcurrentRuns: 1 Command: Name: "pythonshell" ScriptLocation: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref scriptsLocation - 'flow_logs_parser.py' PythonVersion: "3" DefaultArguments: --class: "GlueApp" --enable-job-insights: "false" --extra-py-files: !Join - '' - - 'arn:' - !Ref AWS::Partition - ':s3:::' - !Ref librariesLocation - 'awswrangler-2.14.0-py3-none-any.whl' --job-language: "python" MaxRetries: 1 Timeout: 2880 GlueVersion: "1.0" MaxCapacity: 1 GlueJob4: Type: "AWS::Glue::Job" Properties: Name: "sg-analysis-email-usage-report" Description: "Job to parse flow logs and calculate usage" Role: !GetAtt IAMRole2.Arn ExecutionProperty: MaxConcurrentRuns: 1 Command: Name: "pythonshell" ScriptLocation: !Join - '' - - 'arn:' - !Ref "AWS::Partition" - ':s3:::' - !Ref scriptsLocation - 'sg_usage_analysis.py' PythonVersion: "3" MaxRetries: 1 Timeout: 2880 GlueVersion: "1.0" MaxCapacity: 1 GlueJob5: Type: "AWS::Glue::Job" Properties: Name: "sg-analysis-dynamodb-tables" Description: "Job to create necessary DynamoDB tables to capture rules and usage information" Role: !GetAtt IAMRole2.Arn ExecutionProperty: MaxConcurrentRuns: 1 Command: Name: "pythonshell" ScriptLocation: !Join - '' - - 'arn:' - !Ref "AWS::Partition" - ':s3:::' - !Ref scriptsLocation - 'create_dynamodb_tables.py' PythonVersion: "3" MaxRetries: 1 Timeout: 2880 GlueVersion: "1.0" MaxCapacity: 1