# # Copyright Amazon.com, Inc. and its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT # # Licensed under the MIT License. See the LICENSE accompanying this file # for the specific language governing permissions and limitations under # the License. # AWSTemplateFormatVersion: "2010-09-09" Description: Sample AWS WAF Dashboard build on Amazon Elasticsearch Service. Parameters: DataNodeEBSVolumeSize: Type: Number Default: 100 Description: Elasticsearch volume disk size NodeType: Type: String Default: m5.large.elasticsearch Description: Elasticsearch Node Type ElasticSerchDomainName: Type: String Default: 'waf-dashboards' AllowedPattern: "[a-z\\-]*" Description: Elasticsearch domain name UserEmail: Type: String Default: 'your@email.com' Description: Dashboard user e-mail address ESConfigBucket: Type: String AllowedPattern: ^[A-Za-z0-9][A-Za-z0-9\-_]{3,62}$ Description: ElasticSearch configuration lambda code bucket Resources: UserPool: Type: AWS::Cognito::UserPool Properties: UserPoolName: !Sub ${AWS::StackName}-user-pool AutoVerifiedAttributes: - email Schema: - Name: email AttributeDataType: String Mutable: false Required: true - Name: name AttributeDataType: String Mutable: true Required: true UserPoolDomain: Type: AWS::Cognito::UserPoolDomain Properties: UserPoolId: !Ref UserPool Domain: !Sub ${AWS::StackName}-user-pool-domain UserPoolClient: Type: AWS::Cognito::UserPoolClient Properties: ClientName: !Sub ${AWS::StackName}-user-pool-client GenerateSecret: false UserPoolId: !Ref UserPool IdentityPool: Type: AWS::Cognito::IdentityPool Properties: IdentityPoolName: "WAFKibanaIdentityPool" AllowUnauthenticatedIdentities: false CognitoIdentityProviders: - ClientId: !Ref UserPoolClient ProviderName: !GetAtt UserPool.ProviderName UserPoolUser: Type: AWS::Cognito::UserPoolUser Properties: DesiredDeliveryMediums: - EMAIL UserAttributes: - Name: email Value: !Ref UserEmail Username: !Ref UserEmail UserPoolId: !Ref UserPool CognitoAuthenticatedRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: "sts:AssumeRoleWithWebIdentity" Principal: Federated: cognito-identity.amazonaws.com Condition: StringEquals: "cognito-identity.amazonaws.com:aud": !Ref IdentityPool ForAnyValue:StringLike: "cognito-identity.amazonaws.com:amr": authenticated Policies: - PolicyName: CognitoAuthorizedPolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: es:ESHttp* Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName}/* IdentityPoolRoleMapping: Type: AWS::Cognito::IdentityPoolRoleAttachment Properties: IdentityPoolId: !Ref IdentityPool Roles: authenticated: !GetAtt CognitoAuthenticatedRole.Arn LogEnrichmentLambdaRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Action: sts:AssumeRole Effect: Allow Principal: Service: lambda.amazonaws.com Version: 2012-10-17 ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole LogEnrichmentLambda: Type: AWS::Lambda::Function Properties: FunctionName: !Sub ${AWS::StackName}-log-enrichment Handler: index.lambda_handler MemorySize: 256 Runtime: python3.7 Timeout: 120 Role: !GetAtt LogEnrichmentLambdaRole.Arn Code: ZipFile: | import base64 import json import pyasn from user_agents import parse from counter_robots import is_machine, is_robot print('Loading AWS WAF Logs Enrichment function') asndb = pyasn.pyasn('/opt/ipasn.dat') ua = {} def getUAInfo(user_agent_string): if user_agent_string in ua: return ua[user_agent_string] else: info = {} user_agent = parse(user_agent_string) browser = {} browser['family'] = user_agent.browser.family browser['version'] = user_agent.browser.version_string info['browser'] = browser os = {} os['family'] = user_agent.os.family os['version'] = user_agent.os.version_string info['os'] = os device = {} device['family'] = user_agent.device.family device['brand'] = user_agent.device.brand device['model'] = user_agent.device.model info['device'] = device ua[user_agent_string] = info return info def lambda_handler(event, context): output = [] for record in event['records']: payload = base64.b64decode(record['data']) log = json.loads(payload) # Flattens the header fields flatheaders = {} for item in log['httpRequest']['headers']: flatheaders[item['name'].lower()] = item['value'] del(log['httpRequest']['headers']) log['httpRequest']['headers'] = flatheaders # Parse and add webacl information webacl_info = log['webaclId'].split(":") log['webaclRegion'] = webacl_info[3] log['webaclAccount'] = webacl_info[4] webacl_name = webacl_info[5].split("/") log['webaclName'] = webacl_name[2] # Adds ASN number ip = "" if 'x-forwarded-for' in log['httpRequest']['headers']: ip = log['httpRequest']['headers']['x-forwarded-for'] # XFF header may come with more than one IP address, we'll use the first appearance in the list to report the ASN if len(ip) > 15: print(f'[WARNING] XFF with multiple IP addresses: {ip}; considering last') ip = ip.split(', ')[-1] else: ip = log['httpRequest']['clientIp'] try: log['httpRequest']['asn'] = asndb.lookup(ip)[0] except Exception as e: log['httpRequest']['asn'] = 'unknown' print(f'[ERROR] Got error {e}, while processing ASN for IP {ip}') print(e) if 'user-agent' in log['httpRequest']['headers']: user_agent_string = log['httpRequest']['headers']['user-agent'] # Adds user-agent information ua_info = getUAInfo(user_agent_string) log['httpRequest']['browser'] = ua_info['browser'] log['httpRequest']['os'] = ua_info['os'] log['httpRequest']['device'] = ua_info['device'] # Adds robot information device_type = '' if is_machine(user_agent_string): device_type = 'machine' elif is_robot(user_agent_string): device_type = 'robot' else: device_type = 'other' log['httpRequest']['deviceType'] = device_type payload = json.dumps(log) output_record = { 'recordId': record['recordId'], 'result': 'Ok', 'data': base64.b64encode(payload.encode('utf-8') + b'\n').decode('utf-8') } output.append(output_record) print('Successfully processed {} records.'.format(len(event['records']))) return {'records': output} CodebuildRole: DependsOn: LogEnrichmentLambda Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Action: - sts:AssumeRole Effect: Allow Principal: Service: codebuild.amazonaws.com ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Policies: - PolicyName: CodeBuildPolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - lambda:GetLayerVersion - lambda:PublishLayerVersion Resource: "*" - Effect: Allow Action: lambda:UpdateFunctionConfiguration Resource: !GetAtt LogEnrichmentLambda.Arn LayerCodebuildProject: Type: AWS::CodeBuild::Project Properties: Artifacts: Type: NO_ARTIFACTS Description: !Sub "${AWS::StackName} Lambda Layer build" Environment: ComputeType: BUILD_GENERAL1_SMALL Image: aws/codebuild/amazonlinux2-x86_64-standard:3.0 Type: LINUX_CONTAINER EnvironmentVariables: - Name: LAMBDA_FUNCTION_NAME Type: PLAINTEXT Value: !Sub ${AWS::StackName}-log-enrichment - Name: LAMBDA_LAYER_NAME Type: PLAINTEXT Value: !Sub ${AWS::StackName}-log-enrichment-layer LogsConfig: CloudWatchLogs: Status: ENABLED Name: !Sub "${AWS::StackName}-lambda-layer-build" ServiceRole: !Ref CodebuildRole Source: Type: NO_SOURCE BuildSpec: | version: 0.2 phases: install: commands: - echo "Install Step" #- yum update -y - yum install -y gcc python3.7 python3-devel python3-pip python-virtualenv zip pre_build: commands: - echo "Pre-build Step" - PYTHON_PATH=build/python/lib/python3.7/site-packages - virtualenv -p python3.7 venv build: commands: - echo "Build Step" - . venv/bin/activate - $VIRTUAL_ENV/bin/pip3 install -U --target $VIRTUAL_ENV/lib/python3.7/site-packages pyasn - $VIRTUAL_ENV/bin/pip3 install -U user_agents - $VIRTUAL_ENV/bin/pip3 install -U counter_robots - $VIRTUAL_ENV/lib/python3.7/site-packages/bin/pyasn_util_download.py --latest - RIB_FILE=`ls rib.*` - $VIRTUAL_ENV/lib/python3.7/site-packages/bin/pyasn_util_convert.py --single $RIB_FILE ipasn.dat - mkdir -p ./$PYTHON_PATH - cp -a $VIRTUAL_ENV/lib/python3.7/site-packages/. ./$PYTHON_PATH - rm -rf ./$PYTHON_PATH/wheel* - rm -rf ./$PYTHON_PATH/easy-install* - rm -rf ./$PYTHON_PATH/setuptools* - rm -rf ./$PYTHON_PATH/virtualenv* - rm -rf ./$PYTHON_PATH/pip* - rm -rf ./$PYTHON_PATH/_virtualenv* - rm -rf ./$PYTHON_PATH/bin/* - mv ipasn.dat ./build - cd ./build - zip -r layer.zip . post_build: commands: - LAMBDA_LAYER_VERSION=`aws lambda publish-layer-version --layer-name $LAMBDA_LAYER_NAME --description "AWS WAF Logs Enrichment Tools and Database" --zip-file fileb://layer.zip --compatible-runtimes python3.7 --query 'LayerVersionArn' --output text` - aws lambda update-function-configuration --function-name $LAMBDA_FUNCTION_NAME --layers $LAMBDA_LAYER_VERSION TimeoutInMinutes: 15 BuildEventBridgeRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Action: sts:AssumeRole Effect: Allow Principal: Service: events.amazonaws.com Version: 2012-10-17 Policies: - PolicyName: CodeBuildProjectStartBuild PolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Action: codebuild:StartBuild Resource: !GetAtt LayerCodebuildProject.Arn RebuildEnrichmentLayerTrigger: Type: AWS::Events::Rule Properties: Description: Rebuilds Lambda Layer for AWS WAF Log Enrichment Name: !Sub ${AWS::StackName}-layer-monthly-build RoleArn: !GetAtt BuildEventBridgeRole.Arn ScheduleExpression: cron(0 0 1 * ? *) State: ENABLED Targets: - Arn: !GetAtt LayerCodebuildProject.Arn Id: !Sub ${AWS::StackName}-Codebuild-Project-Start-Build RoleArn: !GetAtt BuildEventBridgeRole.Arn ESCognitoRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: es.amazonaws.com Action: "sts:AssumeRole" ManagedPolicyArns: - 'arn:aws:iam::aws:policy/AmazonESCognitoAccess' Path: / ElasticsearchDomain: Type: AWS::Elasticsearch::Domain Properties: DomainName: !Ref ElasticSerchDomainName ElasticsearchVersion: 7.7 ElasticsearchClusterConfig: InstanceCount: 1 InstanceType: !Ref NodeType EBSOptions: EBSEnabled: true Iops: 0 VolumeSize: !Ref DataNodeEBSVolumeSize VolumeType: 'gp2' SnapshotOptions: AutomatedSnapshotStartHour: '0' CognitoOptions: Enabled: true IdentityPoolId: !Ref IdentityPool UserPoolId: !Ref UserPool RoleArn: !GetAtt ESCognitoRole.Arn AccessPolicies: Version: "2012-10-17" Statement: - Effect: Allow Principal: AWS: !GetAtt ESConfigRole.Arn Action: es:ESHttp* Resource: - !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName}/* - Effect: Allow Principal: AWS: !GetAtt CognitoAuthenticatedRole.Arn Action: es:ESHttp* Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName}/* S3Bucket: Type: AWS::S3::Bucket DeletionPolicy: Retain DeliveryStreamRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: sts:AssumeRole Principal: Service: firehose.amazonaws.com Policies: - PolicyName: DeliveryStreamRolePolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - s3:AbortMultipartUpload - s3:GetBucketLocation - s3:GetObject - s3:ListBucket - s3:ListBucketMultipartUploads - s3:PutObject Resource: - !Sub arn:aws:s3:::${S3Bucket} - !Sub arn:aws:s3:::${S3Bucket}/* - Effect: Allow Action: - lambda:InvokeFunction - lambda:GetFunctionConfiguration Resource: !GetAtt LogEnrichmentLambda.Arn - Effect: Allow Action: - es:DescribeElasticsearchDomain - es:DescribeElasticsearchDomains - es:DescribeElasticsearchDomainConfig - es:ESHttp* Resource: - !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName} - !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName}/* - Effect: Allow Action: - logs:PutLogEvents Resource: "*" - Effect: Allow Action: - kinesis:DescribeStream - kinesis:GetShardIterator - kinesis:GetRecords Resource: !Sub arn:aws:kinesis:${AWS::Region}:${AWS::AccountId}:stream/%FIREHOSE_STREAM_NAME% KinesisFirehoseDeliveryStream: Type: AWS::KinesisFirehose::DeliveryStream Properties: DeliveryStreamName: !Sub aws-waf-logs-${AWS::StackName}-delivery DeliveryStreamType: DirectPut ElasticsearchDestinationConfiguration: BufferingHints: IntervalInSeconds: 60 SizeInMBs: 5 DomainARN: !GetAtt ElasticsearchDomain.Arn IndexName: waf-logs IndexRotationPeriod: OneDay ProcessingConfiguration: Enabled: true Processors: - Type: Lambda Parameters: - ParameterName: LambdaArn ParameterValue: !GetAtt LogEnrichmentLambda.Arn RetryOptions: DurationInSeconds: 10 RoleARN: !GetAtt DeliveryStreamRole.Arn S3BackupMode: AllDocuments S3Configuration: BucketARN: !GetAtt S3Bucket.Arn RoleARN: !GetAtt DeliveryStreamRole.Arn BufferingHints: IntervalInSeconds: 60 SizeInMBs: 50 CompressionFormat: GZIP DependsOn: WAFOperationsSetUpLambda ESConfigRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: lambda.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Policies: - PolicyName: ESConfigPolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: es:ESHttp* Resource: !Sub arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${ElasticSerchDomainName}/* ESConfigFunction: Type: AWS::Lambda::Function Properties: FunctionName: !Sub ${AWS::StackName}-es-config-function Handler: lambda_function.lambda_handler Runtime: python3.7 Timeout: 30 Role: !GetAtt ESConfigRole.Arn Code: S3Bucket: !Ref ESConfigBucket S3Key: esconfig.zip ESConfiguration: Type: Custom::WAFDashboards Properties: ServiceToken: !GetAtt ESConfigFunction.Arn Host: !GetAtt ElasticsearchDomain.DomainEndpoint Region: !Ref AWS::Region WAFOperationsSetUpRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Statement: - Action: - sts:AssumeRole Effect: Allow Principal: Service: - lambda.amazonaws.com Version: 2012-10-17 Policies: - PolicyName: WAFOperationsSetUpLambdaPolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: codebuild:StartBuild Resource: !GetAtt LayerCodebuildProject.Arn - Effect: Allow Action: - s3:DeleteObject - s3:ListBucket Resource: - !Sub arn:aws:s3:::${S3Bucket} - !Sub arn:aws:s3:::${S3Bucket}/* ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole WAFOperationsSetUpLambda: Type: AWS::Lambda::Function DependsOn: - S3Bucket - LayerCodebuildProject Properties: Code: ZipFile: !Sub | import boto3 import cfnresponse def lambda_handler(event, context): print(event) try: if event['RequestType'] == 'Create': client = boto3.client(service_name='codebuild', region_name='${AWS::Region}') new_build = client.start_build(projectName='${AWS::StackName}-lambda-layer-build') cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) return if event['RequestType'] == 'Delete': s3 = boto3.resource('s3') bucket = s3.Bucket('${S3Bucket}') bucket.objects.all().delete() cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) return if event['RequestType'] == 'Update': cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) return except Exception as err: cfnresponse.send(event, context, cfnresponse.FAILED, {}) return FunctionName: !Sub ${AWS::StackName}-waf-operations-set-up Handler: index.lambda_handler Role: !GetAtt WAFOperationsSetUpRole.Arn Runtime: python3.7 Timeout: 180 FirstInvokeWAFOperationsSetUpLambda: Type: AWS::CloudFormation::CustomResource DependsOn: - WAFOperationsSetUpLambda - S3Bucket Version: "1.0" Properties: ServiceToken: !GetAtt WAFOperationsSetUpLambda.Arn WAFv2Modification: Type: AWS::Events::Rule Properties: Description: WAF Dashboard - detects new WebACL and rules for WAFv2. EventPattern: source: - "aws.wafv2" detail-type: - "AWS API Call via CloudTrail" detail: eventSource: - wafv2.amazonaws.com eventName: - CreateWebACL - CreateRule Name: WAFv2Modification State: "ENABLED" Targets: - Arn: !GetAtt KibanaUpdate.Arn Id: "1" KibanaUpdate: Type: "AWS::Lambda::Function" Properties: Handler: "lambda_function.update_kibana" Role: !GetAtt KibanaCustomizerLambdaRole.Arn Code: S3Bucket: !Join - '' - - 'waf-dashboards-' - !Ref "AWS::Region" S3Key: "kibana-customizer-lambda.zip" Runtime: "python3.7" MemorySize: 128 Timeout: 160 Environment: Variables: ES_ENDPOINT : !GetAtt ElasticsearchDomain.DomainEndpoint REGION : !Ref "AWS::Region" ACCOUNT_ID : !Ref "AWS::AccountId" KibanaUpdateWAFv2Permission: Type: AWS::Lambda::Permission Properties: Action: 'lambda:InvokeFunction' FunctionName: !Ref KibanaUpdate Principal: "events.amazonaws.com" SourceArn: !GetAtt WAFv2Modification.Arn KibanaCustomizerLambdaRole: Type: "AWS::IAM::Role" Properties: AssumeRolePolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Principal: Service: lambda.amazonaws.com Action: "sts:AssumeRole" Policies: - PolicyName: KibanaCustomizerPolicy PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - 'es:UpdateElasticsearchDomainConfig' - 'logs:CreateLogGroup' - 'logs:CreateLogStream' - 'logs:PutLogEvents' - 'events:PutRule' - 'events:DeleteRule' - 'lambda:AddPermission' - 'events:PutTargets' - 'events:RemoveTargets' - 'lambda:RemovePermission' - 'iam:PassRole' - 'waf:ListWebACLs' - 'waf-regional:ListWebACLs' - 'waf:ListRules' - 'waf-regional:ListRules' - 'wafv2:ListWebACLs' Resource: "*" Outputs: UserPoolId: Description: UserPool::Id Value: !Ref UserPool UserPoolClientId: Description: UserPoolClient::Id Value: !Ref UserPoolClient IdentityPoolId: Description: IdentityPool::Id Value: !Ref IdentityPool ElasticSearchURL: Description: ElasticSearch URL Value: !GetAtt ElasticsearchDomain.DomainEndpoint DashboardLinkOutput: Description: Link to WAF Dashboard Value: !Join - '' - - 'https://' - !GetAtt ElasticsearchDomain.DomainEndpoint - '/_plugin/kibana/' KinesisFirehoseDeliveryStreamArn: Description: kenesis Firehose Value: !GetAtt KinesisFirehoseDeliveryStream.Arn S3bucketName: Description: S3 bucket name Value: !Sub ${S3Bucket}