# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: MIT-0 [DEFAULT] ############################################################################## # Common ############################################################################## bump_version = 2.6.1 custom_user_agent = AesSiemEsLoader # es-loader のユーザーエージェント名 # user agent name of es-loader custom_user_agent_ver = ${bump_version} # es-loader のユーザーエージェントのバージョン。変更はしないで下さい # custom user agent version. do not edit custom_user_agent_ver ignore_own_logs = True # es-loader から S3 バケットへの GetObject のログでuser agent が # custom と同じであれば Amazon ES に取り込まない # 値は Amazon ES にログを取り込まない(無視)なら True、取り込むなら False exclude_log_patterns_filename = exclude_log_patterns.csv # Amazon ESに取り込まないログの条件を指定したCSVのファイル名 # geoipのDBが保存されているS3バケットに保存 # CSV format and example # log_type,field,pattern,pattern_type,comment # cloudtrail,eventSource,athena.amazonaws.com,text,comment for your memo # cloudtrail,userIdentity.invokedBy,.*\.amazonaws.com,regex,regex is acceptable ############################################################################## # Base info of each log type ############################################################################## # For s3 bucket s3_key = dummydummydummy # S3に保存されたときに、ログの種類を判別するための文字列。正規表現の使用可能。 # A string to determine the log type when log are exported to S3. # Regular expressions can be available. s3_key_ignored = # s3_keyにマッチしても無視したいログがあれば、それを識別する文字列を指定 # If there is a log that you want to ignore even if it matches s3_key, # specify a character string to identify it # For CloudWatch logs via_cwl = False # ログ受信 に CloudWatch Logs 経由をしているか、していないか。 # Are logs received via CloudWatdh Logs? # For Firelens via_firelens = False # ログ受信 に Firelens 経由をして、enable-ecs-log-metadata のフィールドが # 付与されている場合は、True # Are logs received via Firelens with ecs-log-metadata? If so, True. # https://docs.aws.amazon.com/ja_jp/AmazonECS/latest/developerguide/using_firelens.html#firelens-taskdef-metadata # For Container ignore_container_stderr = True # コンテナのログで stderr だった場合、無視して、取り込むのは stdout のみとする # For CloudWatch logs loggroup = dummy # CloudWatch => Kinesis Data Stream 経由でログを受信する時に指定 # Specify string when receiving logs via Kinesis Data Stream via CloudWatch Logs file_format = # ログフォーマット。text, json, csv, multiline を指定可能 # This is a log file format. Available values are json, csv, text and multiline json_delimiter = # 1つのJSONに複数のログがあるときに、delimiterとなっているフィールドを代入 # Substitute the delimiter field when there are multiple logs in one JSON text_header_line_number = 0 # テキスト形式のログで無視したいヘッダーの行数 # if text logs have header, input number lines of header to ignore it multiline_firstline = # 複数行のログで一行目の最初の文字。正規表現で指定。 # string for detecting the start line of the multiline log. string is regex. xml_firstline = # 複数行のログで一行目の最初の文字。正規表現で指定。 # string for detecting the start line of the xml. string is regex. max_log_count = 100000 # 最大ログ処理数。超えた場合はログを分割して処理 # maximum number of logs. if over, logs will be split with SQS index_name = # Amazn ES のインデックス名 # index name of elasticsearch. Do not include suffix such as -2020-01-23 index_rotation = monthly # ログのローテーションを設定 # auto, yearly, monthly, weekly or daily # auto: managed by Index State Management, eg) log-aws-xxxx-000001 # yearly: eg) log-aws-xxxx-2020 # monthly: eg) log-aws-xxxx-2020-12 # daily: eg) log-aws-xxxx-2020-12-23 # weekly: eg) log-aws-xxxx-2020-w52 # Weekly number is ISO8601 style, starting with Monday index_time = @timestamp # インデックス名に付与するローテーション日付の基準日時 # ログが発生した時間か、ログを SIEM で受信した日時かを選択 # @timestamp なら発生時間、event_ingested なら SIEM が受信した時間 # rotation が auto ならこの値は無視される # value is @timestamp or event_ingested # which timestamp you want to use as index. # @timestamp, which is generated time in log source, is recomended when you use batch mode. # event_ingested, which is received time to es-loader, is recomended when you use stream mode only. # if index_rotation = auto, this value is ignored index_tz = +00:00 # インデックスに付与する時間のタイムゾーン # +09:00 とすれば日本時間の0時にローテーションされるので管理をしやすい # rotation が auto ならこの値は無視される # timezone of index rotation. # you can use only following time format as timezone, -04:00, +00:00, +09:00, etc. # + or - is needed. You can't use string such as JST. # if index_rotation = auto, this value is ignored ############################################################################## # Extract ############################################################################## doc_id = # Amazon ES でドキュメントのキーとなる _id に代入するフィールド名 # 通常はログの重複を避けるために、オリジナルIDを指定 # ID がなければ空欄にする。生ログ全体の MD5 ハッシュ値が代入される # The original field name to assign to the key _id of the document in Amazon ES. # Normally specify the original log ID to avoid duplicate logs. # If there is no ID, leave it blank. The MD5 hash value of the entire raw log is assigned doc_id_suffix = # ログによってはオリジナルの ID が重複するので ID に付与したいフィールドを指定 # 重複した ID は @id、フィールドを付与してユニークになった ID は、_id に入力される # 複数のログなのに同じ ID はまれなので通常は空欄 # Since the original ID is duplicated depending on the log, specify the field # you want to give additional ID # Normally blank because the same ID is rare even though there are multiple logs timestamp_key = # @timestamp に代入する生ログのオリジナルフィールド名 # Original field name of raw log to be assigned to @timestamp # special key) # cwe_timestamp: EventBridge経由の受信でその時間を使用する場合 iso8601 # cwl_timestamp: CloudWatch Logs経由の受信でその時間を使用する場合 # file_timestamp: s3keyやファイル名から日時を抽出する timestamp_format = iso8601 # timestamp と指定されたフィールドのタイムフォーマットを指定 # 代入できるのは、epoch, syslog, iso8601 # 該当がなければ Python の Datetime フォーマットで設定 # Specifies the time format of the field specified as timestamp # you can input epoch, syslog, iso8601 or python datetime format # following format are iso8601 # 2020-12-23T01:23:45.123456Z # 2020-12-23 01:23:45.123456Z # 2020-12-23 01:23:45.123456+09:00 # you can check by "datetime.datetime.fromisoformat('2020-12-23 01:23:45')" # python datetime format # https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes timestamp_tz = +00:00 # 生ログのオリジナルフィールドのタイムスタンプの文字列にタイムゾーンがない場合に想定するタイムゾーン # 入力フォーマットは、+09:00 など、プラスマイナスの後にコロン区切りの時間表記 # Time zone assumed when there is no time zone in the timestamp string of # the original field of the raw log # DO NOT USE if the logs have timezone or you can parse timezone. # you can use only time format as timezone, -04:00, +00:00, +09:00, etc. # + or - is needed. You can't use string such as JST. timestamp_nano = False # nano秒が含まれていれば、切り捨ててmicro秒に変換 # trauncate nano seconds to micro seconds file_timestamp_format = # s3keyやファイル名から日時を抽出するための名前付き正規表現 # year, month, day, hour, minute, second, microsecond # example for firelens error log # file_timestamp_format = -(?P\d{4})-(?P\d{2})-(?P\d{2})-(?P\d{2})-(?P\d{2})-(?P\d{2})- ############################################################################## # Transform ############################################################################## ecs_version = 1.10.0 # Elastic Common Schema (ECS) のバージョンを定数で指定 cloud_provider = aws renamed_newfields = # オリジナルログのフィールド名を変更する。新しいフィールド名をリストで列挙して、 # 個々のフィールドを 新しいフィールド名 = 変更前のフィールド名 で定義 # Rename original log field name. # ex) # renamed_newfields = newfield1 newfield2 newfield3 # newfield1 = oldfield1 # newfield2 = oldfield2 # newfield3 = oldfield3 field_prefix = # オリジナルログの前にフィールドを付与する ecs = # 正規化して使う ECS フィールドをスペース区切りで列挙 # 次に正規化として使うECSフィールドを key、オリジナルフィールドを value に入力 # example) # ecs = source.ip destination.ip # source.ip = src_ip # destination.ip = dest_ip static_ecs = # 定数を入力したい ECS フィールドをスペース区切りで列挙 # 次に ECS フィールドを key、定数を value に入力 # example) network and event are constant # static_ecs = event.kind event.category # event.kind = event # event.category = network script_ecs = # scriptで処理する ECS フィールドをスペース区切りで列挙 # ドキュメトで自動生成するためと、scriptのモジュールを実行させるために入力 # 実際のロジックはモジュール内で処理 geoip = # GeopIP を付与したい ECS フィールド名をスペース区切りで列挙 # List the ECS field names to which you want to give GeopIP, separated by spaces # example) # geoip = source destination json_to_text = # json を テキストとして処理させたい生ログのオリジナルフィールド # Original field of raw log that wants json to be processed as text [vpcflowlogs] index_name = log-aws-vpcflowlogs s3_key = vpcflowlogs loggroup = vpcflow file_format = csv timestamp_key = end timestamp_format = epoch ecs = cloud.account.id cloud.instance.id cloud.region destination.address destination.ip destination.port event.action network.bytes network.direction network.iana_number network.packets network.type related.ip source.address source.bytes source.ip source.packets source.port cloud.account.id = account_id cloud.instance.id = instance_id cloud.region = region destination.address = dstaddr destination.ip = dstaddr destination.port = dstport event.action = action network.bytes = bytes network.direction = flow_direction network.iana_number = protocol network.packets = packets network.type = type related.ip = [srcaddr, dstaddr, pkt_srcaddr, pkt_dstaddr] source.address = ${source.ip} source.bytes = bytes source.ip = srcaddr source.packets = packets source.port = srcport static_ecs = event.kind event.category event.kind = event event.category = network script_ecs = event.action event.outcome network.transport geoip = source destination [cloudtrail] index_name = log-aws-cloudtrail s3_key = CloudTrail/|CloudTrail-Insight/ file_format = json max_log_count = 50000 json_delimiter = Records doc_id = eventID timestamp_key = eventTime loggroup = cloudtrail json_to_text = requestParameters.attribute requestParameters.authParameters requestParameters.bucketPolicy.Statement requestParameters.content requestParameters.DescribeFlowLogsRequest requestParameters.ebsOptimized requestParameters.filter requestParameters.iamInstanceProfile requestParameters.instanceType requestParameters.partitionInputList requestParameters.principal requestParameters.result requestParameters.schedule requestParameters.scope requestParameters.sort requestParameters.sortBy requestParameters.source requestParameters.status requestParameters.subnets requestParameters.tableInput.parameters requestParameters.target responseElements.dBSubnetGroup responseElements.description responseElements.endpoint responseElements.role responseElements.subnets serviceEventDetails.eventRequestDetails ecs = cloud.account.id cloud.instance.id cloud.region error.code error.message event.action event.module related.ip rule.name source.address source.ip user.id user.name user_agent.original cloud.account.id = recipientAccountId cloud.instance.id = requestParameters.instanceId responseElements.instancesSet.items.0.instanceId requestParameters.DescribeInstanceCreditSpecificationsRequest.InstanceId.content cloud.region = awsRegion error.code = errorCode error.message = errorMessage event.action = eventName event.module = eventSource related.ip = [${source.ip}] rule.name = eventName source.address = ${source.ip} source.ip = sourceIPAddress user.id = userIdentity.accessKeyId user.name = userIdentity.userName userIdentity.arn userIdentity.principalId user_agent.original = userAgent static_ecs = event.kind event.category event.kind = event event.category = iam script_ecs = destination.address destination.ip destination.nat.ip event.outcome user.name geoip = source [networkfirewall] index_name = log-aws-networkfirewall s3_key = _network-firewall_ file_format = json timestamp_key = event.timestamp #'2021-03-08T09:24:27.005808+0000' ecs = destination.ip destination.port cloud.availability_zone event.action event.module event.severity http.request.method network.bytes network.packets network.protocol related.ip rule.id rule.name rule.version service.node.name source.bytes source.ip source.packets source.port url.domain user_agent.original destination.ip = event.dest_ip destination.port = event.dest_port cloud.availability_zone = availability_zone event.action = event.alert.action event.module = event.event_type event.severity = event.alert.severity http.request.method = event.http.http_method network.bytes = event.netflow.bytes network.packets = event.netflow.pkts network.protocol = event.app_proto related.ip = [event.dest_ip, event.src_ip] rule.id = event.alert.signature_id rule.name = event.alert.signature rule.version = event.alert.rev service.node.name = firewall_name source.bytes = event.netflow.bytes source.ip = event.src_ip source.packets = event.netflow.pkts source.port = event.src_port url.domain = event.http.hostname event.tls.sni user_agent.original = event.http.http_user_agent static_ecs = event.kind event.category event.kind = event event.category = network script_ecs = network.transport event.kind geoip = source destination [guardduty] index_name = log-aws-guardduty s3_key = GuardDuty loggroup = duty file_format = json doc_id = id doc_id_suffix = service.count timestamp_key = updatedAt json_to_text = service.additionalInfo.unusual nw1 = service.action.awsApiCallAction.remoteIpDetails nw2 = service.action.networkConnectionAction.remoteIpDetails nw3 = service.action.portProbeAction.portProbeDetails.0.remoteIpDetails ecs = cloud.instance.id destination.address destination.ip destination.nat.ip destination.port dns.question.name event.severity related.ip related.user rule.name rule.description source.address source.ip source.geo.location source.geo.city_name source.geo.country_name source.port user.id user.name cloud.instance.id = resource.instanceDetails.instanceId destination.address = ${destination.ip} destination.ip = resource.instanceDetails.networkInterfaces.0.privateIpAddress service.action.networkConnectionAction.localIpDetails.ipAddressV4 destination.nat.ip = resource.instanceDetails.networkInterfaces.0.publicIp destination.port = service.action.portProbeAction.portProbeDetails.localPortDetails.0.port service.action.networkConnectionAction.localPortDetails.port dns.question.name = service.action.dnsRequestAction.domain event.severity = severity related.ip = [resource.instanceDetails.networkInterfaces.0.privateIpAddress, service.action.networkConnectionAction.localIpDetails.ipAddressV4, resource.instanceDetails.networkInterfaces.0.publicIp, ${nw1}.ipAddressV4, ${nw2}.ipAddressV4, ${nw3}.ipAddressV4] related.user = [resource.accessKeyDetails.userName] rule.name = type rule.description = title source.address = ${source.ip} source.ip = ${nw1}.ipAddressV4 ${nw2}.ipAddressV4 ${nw3}.ipAddressV4 source.port = service.action.networkConnectionAction.remotePortDetails.port source.geo.location = ${nw1}.geoLocation ${nw2}.geoLocation ${nw3}.geoLocation source.geo.city_name = ${nw1}.city.cityName ${nw2}.city.cityName ${nw3}.country.city.cityName source.geo.country_name = ${nw1}.country.countryName ${nw2}.country.countryName ${nw3}.country.countryName user.id = resource.accessKeyDetails.accessKeyId user.name = resource.accessKeyDetails.userName static_ecs = event.kind event.category event.kind = alert event.category = intrusion_detection script_ecs = destination.address destination.ip destination.nat.ip destination.port event.category network.direction source.address source.ip source.port geoip = source destination [securityhub] index_name = log-aws-securityhub s3_key = /security-hub/ file_format = json json_delimiter = findings timestamp_key = UpdatedAt doc_id = Id doc_id_suffix = ProductFields.aws/guardduty/service/count via_cwl = True # GuardDuty nw1 = ProductFields.aws/guardduty/service/action/awsApiCallAction/remoteIpDetails nw2 = ProductFields.aws/guardduty/service/action/networkConnectionAction/remoteIpDetails nw3 = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails.0_/remoteIpDetails ecs = cloud.account.id cloud.region destination.address destination.ip destination.nat.ip destination.port dns.question.name event.severity event.risk_score_norm related.ip rule.name rule.description source.address source.ip source.port cloud.account.id = AwsAccountId cloud.region = Resources.0.Region destination.address = ${destination.ip} destination.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/privateIpAddress ProductFields.aws/guardduty/service/action/networkConnectionAction/localIpDetails/ipAddressV4 destination.nat.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/publicIp destination.port = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails/localPortDetails.0_/port ProductFields.aws/guardduty/service/action/networkConnectionAction/localPortDetails/port dns.question.name = ProductFields.aws/guardduty/service/action/dnsRequestAction/domain event.severity = Severity.Product event.risk_score_norm = Severity.Normalized related.ip = [ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/privateIpAddress, ProductFields.aws/guardduty/service/action/networkConnectionAction/localIpDetails/ipAddressV4, ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/publicIp, ${nw1}/ipAddressV4, ${nw2}/ipAddressV4, ${nw3}/ipAddressV4] rule.name = Types rule.description = Description source.address = ${source.ip} source.ip = ${nw1}/ipAddressV4 ${nw2}/ipAddressV4 ${nw3}/ipAddressV4 source.port = ProductFields.aws/guardduty/service/action/networkConnectionAction/remotePortDetails/port static_ecs = event.kind event.category event.kind = alert event.category = intrusion_detection script_ecs = event.module event.category cloud.instance.id user.id user.name geoip = source destination [nlb] # https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html index_name = log-aws-elb s3_key = elasticloadbalancing_.*T\d{4}Z_[0-9a-z]{8}\.log\.gz$$ file_format = text log_pattern = (?P[^ ]+) (?P[^ ]+) (?P[^ ]+) (?P[^ ]+) (?P[^ ]+) (?P[0-9a-f.:]+):(?P[0-9]+) (?P[^ ]+):(?P[0-9]+) (?P[0-9]+) (-|(?P[0-9]+)) (-|(?P[-0-9]+)) (?P[-0-9]+) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+)) (-|(?P[^ ]+))$$ timestamp_key = timestamp ecs = destination.address destination.ip destination.port http.request.bytes http.response.bytes related.ip source.address source.ip source.port url.domain url.port destination.address = ${destination.ip} destination.ip = destination_ip destination.port = destination_port http.request.bytes = received_bytes http.response.bytes = sent_bytes related.ip = [${source.ip}, ${destination.ip}] source.address = ${source.ip} source.ip = client_ip source.port = client_port url.domain = domain_name url.port = destination_port static_ecs = event.kind event.category event.kind = event event.category = network geoip = source destination [alb] # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html index_name = log-aws-elb s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log\.gz$$ file_format = text log_pattern = (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*):(?P[0-9]*) (?P[^ ]*)[:-](?P[0-9]*) (?P[-.0-9]*) (?P[-.0-9]*) (?P[-.0-9]*) (?P|[-0-9]*) (?P-|[-0-9]*) (?P[-0-9]*) (?P[-0-9]*) \"(?P(-|(?P\w+)) (-|(?P\w*)://\[?(?P[^/]+?)\]?:(?P\d+)(-|(?P/[^?]*?))(\?(?P.*?))?) (-?|\w+/(?P[0-9\.]*)))\" \"(|(?P[^\"]+))\" (?P[()A-Z0-9-]+) (?P[A-Za-z0-9.-]*) (?P[^ ]*) \"(?P[^\"]*)\" \"(?P[^\"]*)\" \"(?P[^\"]*)\" (?P[-.0-9]*) (?P[^ ]*) \"(?P[^\"]*)\" \"(?P[^\"]*)\" \"(?P[^ ]*)\" \"(?P[^\s]+)\" \"(?P[^\s]+)\"( \"(?P[^\s]+)\" \"(?P[^\s]+)\")? timestamp_key = timestamp ecs = destination.address destination.ip destination.port http.request.bytes http.request.method http.response.bytes http.response.status_code http.version related.ip source.address source.ip source.port url.domain url.path url.port url.query url.scheme user_agent.original destination.address = ${destination.ip} destination.ip = target_ip destination.port = target_port http.request.bytes = received_bytes http.request.method = http_method http.response.bytes = sent_bytes http.response.status_code = elb_status_code http.version = http_version related.ip = [${destination.ip}, ${source.ip}, http_host] source.address = ${source.ip} source.ip = client_ip source.port = client_port url.domain = http_host url.path = http_path url.port = http_port url.query = http_query url.scheme = http_protocol user_agent.original = useragent static_ecs = event.kind event.category event.kind = event event.category = web script_ecs = url.full geoip = source destination [clb] # https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html index_name = log-aws-elb s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log$$ file_format = text log_pattern = (?P[^ ]+) (?P[^ ]+) (?P[0-9a-f.:]+):(?P[0-9]+) (-|(?P[0-9a-f.:]+):(?P[-0-9]+)) (?P[0-9\.-]+) (?P[0-9\.-]+) (?P[0-9\.-]+) (?P[0-9\.-]+) (?P[0-9\.-]+) (?P[0-9\.-]+) (?P[0-9\.-]+) \"(?P(-|(?P[\w-]+)) (-|(?P\w*)://\[?(?P[^\[\]]+?)\]?:(?P\d+)(-|(?P/[^?]*?))(\?(?P[^ ]*))?) (- |-|\w+/(?P[0-9\.]*)))\" (-|\"(|(?P.+))\") (?P[^ ]+) (?P[^ ]+)$$ # " close asymmetric bracket timestamp_key = timestamp ecs = destination.address destination.ip destination.port http.request.bytes http.request.method http.response.bytes http.response.status_code http.version related.ip source.address source.ip source.port url.domain url.path url.port url.query url.scheme user_agent.original destination.address = ${destination.ip} destination.ip = backend_ip destination.port = backend_port http.request.bytes = received_bytes http.request.method = http_method http.response.bytes = sent_bytes http.response.status_code = elb_status_code http.version = http_version related.ip = [${destination.ip}, ${source.ip}] source.address = ${source.ip} source.ip = client_ip source.port = client_port url.domain = http_host url.path = http_path url.port = http_port url.query = http_query url.scheme = http_protocol user_agent.original = useragent static_ecs = event.kind event.category event.kind = event event.category = web script_ecs = url.full geoip = source destination [s3accesslog] # https://docs.aws.amazon.com/ja_jp/AmazonS3/latest/dev/LogFormat.html # https://aws.amazon.com/jp/premiumsupport/knowledge-center/analyze-logs-athena/ s3_key = /20\d{2}-[01]\d-\d{2}-\d{2}-\d{2}-\d{2}-[0-9A-Z]{16}$$ log_pattern = (?P[^ ]*) (?P[^ ]*) \[(?P.*?)\] (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (\"(?P[^ ]*) (?P[^ ]*) (?P- |[^ ]*)\"|\"-\"|-) (?P-|[0-9]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (\"(?P[^ ]*)\"|-) (\"(?P.*)\"|-) (?P[^ ]*)(?: (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*) (?P[^ ]*))? # " close asymmetric bracket index_name = log-aws-s3accesslog file_format = text timestamp_key = RequestDateTime timestamp_format = %d/%b/%Y:%H:%M:%S %z ecs = destination.address destination.domain http.request.method http.request.referrer http.response.bytes http.response.status_code related.ip source.address source.ip url.domain url.original user.name user_agent.original destination.address = ${destination.domain} destination.domain = EndPoint http.request.method = RequestURI_operation http.request.referrer = Referrer http.response.bytes = BytesSent http.response.status_code = HTTPstatus related.ip = [${source.ip}] source.address = ${source.ip} source.ip = RemoteIP url.domain = EndPoint url.original = RequestURI_key user.name = Requester user_agent.original = UserAgent static_ecs = event.kind event.category event.kind = event event.category = web script_ecs = user.name cloud.region geoip = source [config-history] index_name = log-aws-config s3_key = _ConfigHistory_ file_format = json json_delimiter = configurationItems timestamp_key = configurationItemCaptureTime ecs = cloud.account.id cloud.region cloud.account.id = awsAccountId cloud.region = awsRegion static_ecs = event.kind event.category event.type event.kind = state event.category = configuration event.type = info script_ecs = @id cloud.instance.id related.hosts related.ip related.user [config-snapshot] # https://docs.aws.amazon.com/config/latest/APIReference/API_ConfigurationItem.html index_name = log-aws-config s3_key = _ConfigSnapshot_ file_format = json json_delimiter = configurationItems timestamp_key = file_timestamp file_timestamp_format = _(?P\d{4})(?P\d{2})(?P\d{2})T(?P\d{2})(?P\d{2})(?P\d{2})Z_ ecs = cloud.account.id cloud.region cloud.account.id = awsAccountId cloud.region = awsRegion static_ecs = event.kind event.category event.type event.kind = state event.category = configuration event.type = info script_ecs = @id cloud.instance.id related.hosts related.ip related.user [config-rules] index_name = log-aws-config s3_key = Config.*Rules file_format = json timestamp_key = notificationCreationTime ecs = cloud.account.id cloud.region cloud.account.id = awsAccountId cloud.region = awsRegion static_ecs = event.kind event.category event.type event.kind = alert event.category = configuration event.type = change script_ecs = @id cloud.instance.id related.hosts related.ip related.user [cloudfront-realtime] # https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/real-time-logs.html index_name = log-aws-cloudfront s3_key = CloudFront/.*/realtime/ file_format = text log_pattern = (?P[0-9\.]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+) timestamp_key = timestamp timestamp_format = epoch ecs = http.request.bytes http.request.method http.request.referrer http.response.bytes http.response.status_code related.ip source.address source.ip source.port url.domain url.path url.query url.scheme user_agent.original http.request.bytes = cs_bytes http.request.referrer = cs_referer http.request.method = cs_method http.response.bytes = sc_bytes http.response.status_code = sc_status related.ip = [${source.ip}] source.address = ${source.ip} source.ip = c_ip source.port = c_port url.domain = cs_host # url.domain should be x_host_header, but cs_host is fine(2020/9/10) url.path = cs_uri_stem url.query = cs_uri_query url.scheme = cs_protocol user_agent.original = cs_user_agent static_ecs = event.kind event.category cloud.region event.kind = event event.category = web cloud.region = global script_ecs = url.path url.full http.version user_agent.original geoip = source [cloudfront-standard] # https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html index_name = log-aws-cloudfront # default log name example: ABCD0123456789.2020-09-05-22.a5fcfc8d.gz s3_key = (^|\/)[0-9A-Z]{13,14}\.20\d{2}-\d{2}-\d{2}-\d{2}.[0-9a-z]{8}\.gz$$ file_format = text text_header_line_number = 2 log_pattern = (?P[0-9-:\t]+)\t(?P[0-9A-Z-]+)\t(?P[0-9]+)\t(?P[0-9a-f.:]+)\t(?P[A-Z]+)\t(?P[0-9A-Za-z.]+)\t(?P[^\t]+)\t(?P[0-9-]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)(\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+)\t(?P[^\t]+))? timestamp_key = date_time ecs = http.request.bytes http.request.method http.request.referrer http.response.bytes http.response.status_code related.ip source.address source.ip source.port url.domain url.path url.query url.scheme user_agent.original http.request.bytes = cs_bytes http.request.referrer = cs_referer http.request.method = cs_method http.response.bytes = sc_bytes http.response.status_code = sc_status related.ip = [${source.ip}] source.address = ${source.ip} source.ip = c_ip source.port = c_port url.domain = x_host_header url.path = cs_uri_stem url.query = cs_uri_query url.scheme = cs_protocol user_agent.original = cs_user_agent static_ecs = event.kind event.category cloud.region event.kind = event event.category = web cloud.region = global script_ecs = url.full http.version user_agent.original geoip = source [waf] # https://docs.aws.amazon.com/ja_jp/waf/latest/developerguide/logging.html index_name = log-aws-waf s3_key = aws-waf-logs-|_waflogs_ file_format = json timestamp_key = timestamp timestamp_format = epoch ecs = event.action http.request.method related.ip rule.name source.address source.ip url.path url.query event.action = action http.request.method = httpRequest.httpMethod related.ip = [${source.ip}] rule.name = terminatingRuleId source.address = ${source.ip} source.ip = httpRequest.clientIp url.path = httpRequest.uri url.query = httpRequest.args static_ecs = event.kind event.category cloud.region event.kind = alert event.category = web cloud.region = global script_ecs = user_agent.original http.request.referrer rule.ruleset cloud.region http.version cloud.account.id geoip = source [route53resolver] # known issue: only first dns answer maps to ECS index_name = log-aws-r53resolver s3_key = /rql/ file_format = json timestamp_key = query_timestamp via_cwl = True ecs = cloud.instance.id cloud.region dns.answers.class dns.answers.data dns.answers.type related.ip source.address source.ip source.port dns.question.class dns.question.name dns.question.type dns.response_code cloud.instance.id = instance cloud.region = region dns.answers.class = answers.0.Class dns.answers.data = answers.0.Rdata dns.answers.type = answers.0.Type related.ip = [${source.ip}] source.address = ${source.ip} source.ip = srcaddr source.port = srcport dns.question.name = query_name dns.question.type = query_type dns.question.class = query_class dns.response_code = rcode static_ecs = event.kind event.category event.kind = event event.category = network script_ecs = dns.answers.data dns.question.name [rds-postgresql] s3_key = [Pp]ostgre index_name = log-aws-rds-postgresql via_cwl = True file_format = multiline multiline_firstline = ^20\d{2}-\d{2}-\d{2} log_pattern = ^(?P[\d-]+\s[\d:.]+)\s(UTC|GMT)(\s|:(|\[local\]|(?P[^ ]+)\((?P\d+)\)):(?P[^ ]*)@(?P[^ ]*):)\[(?P[\d]+)\](:|\s)(?P[\w]+):\s+(?P[\s\S]*) timestamp_key = postgresql_timestamp ecs = log.level process.pid related.ip rds.database_name rds.message source.address source.ip source.port user.name log.level = postgresql_log_level process.pid = postgresql_pid related.ip = ${source.ip} rds.database_name = postgresql_database rds.message = postgresql_message source.address = postgresql_source_address source.ip = postgresql_source_address source.port = postgresql_source_port user.name = postgresql_user static_ecs = event.category event.category = database script_ecs = rds.query rds.query_time rds.cluster_identifier rds.instance_identifier event.category event.action event.outcome [rds-mysql-audit] s3_key = (MySQL|mysql|MariaDB|mariadb).*(audit) index_name = log-aws-rds-mysql via_cwl = True file_format = text # Log Example # Aurora-MySQL 56 # 1626329400410570,database-20210715-instance-1,rdsadmin,localhost,5,0,CONNECT,,,0 # Aurora-MySQL 57 # 20210715 05:45:55,ip-10-1-2-34,rdsadmin,localhost,36469,0,CONNECT,,,0 log_pattern = ^(?P[0-9: ]*),(?P[^,]*),(?P[^,]*),(?P[^,]*),(?P[\d*]*),(?P[\d*]*),(?P[^,]*),(?P[^,]*),(?P.*),(?P[^,]*) timestamp_key = cwl_timestamp timestamp_format = epoch ecs = user.name rds.database_name rds.query related.ip related.user source.address source.ip user.name = mysql_username rds.query related.ip = [mysql_host] related.user = [mysql_username, rds.query] rds.database_name = mysql_database rds.query = mysql_object source.address = mysql_host source.ip = mysql_host script_ecs = rds.query static_ecs = event.category event.module event.category = database event.module = audit [rds-mysql-general] s3_key = (MySQL|mysql|MariaDB|mariadb).*(general) index_name = log-aws-rds-mysql via_cwl = True file_format = multiline multiline_firstline = ^20\d{2}-\d{2}-\d{2} log_pattern = ^(?P[0-9-:.TZ]+)\s+(?P[0-9]+)\s+(?P[\w]+)\t?(?P[\s\S]*)? timestamp_key = cwl_timestamp timestamp_format = epoch ecs = rds.query rds.query = mysql_query script_ecs = rds.query static_ecs = event.category event.module event.category = database event.module = general [rds-mysql-error] s3_key = (MySQL|mysql|MariaDB|mariadb).*(error) index_name = log-aws-rds-mysql via_cwl = True file_format = multiline multiline_firstline = ${rds-mysql-general:multiline_firstline} # two log patterns #134 # "2021-02-18T12:30:41.777501Z 0 [Note] Shutting down slave threads" # "210218 12:30:43 server_audit: STOPPED" log_pattern = ^(?P([0-9-]+(\s|T)+[0-9:.]+Z?))\s+((?P[0-9]+)\s+\[(?P[0-9a-zA-Z]+)\]\s)?(?P.*) timestamp_key = cwl_timestamp timestamp_format = epoch ecs = log.level rds.message log.level = mysql_log_level rds.message = mysql_message mysql_server_audit_message script_ecs = rds.query static_ecs = event.category event.module event.category = database event.module = error [rds-mysql-slowquery] s3_key = (MySQL|mysql|MariaDB|mariadb).*(slowquery) index_name = log-aws-rds-mysql via_cwl = True file_format = multiline multiline_firstline = ^# Time: log_pattern = ^(# Time: (?P[0-9-:.TZ ]+)\n)?# User@Host: (?P[^\[]*).*\[(?P[^\]]*).*Id:\s+(?P[0-9]+)\n# Query_time: (?P[0-9.]+)\s+Lock_time: (?P[0-9.]+).*\n(?P[\s\S]*) # example) # Aurora MySQL v56 # # Time: 210723 12:33:18 # Aurora MySQL v57 # # Time: 2021-02-22T09:08:46.414775Z # timestamp_key = mysql_timestamp # timestamp_format = iso8601 timestamp_key = cwl_timestamp timestamp_format = epoch ecs = rds.query rds.query_time related.ip related.user source.ip user.name rds.query = mysql_query rds.query_time = mysql_query_time related.ip = [mysql_source_ip] related.user = [mysql_username] source.ip = mysql_source_ip user.name = mysql_username script_ecs = rds.query rds.cluster_identifier rds.instance_identifier static_ecs = event.category event.module event.category = database event.module = slowquery [elasticache-redis-slowlog] # https://docs.aws.amazon.com/ja_jp/AmazonElastiCache/latest/red-ug/Log_Delivery.html s3_key = (redis|Redis).*(slow|SLOW) index_name = log-aws-elasticache file_format = json timestamp_key = Timestamp timestamp_format = epoch renamed_newfields = Duration_us Duration_us = Duration (us) field_prefix = elasticache ecs = source.ip source.ip = ClientAddress static_ecs = event.category event.module event.category = database event.module = redis-slowlog script_ecs = source.ip source.port related.ip [msk] s3_key = KafkaBrokerLogs index_name = log-aws-msk file_format = multiline #[2021-02-25 08:55:07,452] multiline_firstline = ^\[20\d{2}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} log_pattern = ^\[(?P[0-9 -:,]+)\] (?P[^ ]+) (?P[\s\S]*) timestamp_key = msk_timestamp timestamp_format = %Y-%m-%d %H:%M:%S,%f ecs = log.level log.level = msk_log_level script_ecs = msk [opensearch-audit] s3_key = (OpenSearch|opensearch).*(Audit|audit) index_name = log-aws-opensearch file_format = json timestamp_key = timestamp via_cwl = True renamed_newfields = timestamp timestamp = @timestamp field_prefix = opensearch ecs = related.hosts related.ip related.user rule.name source.ip user_agent.original user.name related.hosts = [audit_rest_request_headers.Host] related.ip = [audit_request_remote_address, audit_rest_request_headers.Host] related.user = [audit_request_effective_user, audit_request_initiating_user] rule.name = audit_transport_request_type audit_rest_request_method audit_category source.ip = audit_request_remote_address user_agent.original = audit_rest_request_headers.User-Agent user.name = audit_request_effective_user, audit_request_initiating_user static_ecs = event.kind event.category event.type event.kind = event event.category = [database] event.type = [info] script_ecs = event.category event.outcome geoip = source [workspaces-event] s3_key = (WorkSpaces|workspaces).*(Event|event) index_name = log-aws-workspaces file_format = json #"loginTime": "2018-07-01T17:52:51.595Z", timestamp_key = loginTime timestamp_format = iso8601 renamed_newfields = WorkspaceId WorkspaceId = workspaceId ecs = host.id related.ip related.hosts source.ip host.id = workspaceId related.ip = [clientIpAddress] related.hosts = [workspaceId] source.ip = clientIpAddress static_ecs = event.kind event.category event.type event.outcome event.kind = event event.category = [authentication, host] event.type = [info] event.outcome = success geoip = source [workspaces-inventory] s3_key = (WorkSpaces|workspaces).*(Inventory|inventory) index_name = log-aws-workspaces file_format = json json_delimiter = Workspaces #"2021-07-04T09:58:54Z" timestamp_key = cwe_timestamp timestamp_format = iso8601 ecs = host.id host.ip host.name host.hostname related.hosts related.ip related.user user.id user.name host.id = WorkspaceId host.ip = IpAddress host.name = ComputerName host.hostname = ComputerName related.hosts = [ComputerName, WorkspaceId] related.ip = [IpAddress] related.user = [UserName] user.id = UserName user.name = UserName static_ecs = event.kind event.category event.type event.kind = state event.category = [host] event.type = [info] [directory-service] s3_key = /managed-ad/ index_name = log-aws-directory-service file_format = winevtxml via_cwl = True timestamp_key = ${windows-event:timestamp_key} timestamp_nano = ${windows-event:timestamp_nano} timestamp_format = ${windows-event:timestamp_format} ecs = destination.ip destination.port error.code event.code event.module host.name related.ip related.user source.ip source.port user.domain user.id user.name destination.ip = ${windows-event:destination.ip} destination.port = ${windows-event:destination.port} error.code = ${windows-event:error.code} event.code = ${windows-event:event.code} event.module = ${windows-event:event.module} host.name = ${windows-event:host.name} related.ip = ${windows-event:related.ip} related.user = ${windows-event:related.user} source.ip = ${windows-event:source.ip} source.port = ${windows-event:source.port} user.domain = ${windows-event:user.domain} user.id = ${windows-event:user.id} user.name = ${windows-event:user.name} static_ecs = event.kind event.kind = event script_ecs = event.action event.outcome geoip = ${windows-event:geoip} [fsx-win] s3_key = /aws-fsx- index_name = log-aws-fsx-win file_format = winevtxml via_cwl = False timestamp_key = ${windows-event:timestamp_key} timestamp_nano = ${windows-event:timestamp_nano} timestamp_format = ${windows-event:timestamp_format} ecs = destination.ip destination.port error.code event.code event.module host.name related.ip related.user source.ip source.port user.domain user.id user.name destination.ip = ${windows-event:destination.ip} destination.port = ${windows-event:destination.port} error.code = ${windows-event:error.code} event.code = ${windows-event:event.code} event.module = ${windows-event:event.module} host.name = ${windows-event:host.name} related.ip = ${windows-event:related.ip} related.user = ${windows-event:related.user} source.ip = ${windows-event:source.ip} source.port = ${windows-event:source.port} user.domain = ${windows-event:user.domain} user.id = ${windows-event:user.id} user.name = ${windows-event:user.name} static_ecs = event.kind event.kind = event geoip = ${windows-event:geoip} script_ecs = event.action event.outcome [windows-event] s3_key = /[Ww]indows.*[Ee]vent index_name = log-win-event file_format = winevtxml via_cwl = True timestamp_key = Event.System.TimeCreated.SystemTime timestamp_nano = True timestamp_format = iso8601 ecs = destination.ip destination.port error.code event.code event.module host.name related.ip related.user source.ip source.port user.domain user.id user.name destination.ip = Event.EventData.Data.DestAddress destination.port = Event.EventData.Data.DestPort error.code = Event.System.Status event.code = Event.System.EventID event.module = Event.System.Channel host.name = Event.System.Computer related.ip = [Event.EventData.Data.DestAddress, Event.EventData.Data.IpAddress, Event.EventData.Data.SourceAddress] related.user = [Event.EventData.Data.SubjectUserName, Event.EventData.Data.TargetUserName] source.ip = Event.EventData.Data.IpAddress Event.EventData.Data.SourceAddress source.port = Event.EventData.Data.IpPort Event.EventData.Data.SourcePort user.domain = Event.EventData.Data.SubjectDomainName Event.EventData.Data.TargetDomainName user.id = Event.EventData.Data.SubjectUserSid Event.EventData.Data.TargetUserSid user.name = Event.EventData.Data.SubjectUserName Event.EventData.Data.TargetUserName static_ecs = event.kind event.kind = event script_ecs = event.action event.outcome cloud.instance.id geoip = source destination [linux-secure] # Inheritance from linux-os-syslog s3_key = /[Ll]inux.*[Ss]ecure index_name = log-linux-secure via_cwl = ${linux-os-syslog:via_cwl} # loggroup = secure file_format = ${linux-os-syslog:file_format} timestamp_key = ${linux-os-syslog:timestamp_key} timestamp_format = ${linux-os-syslog:timestamp_format} log_pattern = ${linux-os-syslog:log_pattern} ecs = ${linux-os-syslog:ecs} host.hostname = ${linux-os-syslog:host.hostname} process.name = ${linux-os-syslog:process.name} process.pid = ${linux-os-syslog:process.pid} related.hosts = ${linux-os-syslog:related.hosts} static_ecs = ${linux-os-syslog:static_ecs} event.kind = ${linux-os-syslog:event.kind} script_ecs = ${linux-os-syslog:script_ecs} geoip = ${linux-os-syslog:geoip} [linux-os-syslog] # overwrite index_name according to proc s3_key = /rsyslog/ index_name = log-linux-os via_cwl = True # loggroup = secure file_format = text timestamp_key = syslog_timestamp timestamp_format = syslog log_pattern = (?P[^ ]*\s*[^ ]*\s[^ ]*) (?P[^ ]*) (?P[a-zA-Z0-9_\/\.\-]*)(?:\[(?P[0-9]+)\])?(?:[^\:]*\:)? *(?P.*) ecs = host.hostname process.name process.pid related.hosts host.hostname = hostname process.name = proc process.pid = pid related.hosts = hostname static_ecs = event.kind event.kind = event script_ecs = cloud.instance.id event.action event.category event.outcome related.ip related.hosts related.user source.ip source.port user.name user.id geoip = source