# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
[DEFAULT]
##############################################################################
# Common
##############################################################################
bump_version = 2.3.2

custom_user_agent = AesSiemEsLoader
# es-loader のユーザーエージェント名
# user agent name of es-loader

custom_user_agent_ver = ${bump_version}
# es-loader のユーザーエージェントのバージョン。変更はしないで下さい
# custom user agent version. do not edit custom_user_agent_ver

ignore_own_logs = True
# es-loader から S3 バケットへの GetObject のログでuser agent が
# custom と同じであれば Amazon ES に取り込まない
# 値は Amazon ES にログを取り込まない(無視)なら True、取り込むなら False

exclude_log_patterns_filename = exclude_log_patterns.csv
# Amazon ESに取り込まないログの条件を指定したCSVのファイル名
# geoipのDBが保存されているS3バケットに保存
# CSV format and example
# log_type,field,pattern,pattern_type,comment
# cloudtrail,eventSource,athena.amazonaws.com,text,comment for your memo
# cloudtrail,userIdentity.invokedBy,.*\.amazonaws.com,regex,regex is acceptable

##############################################################################
# Base info of each log type
##############################################################################
# For s3 bucket
s3_key = dummydummydummy
# S3に保存されたときに、ログの種類を判別するための文字列。正規表現の使用可能。
# A string to determine the log type when log are exported to S3.
# Regular expressions can be available.

s3_key_ignored =
# s3_keyにマッチしても無視したいログがあれば、それを識別する文字列を指定
# If there is a log that you want to ignore even if it matches s3_key,
# specify a character string to identify it

# For CloudWatch logs
via_cwl = False
# ログ受信 に CloudWatch Logs 経由をしているか、していないか。
# Are logs received via CloudWatdh Logs?

# For Firelens
via_firelens = False
# ログ受信 に Firelens 経由をして、enable-ecs-log-metadata のフィールドが
# 付与されている場合は、True
# Are logs received via Firelens with ecs-log-metadata? If so, True.
# https://docs.aws.amazon.com/ja_jp/AmazonECS/latest/developerguide/using_firelens.html#firelens-taskdef-metadata

# For Container
ignore_container_stderr = True
# コンテナのログで stderr だった場合、無視して、取り込むのは stdout のみとする

# For CloudWatch logs
loggroup = dummy
# CloudWatch => Kinesis Data Stream 経由でログを受信する時に指定
# Specify string when receiving logs via Kinesis Data Stream via CloudWatch Logs

file_format =
# ログフォーマット。text, json, csv, multiline を指定可能
# This is a log file format. Available values are json, csv, text and multiline

json_delimiter =
# 1つのJSONに複数のログがあるときに、delimiterとなっているフィールドを代入
# Substitute the delimiter field when there are multiple logs in one JSON

text_header_line_number = 0
# テキスト形式のログで無視したいヘッダーの行数
# if text logs have header, input number lines of header to ignore it

multiline_firstline =
# 複数行のログで一行目の最初の文字。正規表現で指定。
# string for detecting the start line of the multiline log. string is regex.

max_log_count = 100000
# 最大ログ処理数。超えた場合はログを分割して処理
# maximum number of logs. if over, logs will be split with SQS

index_name =
# Amazn ES のインデックス名
# index name of elasticsearch. Do not include suffix such as -2020-01-23

index_rotation = monthly
# ログのローテーションを設定
# auto, yearly, monthly, weekly or daily
# auto: managed by Index State Management, eg) log-aws-xxxx-000001
# yearly: eg) log-aws-xxxx-2020
# monthly: eg) log-aws-xxxx-2020-12
# daily: eg) log-aws-xxxx-2020-12-23
# weekly: eg) log-aws-xxxx-2020-w52
# Weekly number is ISO8601 style, starting with Monday

index_time = @timestamp
# インデックス名に付与するローテーション日付の基準日時
# ログが発生した時間か、ログを SIEM で受信した日時かを選択
# @timestamp なら発生時間、event_ingested なら SIEM が受信した時間
# rotation が auto ならこの値は無視される
# value is @timestamp or event_ingested
# which timestamp you want to use as index.
# @timestamp, which is generated time in log source, is recomended when you use batch mode.
# event_ingested, which is received time to es-loader, is recomended when you use stream mode only.
# if index_rotation = auto, this value is ignored

index_tz = +00:00
# インデックスに付与する時間のタイムゾーン
# +09:00 とすれば日本時間の0時にローテーションされるので管理をしやすい
# rotation が auto ならこの値は無視される
# timezone of index rotation.
# you can use only following time format as timezone, -04:00, +00:00, +09:00, etc.
# + or - is needed. You can't use string such as JST.
# if index_rotation = auto, this value is ignored

##############################################################################
# Extract
##############################################################################
doc_id =
# Amazon ES でドキュメントのキーとなる _id に代入するフィールド名
# 通常はログの重複を避けるために、オリジナルIDを指定
# ID がなければ空欄にする。生ログ全体の MD5 ハッシュ値が代入される
# The original field name to assign to the key _id of the document in Amazon ES.
# Normally specify the original log ID to avoid duplicate logs.
# If there is no ID, leave it blank. The MD5 hash value of the entire raw log is assigned

doc_id_suffix =
# ログによってはオリジナルの ID が重複するので ID に付与したいフィールドを指定
# 重複した ID は @id、フィールドを付与してユニークになった ID は、_id に入力される
# 複数のログなのに同じ ID はまれなので通常は空欄
# Since the original ID is duplicated depending on the log, specify the field
# you want to give additional ID
# Normally blank because the same ID is rare even though there are multiple logs

timestamp_key =
# @timestamp に代入する生ログのオリジナルフィールド名
# Original field name of raw log to be assigned to @timestamp

timestamp_format = iso8601
# timestamp と指定されたフィールドのタイムフォーマットを指定
# 代入できるのは、epoch, syslog, iso8601
# 該当がなければ Python の Datetime フォーマットで設定
# Specifies the time format of the field specified as timestamp
# you can input  epoch, syslog, iso8601 or python datetime format
# following format are iso8601
# 2020-12-23T01:23:45.123456Z
# 2020-12-23 01:23:45.123456Z
# 2020-12-23 01:23:45.123456+09:00
# you can check by "datetime.datetime.fromisoformat('2020-12-23 01:23:45')"
# python datetime format
# https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes

timestamp_tz = +00:00
# 生ログのオリジナルフィールドのタイムスタンプの文字列にタイムゾーンがない場合に想定するタイムゾーン
# 入力フォーマットは、+09:00 など、プラスマイナスの後にコロン区切りの時間表記
# Time zone assumed when there is no time zone in the timestamp string of
# the original field of the raw log
# DO NOT USE if the logs have timezone or you can parse timezone.
# you can use only time format as timezone, -04:00, +00:00, +09:00, etc.
# + or - is needed. You can't use string such as JST.

timestamp_nano = False
# nano秒が含まれていれば、切り捨ててmicro秒に変換
# trauncate nano seconds to micro seconds

##############################################################################
# Transform
##############################################################################
ecs_version = 1.6.0
# Elastic Common Schema (ECS) のバージョンを定数で指定
cloud_provider = aws

ecs =
# 正規化して使う ECS フィールドをスペース区切りで列挙
# 次に正規化として使うECSフィールドを key、オリジナルフィールドを value に入力
# example)
# ecs = source.ip destination.ip
# source.ip = src_ip
# destination.ip = dest_ip

static_ecs =
# 定数を入力したい ECS フィールドをスペース区切りで列挙
# 次に ECS フィールドを key、定数を value に入力
# example)　network and event are constant
# static_ecs = event.kind event.category
# event.kind = event
# event.category = network

script_ecs =
# scriptで処理する ECS フィールドをスペース区切りで列挙
# ドキュメトで自動生成するためと、scriptのモジュールを実行させるために入力
# 実際のロジックはモジュール内で処理

geoip =
# GeopIP を付与したい ECS フィールド名をスペース区切りで列挙
# List the ECS field names to which you want to give GeopIP, separated by spaces
# example)
# geoip = source destination

json_to_text =
# json を テキストとして処理させたい生ログのオリジナルフィールド
# Original field of raw log that wants json to be processed as text

[vpcflowlogs]
index_name = log-aws-vpcflowlogs
s3_key = vpcflowlogs
loggroup = vpcflow
file_format = csv
timestamp_key = end
timestamp_format = epoch

ecs = cloud.account.id cloud.instance.id cloud.region destination.address destination.ip destination.port event.action network.bytes network.direction network.iana_number network.packets network.type source.address source.bytes source.ip source.packets source.port
cloud.account.id = account_id
cloud.instance.id = instance_id
cloud.region = region
destination.address = dstaddr
destination.ip = dstaddr
destination.port = dstport
event.action = action
network.bytes = bytes
network.direction = flow_direction
network.iana_number = protocol
network.packets = packets
network.type = type
source.address = ${source.ip}
source.bytes = bytes
source.ip = srcaddr
source.packets = packets
source.port = srcport

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = event.action event.outcome network.transport
geoip = source destination

[cloudtrail]
index_name = log-aws-cloudtrail
s3_key = CloudTrail
s3_key_ignored = CloudTrail-Digest
file_format = json
max_log_count = 50000
json_delimiter = Records
doc_id = eventID
timestamp_key = eventTime
loggroup = cloudtrail

json_to_text = requestParameters.attribute
    requestParameters.authParameters
    requestParameters.bucketPolicy.Statement
    requestParameters.content
    requestParameters.DescribeFlowLogsRequest
    requestParameters.ebsOptimized
    requestParameters.filter
    requestParameters.iamInstanceProfile
    requestParameters.instanceType
    requestParameters.partitionInputList
    requestParameters.principal
    requestParameters.result
    requestParameters.sort
    requestParameters.sortBy
    requestParameters.source
    requestParameters.status
    requestParameters.subnets
    requestParameters.target
    responseElements.dBSubnetGroup
    responseElements.description
    responseElements.endpoint
    responseElements.role
    responseElements.subnets
    serviceEventDetails.eventRequestDetails

ecs = cloud.account.id
    cloud.instance.id
    cloud.region
    error.code
    error.message
    event.action
    event.module
    rule.name
    source.address
    source.ip
    user.id
    user.name
    user_agent.original
cloud.account.id = recipientAccountId
cloud.instance.id = requestParameters.instanceId responseElements.instancesSet.items.0.instanceId requestParameters.DescribeInstanceCreditSpecificationsRequest.InstanceId.content
cloud.region = awsRegion
error.code = errorCode
error.message = errorMessage
event.action = eventName
event.module = eventSource
rule.name = eventName
source.address = ${source.ip}
source.ip = sourceIPAddress
user.id = userIdentity.accessKeyId
user.name = userIdentity.userName userIdentity.arn userIdentity.principalId
user_agent.original = userAgent

static_ecs = event.kind event.category
event.kind = event
event.category = iam

script_ecs = destination.address destination.ip destination.nat.ip event.outcome user.name
geoip = source

[networkfirewall]
index_name = log-aws-networkfirewall
s3_key = _network-firewall_
file_format = json
timestamp_key = event.timestamp
#'2021-03-08T09:24:27.005808+0000'

ecs = destination.ip destination.port cloud.availability_zone event.action event.module event.severity http.request.method network.bytes network.packets network.protocol rule.id rule.name rule.version service.node.name source.bytes source.ip source.packets source.port url.domain user_agent.original
destination.ip = event.dest_ip
destination.port = event.dest_port
cloud.availability_zone = availability_zone
event.action = event.alert.action
event.module = event.event_type
event.severity = event.alert.severity
http.request.method = event.http.http_method
network.protocol = event.app_proto
network.bytes = event.netflow.bytes
network.packets = event.netflow.pkts
url.domain = event.http.hostname event.tls.sni
rule.id = event.alert.signature_id
rule.name = event.alert.signature
rule.version = event.alert.rev
service.node.name = firewall_name
source.bytes = event.netflow.bytes
source.ip = event.src_ip
source.packets = event.netflow.pkts
source.port = event.src_port
user_agent.original = event.http.http_user_agent

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = network.transport event.kind
geoip = source destination

[guardduty]
index_name = log-aws-guardduty
s3_key = GuardDuty
loggroup = duty
file_format = json
doc_id = id
doc_id_suffix = service.count
timestamp_key = updatedAt

json_to_text = service.additionalInfo.unusual

nw1 = service.action.awsApiCallAction.remoteIpDetails
nw2 = service.action.networkConnectionAction.remoteIpDetails
nw3 = service.action.portProbeAction.portProbeDetails.0.remoteIpDetails

ecs = cloud.instance.id destination.address destination.ip destination.nat.ip destination.port dns.question.name event.severity rule.name rule.description source.address source.ip source.geo.location source.geo.city_name source.geo.country_name source.port user.id user.name
cloud.instance.id = resource.instanceDetails.instanceId
destination.address = ${destination.ip}
destination.ip = resource.instanceDetails.networkInterfaces.0.privateIpAddress service.action.networkConnectionAction.localIpDetails.ipAddressV4
destination.nat.ip = resource.instanceDetails.networkInterfaces.0.publicIp
destination.port = service.action.portProbeAction.portProbeDetails.localPortDetails.0.port service.action.networkConnectionAction.localPortDetails.port
dns.question.name = service.action.dnsRequestAction.domain
event.severity = severity
rule.name = type
rule.description = title
source.address = ${source.ip}
source.ip = ${nw1}.ipAddressV4 ${nw2}.ipAddressV4 ${nw3}.ipAddressV4
source.port = service.action.networkConnectionAction.remotePortDetails.port
source.geo.location = ${nw1}.geoLocation ${nw2}.geoLocation ${nw3}.geoLocation
source.geo.city_name = ${nw1}.city.cityName ${nw2}.city.cityName ${nw3}.country.city.cityName
source.geo.country_name = ${nw1}.country.countryName ${nw2}.country.countryName ${nw3}.country.countryName
user.id =  resource.accessKeyDetails.accessKeyId
user.name = resource.accessKeyDetails.userName

static_ecs = event.kind event.category
event.kind = alert
event.category = intrusion_detection

script_ecs = destination.address destination.ip destination.nat.ip destination.port event.category network.direction source.address source.ip source.port
geoip = source destination

[securityhub]
index_name = log-aws-securityhub
s3_key = SecurityHub
file_format = json
json_delimiter = findings
timestamp_key = UpdatedAt
doc_id = Id
doc_id_suffix = ProductFields.aws/guardduty/service/count

# GuardDuty
nw1 = ProductFields.aws/guardduty/service/action/awsApiCallAction/remoteIpDetails
nw2 = ProductFields.aws/guardduty/service/action/networkConnectionAction/remoteIpDetails
nw3 = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails.0_/remoteIpDetails

ecs = cloud.account.id cloud.region destination.address destination.ip destination.nat.ip destination.port dns.question.name event.severity event.risk_score_norm rule.name rule.description source.address source.ip source.port
cloud.account.id = AwsAccountId
cloud.region = Resources.0.Region
destination.address = ${destination.ip}
destination.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/privateIpAddress ProductFields.aws/guardduty/service/action/networkConnectionAction/localIpDetails/ipAddressV4
destination.nat.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/publicIp
destination.port = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails/localPortDetails.0_/port ProductFields.aws/guardduty/service/action/networkConnectionAction/localPortDetails/port
dns.question.name = ProductFields.aws/guardduty/service/action/dnsRequestAction/domain
event.severity = Severity.Product
event.risk_score_norm = Severity.Normalized
rule.name = Types
rule.description = Description
source.address = ${source.ip}
source.ip = ${nw1}/ipAddressV4 ${nw2}/ipAddressV4 ${nw3}/ipAddressV4
source.port = ProductFields.aws/guardduty/service/action/networkConnectionAction/remotePortDetails/port

static_ecs = event.kind event.category
event.kind = alert
event.category = intrusion_detection

script_ecs = event.module event.category cloud.instance.id user.id user.name
geoip = source destination

[nlb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_[0-9a-z]{8}\.log\.gz$$
file_format = text
log_pattern = (?P<listener_type>[^ ]+) (?P<log_entry_version>[^ ]+) (?P<timestamp>[^ ]+) (?P<elb>[^ ]+) (?P<listener>[^ ]+) (?P<client_ip>[0-9a-f.:]+):(?P<client_port>[0-9]+) (?P<destination_ip>[^ ]+):(?P<destination_port>[0-9]+) (?P<connection_time>[0-9]+) (-|(?P<tls_handshake_time>[0-9]+)) (-|(?P<received_bytes>[-0-9]+)) (?P<sent_bytes>[-0-9]+) (-|(?P<incoming_tls_alert>[^ ]+)) (-|(?P<chosen_cert_arn>[^ ]+)) (-|(?P<chosen_cert_serial>[^ ]+)) (-|(?P<tls_cipher>[^ ]+)) (-|(?P<tls_protocol_version>[^ ]+)) (-|(?P<tls_named_group>[^ ]+)) (-|(?P<domain_name>[^ ]+)) (-|(?P<alpn_fe_protocol>[^ ]+)) (-|(?P<alpn_be_protocol>[^ ]+)) (-|(?P<alpn_client_preference_list>[^ ]+))$$
timestamp_key = timestamp

ecs = destination.address destination.ip destination.port http.request.bytes http.response.bytes source.address source.ip source.port url.domain url.port
destination.address = ${destination.ip}
destination.ip = destination_ip
destination.port = destination_port
http.request.bytes = received_bytes
http.response.bytes = sent_bytes
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = domain_name
url.port = destination_port

static_ecs = event.kind event.category
event.kind = event
event.category = network

geoip = source destination

[alb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log\.gz$$
file_format = text
log_pattern = (?P<request_type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request>(-|(?P<http_method>\w+)) (-|(?P<http_protocol>\w*)://\[?(?P<http_host>[^/\[\]]+?)\]?:(?P<http_port>\d+)(-|(?P<http_path>/[^?]*?))(\?(?P<http_query>.*?))?) (-?|\w+/(?P<http_version>[0-9\.]*)))\" \"(|(?P<useragent>[^\"]+))\" (?P<ssl_cipher>[()A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^\"]*)\" \"(?P<lambda_error_reason>[^ ]*)\" \"(?P<target_port_list>[^\s]+)\" \"(?P<target_status_code_list>[^\s]+)\"( \"(?P<classification>[^\s]+)\" \"(?P<classification_reason>[^\s]+)\")?
timestamp_key = timestamp

ecs = destination.address destination.ip destination.port http.request.bytes http.request.method http.response.bytes http.response.status_code http.version source.address source.ip source.port url.domain url.path url.port url.query url.scheme user_agent.original
destination.address = ${destination.ip}
destination.ip = target_ip
destination.port = target_port
http.request.bytes = received_bytes
http.request.method = http_method
http.response.bytes = sent_bytes
http.response.status_code = elb_status_code
http.version = http_version
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = http_host
url.path = http_path
url.port = http_port
url.query = http_query
url.scheme = http_protocol
user_agent.original = useragent

static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = url.full
geoip = source destination

[clb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log$$
file_format = text
log_pattern = (?P<timestamp>[^ ]+) (?P<elb>[^ ]+) (?P<client_ip>[0-9a-f.:]+):(?P<client_port>[0-9]+) (-|(?P<backend_ip>[0-9a-f.:]+):(?P<backend_port>[-0-9]+)) (?P<request_processing_time>[0-9\.-]+) (?P<backend_processing_time>[0-9\.-]+) (?P<response_processing_time>[0-9\.-]+) (?P<elb_status_code>[0-9\.-]+) (?P<backend_status_code>[0-9\.-]+) (?P<received_bytes>[0-9\.-]+) (?P<sent_bytes>[0-9\.-]+) \"(?P<request>(-|(?P<http_method>[\w-]+)) (-|(?P<http_protocol>\w*)://\[?(?P<http_host>[^\[\]]+?)\]?:(?P<http_port>\d+)(-|(?P<http_path>/[^?]*?))(\?(?P<http_query>[^ ]*))?) (- |-|\w+/(?P<http_version>[0-9\.]*)))\" (-|\"(|(?P<useragent>.+))\") (?P<ssl_cipher>[^ ]+) (?P<ssl_protocol>[^ ]+)$$
# " close asymmetric bracket
timestamp_key = timestamp

ecs = destination.address destination.ip destination.port http.request.bytes http.request.method http.response.bytes http.response.status_code http.version source.address source.ip source.port url.domain url.path url.port url.query url.scheme user_agent.original
destination.address = ${destination.ip}
destination.ip = backend_ip
destination.port = backend_port
http.request.bytes = received_bytes
http.request.method = http_method
http.response.bytes = sent_bytes
http.response.status_code = elb_status_code
http.version = http_version
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = http_host
url.path = http_path
url.port = http_port
url.query = http_query
url.scheme = http_protocol
user_agent.original = useragent

static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = url.full
geoip = source destination

[s3accesslog]
# https://docs.aws.amazon.com/ja_jp/AmazonS3/latest/dev/LogFormat.html
# https://aws.amazon.com/jp/premiumsupport/knowledge-center/analyze-logs-athena/
s3_key = /20\d{2}-[01]\d-\d{2}-\d{2}-\d{2}-\d{2}-[0-9A-Z]{16}$$
log_pattern = (?P<BucketOwner>[^ ]*) (?P<Bucket>[^ ]*) \[(?P<RequestDateTime>.*?)\] (?P<RemoteIP>[^ ]*) (?P<Requester>[^ ]*) (?P<RequestID>[^ ]*) (?P<Operation>[^ ]*) (?P<Key>[^ ]*) (\"(?P<RequestURI_operation>[^ ]*) (?P<RequestURI_key>[^ ]*) (?P<RequestURI_httpProtoversion>- |[^ ]*)\"|\"-\"|-) (?P<HTTPstatus>-|[0-9]*) (?P<ErrorCode>[^ ]*) (?P<BytesSent>[^ ]*) (?P<ObjectSize>[^ ]*) (?P<TotalTime>[^ ]*) (?P<TurnAroundTime>[^ ]*) (\"(?P<Referrer>[^ ]*)\"|-) (\"(?P<UserAgent>.*)\"|-) (?P<VersionId>[^ ]*)(?: (?P<HostId>[^ ]*) (?P<SigV>[^ ]*) (?P<CipherSuite>[^ ]*) (?P<AuthType>[^ ]*) (?P<EndPoint>[^ ]*) (?P<TLSVersion>[^ ]*))?
# " close asymmetric bracket
index_name = log-aws-s3accesslog
file_format = text
timestamp_key = RequestDateTime
timestamp_format = %d/%b/%Y:%H:%M:%S %z

ecs = destination.address destination.domain http.request.method http.request.referrer http.response.bytes http.response.status_code source.address source.ip url.domain url.original user.name user_agent.original
destination.address = ${destination.domain}
destination.domain = EndPoint
http.request.method = RequestURI_operation
http.request.referrer = Referrer
http.response.bytes = BytesSent
http.response.status_code = HTTPstatus
source.address = ${source.ip}
source.ip = RemoteIP
url.domain = EndPoint
url.original = RequestURI_key
user.name = Requester
user_agent.original = UserAgent

static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = user.name cloud.region
geoip = source

[cloudfront-realtime]
# https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/real-time-logs.html
index_name = log-aws-cloudfront
s3_key = CloudFront/.*/realtime/
file_format = text
log_pattern = (?P<timestamp>[0-9\.]+)\t(?P<c_ip>[^\t]+)\t(?P<time_to_first_byte>[^\t]+)\t(?P<sc_status>[^\t]+)\t(?P<sc_bytes>[^\t]+)\t(?P<cs_method>[^\t]+)\t(?P<cs_protocol>[^\t]+)\t(?P<cs_host>[^\t]+)\t(?P<cs_uri_stem>[^\t]+)\t(?P<cs_bytes>[^\t]+)\t(?P<x_edge_location>[^\t]+)\t(?P<x_edge_request_id>[^\t]+)\t(?P<x_host_header>[^\t]+)\t(?P<time_taken>[^\t]+)\t(?P<cs_protocol_version>[^\t]+)\t(?P<c_ip_version>[^\t]+)\t(?P<cs_user_agent>[^\t]+)\t(?P<cs_referer>[^\t]+)\t(?P<cs_cookie>[^\t]+)\t(?P<cs_uri_query>[^\t]+)\t(?P<x_edge_response_result_type>[^\t]+)\t(?P<x_forwarded_for>[^\t]+)\t(?P<ssl_protocol>[^\t]+)\t(?P<ssl_cipher>[^\t]+)\t(?P<x_edge_result_type>[^\t]+)\t(?P<fle_encrypted_fields>[^\t]+)\t(?P<fle_status>[^\t]+)\t(?P<sc_content_type>[^\t]+)\t(?P<sc_content_len>[^\t]+)\t(?P<sc_range_start>[^\t]+)\t(?P<sc_range_end>[^\t]+)\t(?P<c_port>[^\t]+)\t(?P<x_edge_detailed_result_type>[^\t]+)\t(?P<c_country>[^\t]+)\t(?P<cs_accept_encoding>[^\t]+)\t(?P<cs_accept>[^\t]+)\t(?P<cache_behavior_path_pattern>[^\t]+)\t(?P<cs_headers>[^\t]+)\t(?P<cs_header_names>[^\t]+)\t(?P<cs_headers_count>[^\t]+)
timestamp_key = timestamp
timestamp_format = epoch

ecs = http.request.bytes http.request.method http.request.referrer http.response.bytes http.response.status_code source.address source.ip source.port url.domain url.path url.query url.scheme user_agent.original
http.request.bytes = cs_bytes
http.request.referrer = cs_referer
http.request.method = cs_method
http.response.bytes = sc_bytes
http.response.status_code = sc_status
source.address = ${source.ip}
source.ip = c_ip
source.port = c_port
url.domain = cs_host
# url.domain should be x_host_header, but cs_host is fine(2020/9/10)
url.path = cs_uri_stem
url.query = cs_uri_query
url.scheme = cs_protocol
user_agent.original = cs_user_agent

static_ecs = event.kind event.category cloud.region
event.kind = event
event.category = web
cloud.region = global

script_ecs = url.path url.full http.version user_agent.original
geoip = source

[cloudfront-standard]
# https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
index_name = log-aws-cloudfront
# default log name example: ABCD0123456789.2020-09-05-22.a5fcfc8d.gz
s3_key = (^|\/)[0-9A-Z]{13,14}\.20\d{2}-\d{2}-\d{2}-\d{2}.[0-9a-z]{8}\.gz$$
file_format = text
text_header_line_number = 2
log_pattern = (?P<date_time>[0-9-:\t]+)\t(?P<x_edge_location>[0-9A-Z-]+)\t(?P<sc_bytes>[0-9]+)\t(?P<c_ip>[0-9a-f.:]+)\t(?P<cs_method>[A-Z]+)\t(?P<cs_host>[0-9A-Za-z.]+)\t(?P<cs_uri_stem>[^\t]+)\t(?P<sc_status>[0-9-]+)\t(?P<cs_referer>[^\t]+)\t(?P<cs_user_agent>[^\t]+)\t(?P<cs_uri_query>[^\t]+)\t(?P<cs_cookie>[^\t]+)\t(?P<x_edge_result_type>[^\t]+)\t(?P<x_edge_request_id>[^\t]+)\t(?P<x_host_header>[^\t]+)\t(?P<cs_protocol>[^\t]+)\t(?P<cs_bytes>[^\t]+)\t(?P<time_taken>[^\t]+)\t(?P<x_forwarded_for>[^\t]+)\t(?P<ssl_protocol>[^\t]+)\t(?P<ssl_cipher>[^\t]+)\t(?P<x_edge_response_result_type>[^\t]+)\t(?P<cs_protocol_version>[^\t]+)\t(?P<fle_status>[^\t]+)\t(?P<fle_encrypted_fields>[^\t]+)(\t(?P<c_port>[^\t]+)\t(?P<time_to_first_byte>[^\t]+)\t(?P<x_edge_detailed_result_type>[^\t]+)\t(?P<sc_content_type>[^\t]+)\t(?P<sc_content_len>[^\t]+)\t(?P<sc_range_start>[^\t]+)\t(?P<sc_range_end>[^\t]+))?
timestamp_key = date_time

ecs = http.request.bytes http.request.method http.request.referrer http.response.bytes http.response.status_code source.address source.ip source.port url.domain url.path url.query url.scheme user_agent.original
http.request.bytes = cs_bytes
http.request.referrer = cs_referer
http.request.method = cs_method
http.response.bytes = sc_bytes
http.response.status_code = sc_status
source.address = ${source.ip}
source.ip = c_ip
source.port = c_port
url.domain = x_host_header
url.path = cs_uri_stem
url.query = cs_uri_query
url.scheme = cs_protocol
user_agent.original = cs_user_agent

static_ecs = event.kind event.category cloud.region
event.kind = event
event.category = web
cloud.region = global

script_ecs = url.full http.version user_agent.original
geoip = source

[waf]
# https://docs.aws.amazon.com/ja_jp/waf/latest/developerguide/logging.html
index_name = log-aws-waf
s3_key = aws-waf-logs-
file_format = json
timestamp_key = timestamp
timestamp_format = epoch

ecs = event.action http.request.method rule.name source.address source.ip url.path url.query
event.action = action
http.request.method = httpRequest.httpMethod
rule.name = terminatingRuleId
source.address = ${source.ip}
source.ip = httpRequest.clientIp
url.path = httpRequest.uri
url.query = httpRequest.args

static_ecs = event.kind event.category cloud.region
event.kind = alert
event.category = web
cloud.region = global

script_ecs = user_agent.original http.request.referrer rule.ruleset cloud.region http.version cloud.account.id
geoip = source

[route53resolver]
# known issue: only first dns answer maps to ECS
index_name = log-aws-r53resolver
s3_key = vpcdnsquerylogs
file_format = json
timestamp_key = query_timestamp

ecs = cloud.instance.id cloud.region dns.answers.class dns.answers.data dns.answers.type source.address source.ip source.port dns.question.class dns.question.name dns.question.type dns.response_code
cloud.instance.id = instance
cloud.region = region
dns.answers.class = answers.0.Class
dns.answers.data = answers.0.Rdata
dns.answers.type = answers.0.Type
source.address = ${source.ip}
source.ip = srcaddr
source.port = srcport
dns.question.name = query_name
dns.question.type = query_type
dns.question.class = query_class
dns.response_code = rcode

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = dns.answers.data dns.question.name

[rds-postgresql]
s3_key = [Pp]ostgre
index_name = log-aws-rds-postgresql

via_cwl = True
file_format = multiline

multiline_firstline = ^20\d{2}-\d{2}-\d{2}
log_pattern = ^(?P<postgresql_timestamp>[\d-]+\s[\d:.]+)\s(UTC|GMT)(\s|:(|\[local\]|(?P<postgresql_source_address>[^ ]+)\((?P<postgresql_source_port>\d+)\)):(?P<postgresql_user>[^ ]*)@(?P<postgresql_database>[^ ]*):)\[(?P<postgresql_pid>[\d]+)\](:|\s)(?P<postgresql_log_level>[\w]+):\s+(?P<postgresql_message>[\s\S]*)

timestamp_key = postgresql_timestamp

ecs = log.level process.pid rds.database_name rds.message source.address source.ip source.port user.name
log.level = postgresql_log_level
process.pid = postgresql_pid
rds.database_name = postgresql_database
rds.message = postgresql_message
source.address = postgresql_source_address
source.ip = postgresql_source_address
source.port = postgresql_source_port
user.name = postgresql_user

static_ecs = event.category
event.category = database

script_ecs = rds.query rds.query_time rds.cluster_identifier rds.instance_identifier event.category event.action event.outcome

[rds-mysql-audit]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(audit)
index_name = log-aws-rds-mysql

via_cwl = True

file_format = text
log_pattern = ^(?P<mysql_timestamp>[0-9: ]*),(?P<mysql_serverhost>[^,]*),(?P<mysql_username>[^,]*),(?P<mysql_host>[^,]*),(?P<mysql_thread_id>[\d*]*),(?P<mysql_queryid>[\d*]*),(?P<mysql_operation>[^,]*),(?P<mysql_database>[^,]*),(?P<mysql_object>.*),(?P<mysql_retcode>[^,]*)
timestamp_key = mysql_timestamp
# eaxmple) 20210222 09:09:00
timestamp_format = %Y%m%d %H:%M:%S

ecs = user.name rds.database_name rds.query source.address source.ip
user.name = mysql_username rds.query
rds.database_name = mysql_database
rds.query = mysql_object
source.address = mysql_host
source.ip = mysql_host

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = audit


[rds-mysql-general]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(general)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ^20\d{2}-\d{2}-\d{2}

log_pattern = ^(?P<mysql_timestamp>[0-9-:.TZ]+)\s+(?P<mysql_thread_id>[0-9]+)\s+(?P<mysql_command>[\w]+)\t?(?P<mysql_query>[\s\S]*)?
timestamp_key = mysql_timestamp
# example) 2021-02-22T09:09:48.140289Z
timestamp_format = iso8601

ecs = rds.query
rds.query = mysql_query

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = general


[rds-mysql-error]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(error)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ${rds-mysql-general:multiline_firstline}

log_pattern = ^(?P<mysql_timestamp>[0-9-:.TZ]+)\s+(?P<mysql_thread_id>[0-9]+)\s+\[(?P<mysql_log_level>[\w]+)\]\s+(?P<mysql_message>[\s\S]*)
timestamp_key = ${rds-mysql-general:timestamp_key}
# example) 2021-02-22T07:50:29.013603Z
timestamp_format = ${rds-mysql-general:timestamp_format}

ecs = log.level rds.message
log.level = mysql_log_level
rds.message = mysql_message

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = error


[rds-mysql-slowquery]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(slowquery)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ^# Time:

log_pattern = ^# Time: (?P<mysql_timestamp>[0-9-:.TZ]+)\n# User@Host: (?P<mysql_username>[^\[]*).*\[(?P<mysql_source_ip>[^\]]*).*Id:\s+(?P<mysql_thread_id>[0-9]+)\n# Query_time: (?P<mysql_query_time>[0-9.]+)\s+Lock_time: (?P<mysql_lock_time>[0-9.]+).*\n(?P<mysql_query>[\s\S]*)
timestamp_key = mysql_timestamp
# example) # Time: 2021-02-22T09:08:46.414775Z
timestamp_format = iso8601

ecs = user.name rds.query rds.query_time source.ip
user.name = mysql_username
rds.query = mysql_query
rds.query_time = mysql_query_time
source.ip = mysql_source_ip

script_ecs = rds.query rds.cluster_identifier rds.instance_identifier

static_ecs = event.category event.module
event.category = database
event.module = slowquery


[msk]
s3_key = KafkaBrokerLogs
index_name = log-aws-msk
file_format = multiline

#[2021-02-25 08:55:07,452]
multiline_firstline = ^\[20\d{2}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
log_pattern = ^\[(?P<msk_timestamp>[0-9 -:,]+)\] (?P<msk_log_level>[^ ]+) (?P<msk_message>[\s\S]*)

timestamp_key = msk_timestamp
timestamp_format = %Y-%m-%d %H:%M:%S,%f

ecs = log.level
log.level = msk_log_level

script_ecs = msk

[linux-secure]
# Inheritance from linux-os-syslog
s3_key = /[Ll]inux.*[Ss]ecure
index_name = log-linux-secure

via_cwl = ${linux-os-syslog:via_cwl}
# loggroup = secure
file_format = ${linux-os-syslog:file_format}

timestamp_key = ${linux-os-syslog:timestamp_key}
timestamp_format = ${linux-os-syslog:timestamp_format}
log_pattern = ${linux-os-syslog:log_pattern}

ecs = ${linux-os-syslog:ecs}
host.hostname = ${linux-os-syslog:host.hostname}
process.name = ${linux-os-syslog:process.name}
process.pid = ${linux-os-syslog:process.pid}

static_ecs = ${linux-os-syslog:static_ecs}
event.kind = ${linux-os-syslog:event.kind}

script_ecs = ${linux-os-syslog:script_ecs}
geoip = ${linux-os-syslog:geoip}

[linux-os-syslog]
# overwrite index_name according to proc
s3_key = /[Ll]inux/
index_name = log-linux-os
via_cwl = True
# loggroup = secure
file_format = text
timestamp_key = syslog_timestamp
timestamp_format = syslog
log_pattern = (?P<syslog_timestamp>[^ ]*\s*[^ ]*\s[^ ]*) (?P<hostname>[^ ]*) (?P<proc>[a-zA-Z0-9_\/\.\-]*)(?:\[(?P<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?P<syslog_message>.*)

ecs = host.hostname process.name process.pid
host.hostname = hostname
process.name = proc
process.pid = pid

static_ecs = event.kind
event.kind = event

script_ecs = cloud.instance.id event.action event.category event.outcome source.ip source.port user.name user.id
geoip = source