# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: MIT-0
[DEFAULT]
##############################################################################
# Common
##############################################################################
bump_version = 2.10.0

custom_user_agent = siem-es-loader
# es-loader のユーザーエージェント名
# user agent name of es-loader

ignore_own_logs = True
# es-loader から S3 バケットへの GetObject のログでuser agent が
# custom と同じであれば Amazon ES に取り込まない
# 値は Amazon ES にログを取り込まない(無視)なら True、取り込むなら False

exclude_log_patterns_filename = exclude_log_patterns.csv
# Amazon ESに取り込まないログの条件を指定したCSVのファイル名
# geoipのDBが保存されているS3バケットに保存
# CSV format and example
# log_type,field,pattern,pattern_type,comment
# cloudtrail,eventSource,athena.amazonaws.com,text,comment for your memo
# cloudtrail,userIdentity.invokedBy,.*\.amazonaws.com,regex,regex is acceptable

##############################################################################
# Base info of each log type
##############################################################################
# For s3 bucket
s3_key = dummydummydummy
# S3に保存されたときに、ログの種類を判別するための文字列。正規表現の使用可能。
# A string to determine the log type when log are exported to S3.
# Regular expressions can be available.

s3_key_ignored =
# s3_keyにマッチしても無視したいログがあれば、それを識別する文字列を指定
# If there is a log that you want to ignore even if it matches s3_key,
# specify a character string to identify it

# For CloudWatch logs
via_cwl = False
# ログ受信 に CloudWatch Logs 経由をしているか、していないか。
# Are logs received via CloudWatdh Logs?

# For Firelens
via_firelens = False
# ログ受信 に Firelens 経由をして、enable-ecs-log-metadata のフィールドが
# 付与されている場合は、True
# Are logs received via Firelens with ecs-log-metadata? If so, True.
# https://docs.aws.amazon.com/ja_jp/AmazonECS/latest/developerguide/using_firelens.html#firelens-taskdef-metadata

# For Container
ignore_container_stderr = True
# コンテナのログで stderr だった場合、無視して、取り込むのは stdout のみとする

# For CloudWatch logs
loggroup = dummy
# CloudWatch => Kinesis Data Stream 経由でログを受信する時に指定
# Specify string when receiving logs via Kinesis Data Stream via CloudWatch Logs

file_format =
# ログフォーマット。text, json, csv, multiline, cef を指定可能
# This is a log file format. Available values are json, csv, text, multiline,
# parquet and cef

json_delimiter =
# 1つのJSONに複数のログがあるときに、delimiterとなっているフィールドを代入
# Substitute the delimiter field when there are multiple logs in one JSON

text_header_line_number = 0
# テキスト形式のログで無視したいヘッダーの行数
# if text logs have header, input number lines of header to ignore it

csv_delimiter =
# CSVの区切り文字。デフォルトはタブを含む空白。
# CSV delimiter. The default is a space containing tabs.
# e.g.
# csv_delimiter = ,

multiline_firstline =
# 複数行のログで一行目の最初の文字。正規表現で指定。
# string for detecting the start line of the multiline log. string is regex.

xml_firstline =
# 複数行のログで一行目の最初の文字。正規表現で指定。
# string for detecting the start line of the xml. string is regex.

max_log_count = 100000
# 最大ログ処理数。超えた場合はログを分割して処理
# maximum number of logs. if over, logs will be split with SQS

index_name =
# Amazn OpenSearch のインデックス名
# index name of OpenSearch. Do not include suffix such as -2020-01-23

index_suffix =
# index 名の直後に付与する文字列
# A string appended immediately after the index name
# e.g.
# index_suffix = prod
# index_rotation = monthly# then, index name will become log-aws-xxx-prod-2023-06

index_rotation = monthly
# ログのローテーションを設定
# auto, yearly, monthly, weekly or daily
# auto: managed by Index State Management, eg) log-aws-xxxx-000001
# yearly: eg) log-aws-xxxx-2020
# monthly: eg) log-aws-xxxx-2020-12
# daily: eg) log-aws-xxxx-2020-12-23
# weekly: eg) log-aws-xxxx-2020-w52
# Weekly number is ISO8601 style, starting with Monday

index_time = @timestamp
# インデックス名に付与するローテーション日付の基準日時
# ログが発生した時間か、ログを SIEM で受信した日時かを選択
# @timestamp なら発生時間、event_ingested なら SIEM が受信した時間
# rotation が auto ならこの値は無視される
# value is @timestamp or event_ingested
# which timestamp you want to use as index.
# @timestamp, which is generated time in log source, is recomended when you use batch mode.
# event_ingested, which is received time to es-loader, is recomended when you use stream mode only.
# if index_rotation = auto, this value is ignored

index_tz = +00:00
# インデックスに付与する時間のタイムゾーン
# +09:00 とすれば日本時間の0時にローテーションされるので管理をしやすい
# rotation が auto ならこの値は無視される
# timezone of index rotation.
# you can use only following time format as timezone, -04:00, +00:00, +09:00, etc.
# + or - is needed. You can't use string such as JST.
# if index_rotation = auto, this value is ignored

##############################################################################
# Extract
##############################################################################
doc_id =
# Amazon ES でドキュメントのキーとなる _id に代入するフィールド名
# 通常はログの重複を避けるために、オリジナルIDを指定
# ID がなければ空欄にする。生ログ全体の MD5 ハッシュ値が代入される
# The original field name to assign to the key _id of the document in Amazon ES.
# Normally specify the original log ID to avoid duplicate logs.
# If there is no ID, leave it blank. The MD5 hash value of the entire raw log is assigned

doc_id_suffix =
# ログによってはオリジナルの ID が重複するので ID に付与したいフィールドを指定
# 重複した ID は @id、フィールドを付与してユニークになった ID は、_id に入力される
# 複数のログなのに同じ ID はまれなので通常は空欄
# Since the original ID is duplicated depending on the log, specify the field
# you want to give additional ID
# Normally blank because the same ID is rare even though there are multiple logs

timestamp_key =
# @timestamp に代入する生ログのオリジナルフィールド名
# Original field name of raw log to be assigned to @timestamp
# special key)
# cwe_timestamp: EventBridge経由の受信でその時間を使用する場合 iso8601
# cwl_timestamp: CloudWatch Logs経由の受信でその時間を使用する場合
# file_timestamp: s3keyやファイル名から日時を抽出する

timestamp_key_list =
# 一つのファイルの中に複数のログフォーマットが含まれていて、timestamp の
# key が複数種類ある特殊な場合に指定。
# スペース区切りで指定する
# Specify when there are multiple log formats in one file and
# there are multiple types of timestamp keys.
# Specify by separating them with spaces.
# e.g.
# timestamp_key_list = time_stamp TimeStamp Date

timestamp_format = iso8601
# timestamp と指定されたフィールドのタイムフォーマットを指定
# 代入できるのは、epoch, syslog, iso8601
# 該当がなければ Python の Datetime フォーマットで設定
# Specifies the time format of the field specified as timestamp
# you can input  epoch, syslog, iso8601 or python datetime format
# following format are iso8601
# 2020-12-23T01:23:45.123456Z
# 2020-12-23 01:23:45.123456Z
# 2020-12-23 01:23:45.123456+09:00
# you can check by "datetime.datetime.fromisoformat('2020-12-23 01:23:45')"
# python datetime format
# https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes

timestamp_format_list =
# 一つのファイルの中に複数のログフォーマットが含まれていて、timestamp の
# format が複数種類ある特殊な場合に指定。
# Python スタイルのリスト型で指定する。
# ダブルクオーテーション " で囲って、カンマ区切り
# Specify when there are multiple log formats in one file and
# there are multiple types of timestamp formats.
# Specify as a Python-style list type.
# Enclose each value in double quotation and separate them with commas.
# e.g.
# timestamp_format_list = ["iso8601", "syslog", "%Y-%m-%d %H:%M:%S,%f"]

timestamp_tz = +00:00
# 生ログのオリジナルフィールドのタイムスタンプの文字列にタイムゾーンがない場合に想定するタイムゾーン
# 入力フォーマットは、+09:00 など、プラスマイナスの後にコロン区切りの時間表記
# Time zone assumed when there is no time zone in the timestamp string of
# the original field of the raw log
# DO NOT USE if the logs have timezone or you can parse timezone.
# you can use only time format as timezone, -04:00, +00:00, +09:00, etc.
# + or - is needed. You can't use string such as JST.

timestamp_nano = False
# nano秒が含まれていれば、切り捨ててmicro秒に変換
# trauncate nano seconds to micro seconds

file_timestamp_format =
# s3keyやファイル名から日時を抽出するための名前付き正規表現
# year, month, day, hour, minute, second, microsecond
# example for firelens error log
# file_timestamp_format = -(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})-(?P<hour>\d{2})-(?P<minute>\d{2})-(?P<second>\d{2})-

##############################################################################
# Transform
##############################################################################
ecs_version = 1.10.0
# Elastic Common Schema (ECS) のバージョンを定数で指定
cloud_provider = aws

renamed_newfields =
# オリジナルログのフィールド名を変更する。新しいフィールド名をリストで列挙して、
# 個々のフィールドを 新しいフィールド名 = 変更前のフィールド名 で定義
# Rename original log field name.
# ex)
# renamed_newfields = newfield1 newfield2 newfield3
# newfield1 = oldfield1
# newfield2 = oldfield2
# newfield3 = oldfield3

field_prefix =
# オリジナルログの前にフィールドを付与する

ecs =
# 正規化して使う ECS フィールドをスペース区切りで列挙
# 次に正規化として使うECSフィールドを key、オリジナルフィールドを value に入力
# example)
# ecs = source.ip destination.ip
# source.ip = src_ip
# destination.ip = dest_ip

static_ecs =
# 定数を入力したい ECS フィールドをスペース区切りで列挙
# 次に ECS フィールドを key、定数を value に入力
# example) network and event are constant
# static_ecs = event.kind event.category
# event.kind = event
# event.category = network

script_ecs =
# scriptで処理する ECS フィールドをスペース区切りで列挙
# ドキュメトで自動生成するためと、scriptのモジュールを実行させるために入力
# 実際のロジックはモジュール内で処理

json_to_text =
# json を テキストとして処理させたい生ログのオリジナルフィールド
# Original field of raw log that wants json to be processed as text

##############################################################################
# Enrichment
##############################################################################
geoip =
# GeopIP を付与したい ECS フィールド名をスペース区切りで列挙
# List the ECS field names to which you want to give GeopIP, separated by spaces
# example)
# geoip = source destination

ioc_ip =
ioc_domain =

user_agent_enrichment_field =
# ussr agentのECSフィールド名。user agentからbrower、os、deviceの情報を取得する
# ECS field name of user agent. To parse user agent, os, device information.

[securitylake]
index_name = log-ocsf

s3_key = [0-9a-f]{32}\.gz\.parquet|[Ss]ecurity[Ll]ake/
file_format = parquet
timestamp_key = time
timestamp_format = "epoch"
json_to_text = data
    enrichments.data
    raw_data
    evidence
    finding.supporting_data
    malware.cves.cvss.metrics
    unmapped

ecs = cloud.account.id
    cloud.instance.id
    destination.address
    destination.ip
    destination.port
    related.host
    related.ip
    source.address
    source.ip
    source.port
    user.id
    user.name
    user_agent.original

cloud.account.id = cloud.account_uid
    cloud.account.uid
cloud.instance.id = device.instance_uid
    dst_endpoint.instance_uid
    src_endpoint.instance_uid
destination.address = dst_endpoint.ip
    dst_endpoint.domain
destination.ip = dst_endpoint.ip
destination.port = dst_endpoint.port
related.host = [device.instance_uid dst_endpoint.instance_uid, src_endpoint.instance_uid]
related.ip = [dst_endpoint.ip, src_endpoint.ip]
source.address = src_endpoint.ip
    src_endpoint.domain
source.ip = src_endpoint.ip
source.port = src_endpoint.port
user.id = actor.user.credential_uid
user.name = actor.user.name
    actor.user.uid
user_agent.original = http_request.user_agent

script_ecs = index_name user.name

geoip = source destination
ioc_ip = source.ip destination.ip related.ip
user_agent_enrichment_field = user_agent


[ocsf-json]
index_name = log-ocsf
s3_key = ocsf-json/
file_format = json
timestamp_key = ${securitylake:timestamp_key}
timestamp_format = ${securitylake:timestamp_format}
json_to_text = ${securitylake:json_to_text}

ecs = ${securitylake:ecs}

cloud.account.id = ${securitylake:cloud.account.id}
cloud.instance.id = ${securitylake:cloud.instance.id}
destination.address = ${securitylake:destination.address}
destination.ip = ${securitylake:destination.ip}
destination.port = ${securitylake:destination.port}
related.host = ${securitylake:related.host}
related.ip = ${securitylake:related.ip}
source.address = ${securitylake:source.address}
source.ip = ${securitylake:source.ip}
source.port = ${securitylake:source.port}
user.id = ${securitylake:user.id}
user.name = ${securitylake:user.name}
user_agent.original = ${securitylake:user_agent.original}

script_ecs = ${securitylake:script_ecs}

geoip = ${securitylake:geoip}
ioc_ip = ${securitylake:ioc_ip}
user_agent_enrichment_field = ${securitylake:user_agent_enrichment_field}


[vpcflowlogs]
index_name = log-aws-vpcflowlogs
s3_key = vpcflowlogs
loggroup = vpcflow
file_format = csv
timestamp_key = end
timestamp_format = epoch

ecs = cloud.account.id
    cloud.instance.id
    cloud.region
    destination.address
    destination.ip
    destination.port
    event.action
    network.bytes
    network.direction
    network.iana_number
    network.packets
    network.type
    related.ip
    source.address
    source.bytes
    source.ip
    source.packets
    source.port
cloud.account.id = account_id
cloud.instance.id = instance_id
cloud.region = region
destination.address = dstaddr
destination.ip = dstaddr
destination.port = dstport
event.action = action
network.bytes = bytes
network.direction = flow_direction
network.iana_number = protocol
network.packets = packets
network.type = type
related.ip = [srcaddr, dstaddr, pkt_srcaddr, pkt_dstaddr]
source.address = ${source.ip}
source.bytes = bytes
source.ip = srcaddr
source.packets = packets
source.port = srcport

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = event.action event.outcome network.transport

geoip = source destination
ioc_ip = source.ip destination.ip related.ip


[cloudtrail]
index_name = log-aws-cloudtrail
s3_key = CloudTrail/|CloudTrail-Insight/
file_format = json
max_log_count = 50000
json_delimiter = Records
doc_id = eventID
timestamp_key = eventTime
loggroup = cloudtrail

json_to_text = additionalEventData.requestParameters
    requestParameters.attribute
    requestParameters.authParameters
    requestParameters.bucketPolicy.Statement
    requestParameters.content
    requestParameters.DescribeFlowLogsRequest
    requestParameters.description
    requestParameters.ebsOptimized
    requestParameters.filter
    requestParameters.iamInstanceProfile
    requestParameters.imageId
    requestParameters.instanceType
    requestParameters.partitionInputList
    requestParameters.principal
    requestParameters.result
    requestParameters.resource
    requestParameters.schedule
    requestParameters.scope
    requestParameters.sort
    requestParameters.sortBy
    requestParameters.source
    requestParameters.status
    requestParameters.subnets
    requestParameters.tableInput.parameters
    requestParameters.target
    responseElements.dBSubnetGroup
    responseElements.data
    responseElements.description
    responseElements.endpoint
    responseElements.imageId
    responseElements.result
    responseElements.role
    responseElements.subnets
    serviceEventDetails.eventRequestDetails

ecs = cloud.account.id
    cloud.instance.id
    cloud.region
    error.code
    error.message
    event.action
    event.module
    related.ip
    rule.name
    source.address
    source.ip
    user.id
    user.name
    user_agent.original
cloud.account.id = recipientAccountId
cloud.instance.id = requestParameters.instanceId
    responseElements.instancesSet.items.0.instanceId
    requestParameters.instancesSet.items.0.instanceId
    requestParameters.DescribeInstanceCreditSpecificationsRequest.InstanceId.content
    requestParameters.AssociateIamInstanceProfileRequest.InstanceId
cloud.region = awsRegion
error.code = errorCode
error.message = errorMessage
event.action = eventName
event.module = eventSource
related.ip = [${source.ip}]
rule.name = eventName
source.address = ${source.ip}
source.ip = sourceIPAddress
user.id = userIdentity.accessKeyId
user.name = userIdentity.userName userIdentity.arn userIdentity.principalId
user_agent.original = userAgent

static_ecs = event.kind event.category
event.kind = event
event.category = iam

script_ecs = destination.address destination.ip destination.nat.ip event.outcome user.name

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[networkfirewall]
index_name = log-aws-networkfirewall
s3_key = _network-firewall_
file_format = json
timestamp_key = event.timestamp
#'2021-03-08T09:24:27.005808+0000'

ecs = destination.ip
    destination.port
    cloud.availability_zone
    event.action
    event.module
    event.severity
    http.request.method
    network.bytes
    network.packets
    network.protocol
    related.ip
    rule.id
    rule.name
    rule.version
    service.node.name
    source.bytes
    source.ip
    source.packets
    source.port
    url.domain
    user_agent.original
destination.ip = event.dest_ip
destination.port = event.dest_port
cloud.availability_zone = availability_zone
event.action = event.alert.action
event.module = event.event_type
event.severity = event.alert.severity
http.request.method = event.http.http_method
network.bytes = event.netflow.bytes
network.packets = event.netflow.pkts
network.protocol = event.app_proto
related.ip = [event.dest_ip, event.src_ip]
rule.id = event.alert.signature_id
rule.name = event.alert.signature
rule.version = event.alert.rev
service.node.name = firewall_name
source.bytes = event.netflow.bytes
source.ip = event.src_ip
source.packets = event.netflow.pkts
source.port = event.src_port
url.domain = event.http.hostname event.tls.sni
user_agent.original = event.http.http_user_agent

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = network.transport event.kind

geoip = source destination
ioc_ip = source.ip destination.ip
user_agent_enrichment_field = user_agent


[guardduty]
index_name = log-aws-guardduty
s3_key = /GuardDuty/
loggroup = duty
file_format = json
doc_id = id
doc_id_suffix = service.count
timestamp_key = updatedAt

json_to_text = service.additionalInfo.unusual

nw1 = service.action.awsApiCallAction.remoteIpDetails
nw2 = service.action.networkConnectionAction.remoteIpDetails
nw3 = service.action.portProbeAction.portProbeDetails.0.remoteIpDetails
nw4 = service.action.kubernetesApiCallAction.remoteIpDetails

ecs = cloud.instance.id
    destination.address
    destination.ip
    destination.nat.ip
    destination.port
    dns.question.name
    event.severity
    related.ip
    related.user
    rule.id
    rule.name
    rule.description
    source.address
    source.ip
    source.geo.location
    source.geo.city_name
    source.geo.country_name
    source.port
    user.id
    user.name
cloud.instance.id = resource.instanceDetails.instanceId
destination.address = ${destination.ip}
destination.ip = resource.instanceDetails.networkInterfaces.0.privateIpAddress service.action.networkConnectionAction.localIpDetails.ipAddressV4
destination.nat.ip = resource.instanceDetails.networkInterfaces.0.publicIp
destination.port = service.action.portProbeAction.portProbeDetails.localPortDetails.0.port service.action.networkConnectionAction.localPortDetails.port
dns.question.name = service.action.dnsRequestAction.domain
event.severity = severity
related.ip = [resource.instanceDetails.networkInterfaces.0.privateIpAddress, service.action.networkConnectionAction.localIpDetails.ipAddressV4, resource.instanceDetails.networkInterfaces.0.publicIp, ${nw1}.ipAddressV4, ${nw2}.ipAddressV4, ${nw3}.ipAddressV4, ${nw4}.ipAddressV4, ${nw1}.ipAddressV6, ${nw2}.ipAddressV6, ${nw3}.ipAddressV6, ${nw4}.ipAddressV6, service.action.kubernetesApiCallAction.sourceIPs]
related.user = [resource.accessKeyDetails.userName]
rule.id = type
rule.name = title
rule.description = description
source.address = ${source.ip}
source.ip = ${nw1}.ipAddressV4
    ${nw2}.ipAddressV4
    ${nw3}.ipAddressV4
    ${nw4}.ipAddressV4
    ${nw1}.ipAddressV6
    ${nw2}.ipAddressV6
    ${nw3}.ipAddressV6
    ${nw4}.ipAddressV6
source.port = service.action.networkConnectionAction.remotePortDetails.port
source.geo.location = ${nw1}.geoLocation ${nw2}.geoLocation ${nw3}.geoLocation
source.geo.city_name = ${nw1}.city.cityName ${nw2}.city.cityName ${nw3}.country.city.cityName
source.geo.country_name = ${nw1}.country.countryName
    ${nw2}.country.countryName
    ${nw3}.country.countryName
    ${nw4}.country.countryName
user.id =  resource.accessKeyDetails.accessKeyId
user.name = resource.accessKeyDetails.userName

static_ecs = event.kind event.category
event.kind = alert
event.category = intrusion_detection

script_ecs = destination.address destination.ip destination.nat.ip destination.port event.category network.direction source.address source.ip source.port

geoip = source destination
ioc_ip = source.ip destination.ip
ioc_domain = dns.question.name

[inspector]
index_name = log-aws-inspector
s3_key = Inspector2_Finding
file_format = json
timestamp_key = updatedAt
timestamp_format = %b %d, %Y, %I:%M:%S %p
doc_id = findingArn
# doc_id will be overwritten by findingArn's md5 with sf_inspector.py
field_prefix = inspector

ecs = cloud.instance.id
    related.hosts
    related.ip
    rule.name
    vulnerability.category
    vulnerability.description
    vulnerability.id
    vulnerability.reference
    vulnerability.score.base
    vulnerability.score.version
    vulnerability.severity
cloud.instance.id = resources.0.id
related.hosts = [resources.0.id]
related.ip = resources.0.details.awsEc2Instance.ipV4Addresses
rule.name = title
vulnerability.category = [type, resources.0.details.awsEc2Instance.platform, resources.0.details.awsEcrContainerImage.platform, resources.0.type]
vulnerability.description = description
vulnerability.id = packageVulnerabilityDetails.vulnerabilityId
vulnerability.reference = packageVulnerabilityDetails.referenceUrls packageVulnerabilityDetails.sourceUrl
vulnerability.score.base = inspectorScoreDetails.adjustedCvss.score
vulnerability.score.version = inspectorScoreDetails.adjustedCvss.version
vulnerability.severity = severity

script_ecs = vulnerability.description

static_ecs = rule.category vulnerability.scanner.vendor
rule.category = vulnerability
vulnerability.scanner.vendor = inspector

[securityhub]
index_name = log-aws-securityhub
s3_key = SecurityHub|securityhub
file_format = json
json_delimiter = findings
timestamp_key_list = cwe_timestamp UpdatedAt
doc_id = Id
doc_id_suffix =

# GuardDuty
nw1 = ProductFields.aws/guardduty/service/action/awsApiCallAction/remoteIpDetails
nw2 = ProductFields.aws/guardduty/service/action/networkConnectionAction/remoteIpDetails
nw3 = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails.0_/remoteIpDetails
nw4 = ProductFields.aws/guardduty/service/action/kubernetesApiCallAction/remoteIpDetails

ecs = cloud.account.id
    cloud.region
    destination.address
    destination.ip
    destination.nat.ip
    destination.port
    dns.question.name
    event.severity
    event.risk_score_norm
    related.ip
    rule.id
    rule.name
    rule.description
    source.address
    source.ip source.port
cloud.account.id = AwsAccountId
cloud.region = Resources.0.Region
destination.address = ${destination.ip}
destination.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/privateIpAddress ProductFields.aws/guardduty/service/action/networkConnectionAction/localIpDetails/ipAddressV4
destination.nat.ip = ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/publicIp
destination.port = ProductFields.aws/guardduty/service/action/portProbeAction/portProbeDetails/localPortDetails.0_/port ProductFields.aws/guardduty/service/action/networkConnectionAction/localPortDetails/port
dns.question.name = ProductFields.aws/guardduty/service/action/dnsRequestAction/domain
event.severity = Severity.Product
event.risk_score_norm = Severity.Normalized
related.ip = [ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/privateIpAddress, ProductFields.aws/guardduty/service/action/networkConnectionAction/localIpDetails/ipAddressV4, ProductFields.aws/guardduty/resource/instanceDetails/networkInterfaces.0_/publicIp, ${nw1}/ipAddressV4, ${nw2}/ipAddressV4, ${nw3}/ipAddressV4, ${nw4}/ipAddressV4, ${nw1}/ipAddressV6, ${nw2}/ipAddressV6, ${nw3}/ipAddressV6, ${nw4}/ipAddressV6]
rule.id = Types
rule.name = Title
rule.description = Description
source.address = ${source.ip}
source.ip = ${nw1}/ipAddressV4 ${nw2}/ipAddressV4 ${nw3}/ipAddressV4
source.port = ProductFields.aws/guardduty/service/action/networkConnectionAction/remotePortDetails/port

static_ecs = event.kind event.category
event.kind = alert
event.category = intrusion_detection

script_ecs = event.module event.category cloud.instance.id user.id user.name

geoip = source destination
ioc_ip = source.ip destination.ip
ioc_domain = dns.question.name


[nlb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_[0-9a-z]{8}\.log\.gz$$
file_format = text
log_pattern = (?P<listener_type>[^ ]+) (?P<log_entry_version>[^ ]+) (?P<timestamp>[^ ]+) (?P<elb>[^ ]+) (?P<listener>[^ ]+) (?P<client_ip>[0-9a-f.:]+):(?P<client_port>[0-9]+) (?P<destination_ip>[^ ]+):(?P<destination_port>[0-9]+) (?P<connection_time>[0-9]+) (-|(?P<tls_handshake_time>[0-9]+)) (-|(?P<received_bytes>[-0-9]+)) (?P<sent_bytes>[-0-9]+) (-|(?P<incoming_tls_alert>[^ ]+)) (-|(?P<chosen_cert_arn>[^ ]+)) (-|(?P<chosen_cert_serial>[^ ]+)) (-|(?P<tls_cipher>[^ ]+)) (-|(?P<tls_protocol_version>[^ ]+)) (-|(?P<tls_named_group>[^ ]+)) (-|(?P<domain_name>[^ ]+)) (-|(?P<alpn_fe_protocol>[^ ]+)) (-|(?P<alpn_be_protocol>[^ ]+)) (-|(?P<alpn_client_preference_list>[^ ]+))
timestamp_key = timestamp

ecs = destination.address
    destination.ip
    destination.port
    http.request.bytes
    http.response.bytes
    related.ip
    source.address
    source.ip
    source.port
    url.domain
    url.port
destination.address = ${destination.ip}
destination.ip = destination_ip
destination.port = destination_port
http.request.bytes = received_bytes
http.response.bytes = sent_bytes
related.ip = [${source.ip}, ${destination.ip}]
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = domain_name
url.port = destination_port

static_ecs = event.kind event.category
event.kind = event
event.category = network

geoip = source destination
ioc_ip = source.ip


[alb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log\.gz$$
file_format = text
log_pattern = (?P<request_type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) (?P<target_ip>[^ ]*)[:-](?P<target_port>[0-9]*) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request>(-|(?P<http_method>\w+)) (-|(?P<http_protocol>\w*)://\[?(?P<http_host>[^/]+?)\]?:(?P<http_port>\d+)(-|(?P<http_path>/[^?]*?))(\?(?P<http_query>.*?))?) (-?|\w+/(?P<http_version>[0-9\.]*)))\" \"(|(?P<useragent>[^\"]+))\" (?P<ssl_cipher>[()A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\" \"(?P<domain_name>[^\"]*)\" \"(?P<chosen_cert_arn>[^\"]*)\" (?P<matched_rule_priority>[-.0-9]*) (?P<request_creation_time>[^ ]*) \"(?P<actions_executed>[^\"]*)\" \"(?P<redirect_url>[^\"]*)\" \"(?P<lambda_error_reason>[^ ]*)\" \"(?P<target_port_list>[^\s]+)\" \"(?P<target_status_code_list>[^\s]+)\"( \"(?P<classification>[^\s]+)\" \"(?P<classification_reason>[^\s]+)\")?
timestamp_key = timestamp

ecs = destination.address
    destination.ip
    destination.port
    http.request.bytes
    http.request.method
    http.response.bytes
    http.response.status_code
    http.version
    related.ip
    source.address
    source.ip
    source.port
    url.domain
    url.path
    url.port
    url.query
    url.scheme
    user_agent.original
destination.address = ${destination.ip}
destination.ip = target_ip
destination.port = target_port
http.request.bytes = received_bytes
http.request.method = http_method
http.response.bytes = sent_bytes
http.response.status_code = elb_status_code
http.version = http_version
related.ip = [${destination.ip}, ${source.ip}, http_host]
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = http_host
url.path = http_path
url.port = http_port
url.query = http_query
url.scheme = http_protocol
user_agent.original = useragent


static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = url.full

geoip = source destination
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[clb]
# https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
index_name = log-aws-elb
s3_key = elasticloadbalancing_.*T\d{4}Z_\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}_\w*\.log$$
file_format = text
log_pattern = (?P<timestamp>[^ ]+) (?P<elb>[^ ]+) (?P<client_ip>[0-9a-f.:]+):(?P<client_port>[0-9]+) (-|(?P<backend_ip>[0-9a-f.:]+):(?P<backend_port>[-0-9]+)) (?P<request_processing_time>[0-9\.-]+) (?P<backend_processing_time>[0-9\.-]+) (?P<response_processing_time>[0-9\.-]+) (?P<elb_status_code>[0-9\.-]+) (?P<backend_status_code>[0-9\.-]+) (?P<received_bytes>[0-9\.-]+) (?P<sent_bytes>[0-9\.-]+) \"(?P<request>(-|(?P<http_method>[\w-]+)) (-|(?P<http_protocol>\w*)://\[?(?P<http_host>[^\[\]]+?)\]?:(?P<http_port>\d+)(-|(?P<http_path>/[^?]*?))(\?(?P<http_query>[^ ]*))?) (- |-|\w+/(?P<http_version>[0-9\.]*)))\" (-|\"(|(?P<useragent>.+))\") (?P<ssl_cipher>[^ ]+) (?P<ssl_protocol>[^ ]+)$$
# " close asymmetric bracket
timestamp_key = timestamp

ecs = destination.address
    destination.ip
    destination.port
    http.request.bytes
    http.request.method
    http.response.bytes
    http.response.status_code
    http.version
    related.ip
    source.address
    source.ip
    source.port
    url.domain
    url.path
    url.port
    url.query
    url.scheme
    user_agent.original
destination.address = ${destination.ip}
destination.ip = backend_ip
destination.port = backend_port
http.request.bytes = received_bytes
http.request.method = http_method
http.response.bytes = sent_bytes
http.response.status_code = elb_status_code
http.version = http_version
related.ip = [${destination.ip}, ${source.ip}]
source.address = ${source.ip}
source.ip = client_ip
source.port = client_port
url.domain = http_host
url.path = http_path
url.port = http_port
url.query = http_query
url.scheme = http_protocol
user_agent.original = useragent

static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = url.full

geoip = source destination
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[s3accesslog]
# https://docs.aws.amazon.com/ja_jp/AmazonS3/latest/dev/LogFormat.html
# https://aws.amazon.com/jp/premiumsupport/knowledge-center/analyze-logs-athena/
s3_key = 20\d{2}-[01]\d-\d{2}-\d{2}-\d{2}-\d{2}-[0-9A-Z]{16}$$
log_pattern = (?P<BucketOwner>[^ ]*) (?P<Bucket>[^ ]*) \[(?P<RequestDateTime>.*?)\] (?P<RemoteIP>[^ ]*) (?P<Requester>[^ ]*) (?P<RequestID>[^ ]*) (?P<Operation>[^ ]*) (?P<Key>[^ ]*) (\"(?P<RequestURI_operation>[^ ]*) (?P<RequestURI_key>[^ ]*) (?P<RequestURI_httpProtoversion>- |[^ ]*)\"|\"-\"|-) (?P<HTTPstatus>-|[0-9]*) (?P<ErrorCode>[^ ]*) (?P<BytesSent>[^ ]*) (?P<ObjectSize>[^ ]*) (?P<TotalTime>[^ ]*) (?P<TurnAroundTime>[^ ]*) (\"(?P<Referrer>.*?)\"|-) (\"(?P<UserAgent>.*?)\"|-) (?P<VersionId>[^ ]*)(?: (?P<HostId>[^ ]*) (?P<SigV>[^ ]*) (?P<CipherSuite>[^ ]*) (?P<AuthType>[^ ]*) (?P<EndPoint>[^ ]*) (?P<TLSVersion>[^ ]*))?( (?P<S3AccessPointARN>arn:[^ ]*|-))?
# " close asymmetric bracket
index_name = log-aws-s3accesslog
file_format = text
timestamp_key = RequestDateTime
timestamp_format = %d/%b/%Y:%H:%M:%S %z

ecs = destination.address
    destination.domain
    http.request.method
    http.request.referrer
    http.response.bytes
    http.response.status_code
    related.ip
    source.address
    source.ip
    url.domain
    url.original
    user.name
    user_agent.original
destination.address = ${destination.domain}
destination.domain = EndPoint
http.request.method = RequestURI_operation
http.request.referrer = Referrer
http.response.bytes = BytesSent
http.response.status_code = HTTPstatus
related.ip = [${source.ip}]
source.address = ${source.ip}
source.ip = RemoteIP
url.domain = EndPoint
url.original = RequestURI_key
user.name = Requester
user_agent.original = UserAgent

static_ecs = event.kind event.category
event.kind = event
event.category = web

script_ecs = user.name cloud.region

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[config-history]
index_name = log-aws-config
s3_key = _ConfigHistory_
file_format = json
json_delimiter = configurationItems
timestamp_key = configurationItemCaptureTime

ecs = cloud.account.id
    cloud.region
cloud.account.id = awsAccountId
cloud.region = awsRegion

static_ecs = event.kind event.category event.type
event.kind = state
event.category = configuration
event.type = info

script_ecs = @id cloud.instance.id related.hosts related.ip related.user

[config-snapshot]
# https://docs.aws.amazon.com/config/latest/APIReference/API_ConfigurationItem.html
index_name = log-aws-config
s3_key = _ConfigSnapshot_
file_format = json
json_delimiter = configurationItems
timestamp_key = file_timestamp
file_timestamp_format = _(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})T(?P<hour>\d{2})(?P<minute>\d{2})(?P<second>\d{2})Z_
json_to_text = configuration.EventPattern.source

ecs = cloud.account.id
    cloud.region
cloud.account.id = awsAccountId
cloud.region = awsRegion

static_ecs = event.kind event.category event.type
event.kind = state
event.category = configuration
event.type = info

script_ecs = @id cloud.instance.id related.hosts related.ip related.user

[config-rules]
index_name = log-aws-config
s3_key = Config.*Rules
file_format = json
timestamp_key = notificationCreationTime

ecs = cloud.account.id
    cloud.region
cloud.account.id = awsAccountId
cloud.region = awsRegion

static_ecs = event.kind event.category event.type
event.kind = alert
event.category = configuration
event.type = change

script_ecs = @id cloud.instance.id related.hosts related.ip related.user

[cloudfront-realtime]
# https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/real-time-logs.html
index_name = log-aws-cloudfront
s3_key = CloudFront/.*/realtime/
file_format = text
log_pattern = (?P<timestamp>[0-9\.]+)\t(?P<c_ip>[^\t]+)\t(?P<time_to_first_byte>[^\t]+)\t(?P<sc_status>[^\t]+)\t(?P<sc_bytes>[^\t]+)\t(?P<cs_method>[^\t]+)\t(?P<cs_protocol>[^\t]+)\t(?P<cs_host>[^\t]+)\t(?P<cs_uri_stem>[^\t]+)\t(?P<cs_bytes>[^\t]+)\t(?P<x_edge_location>[^\t]+)\t(?P<x_edge_request_id>[^\t]+)\t(?P<x_host_header>[^\t]+)\t(?P<time_taken>[^\t]+)\t(?P<cs_protocol_version>[^\t]+)\t(?P<c_ip_version>[^\t]+)\t(?P<cs_user_agent>[^\t]+)\t(?P<cs_referer>[^\t]+)\t(?P<cs_cookie>[^\t]+)\t(?P<cs_uri_query>[^\t]+)\t(?P<x_edge_response_result_type>[^\t]+)\t(?P<x_forwarded_for>[^\t]+)\t(?P<ssl_protocol>[^\t]+)\t(?P<ssl_cipher>[^\t]+)\t(?P<x_edge_result_type>[^\t]+)\t(?P<fle_encrypted_fields>[^\t]+)\t(?P<fle_status>[^\t]+)\t(?P<sc_content_type>[^\t]+)\t(?P<sc_content_len>[^\t]+)\t(?P<sc_range_start>[^\t]+)\t(?P<sc_range_end>[^\t]+)\t(?P<c_port>[^\t]+)\t(?P<x_edge_detailed_result_type>[^\t]+)\t(?P<c_country>[^\t]+)\t(?P<cs_accept_encoding>[^\t]+)\t(?P<cs_accept>[^\t]+)\t(?P<cache_behavior_path_pattern>[^\t]+)\t(?P<cs_headers>[^\t]+)\t(?P<cs_header_names>[^\t]+)\t(?P<cs_headers_count>[^\t]+)
timestamp_key = timestamp
timestamp_format = epoch

ecs = http.request.bytes
    http.request.method
    http.request.referrer
    http.response.bytes
    http.response.status_code
    related.ip
    source.address
    source.ip
    source.port
    url.domain
    url.path
    url.query
    url.scheme
    user_agent.original
http.request.bytes = cs_bytes
http.request.referrer = cs_referer
http.request.method = cs_method
http.response.bytes = sc_bytes
http.response.status_code = sc_status
related.ip = [${source.ip}]
source.address = ${source.ip}
source.ip = c_ip
source.port = c_port
url.domain = cs_host
# url.domain should be x_host_header, but cs_host is fine(2020/9/10)
url.path = cs_uri_stem
url.query = cs_uri_query
url.scheme = cs_protocol
user_agent.original = cs_user_agent

static_ecs = event.kind event.category cloud.region
event.kind = event
event.category = web
cloud.region = global

script_ecs = url.path url.full http.version user_agent.original

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[cloudfront-standard]
# https://docs.aws.amazon.com/ja_jp/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html
index_name = log-aws-cloudfront
# default log name example: ABCD0123456789.2020-09-05-22.a5fcfc8d.gz
s3_key = (^|\/)[0-9A-Z]{12,14}\.20\d{2}-\d{2}-\d{2}-\d{2}.[0-9a-z]{8}\.gz$$
file_format = text
text_header_line_number = 2
log_pattern = (?P<date_time>[0-9-:\t]+)\t(?P<x_edge_location>[0-9A-Z-]+)\t(?P<sc_bytes>[0-9]+)\t(?P<c_ip>[0-9a-f.:]+)\t(?P<cs_method>[A-Z]+)\t(?P<cs_host>[0-9A-Za-z.]+)\t(?P<cs_uri_stem>[^\t]+)\t(?P<sc_status>[0-9-]+)\t(?P<cs_referer>[^\t]+)\t(?P<cs_user_agent>[^\t]+)\t(?P<cs_uri_query>[^\t]+)\t(?P<cs_cookie>[^\t]+)\t(?P<x_edge_result_type>[^\t]+)\t(?P<x_edge_request_id>[^\t]+)\t(?P<x_host_header>[^\t]+)\t(?P<cs_protocol>[^\t]+)\t(?P<cs_bytes>[^\t]+)\t(?P<time_taken>[^\t]+)\t(?P<x_forwarded_for>[^\t]+)\t(?P<ssl_protocol>[^\t]+)\t(?P<ssl_cipher>[^\t]+)\t(?P<x_edge_response_result_type>[^\t]+)\t(?P<cs_protocol_version>[^\t]+)\t(?P<fle_status>[^\t]+)\t(?P<fle_encrypted_fields>[^\t]+)(\t(?P<c_port>[^\t]+)\t(?P<time_to_first_byte>[^\t]+)\t(?P<x_edge_detailed_result_type>[^\t]+)\t(?P<sc_content_type>[^\t]+)\t(?P<sc_content_len>[^\t]+)\t(?P<sc_range_start>[^\t]+)\t(?P<sc_range_end>[^\t]+))?
timestamp_key = date_time

ecs = http.request.bytes
    http.request.method
    http.request.referrer
    http.response.bytes
    http.response.status_code
    related.ip
    source.address
    source.ip
    source.port
    url.domain
    url.path
    url.query
    url.scheme
    user_agent.original
http.request.bytes = cs_bytes
http.request.referrer = cs_referer
http.request.method = cs_method
http.response.bytes = sc_bytes
http.response.status_code = sc_status
related.ip = [${source.ip}]
source.address = ${source.ip}
source.ip = c_ip
source.port = c_port
url.domain = x_host_header
url.path = cs_uri_stem
url.query = cs_uri_query
url.scheme = cs_protocol
user_agent.original = cs_user_agent

static_ecs = event.kind event.category cloud.region
event.kind = event
event.category = web
cloud.region = global

script_ecs = url.full http.version user_agent.original

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[waf]
# https://docs.aws.amazon.com/ja_jp/waf/latest/developerguide/logging.html
index_name = log-aws-waf
s3_key = aws-waf-logs-|_waflogs_
file_format = json
timestamp_key = timestamp
timestamp_format = epoch

ecs = event.action
    http.request.method
    related.ip
    rule.name
    source.address
    source.ip
    url.path
    url.query
event.action = action
http.request.method = httpRequest.httpMethod
related.ip = [${source.ip}]
rule.name = terminatingRuleId
source.address = ${source.ip}
source.ip = httpRequest.clientIp
url.path = httpRequest.uri
url.query = httpRequest.args

static_ecs = event.kind event.category cloud.region
event.kind = alert
event.category = web
cloud.region = global

script_ecs = user_agent.original http.request.referrer rule.ruleset cloud.region http.version cloud.account.id

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[route53resolver]
# known issue: only first dns answer maps to ECS
index_name = log-aws-r53resolver
s3_key = vpcdnsquerylogs
file_format = json
timestamp_key = query_timestamp

ecs = cloud.instance.id
    cloud.region
    dns.answers.class
    dns.answers.data
    dns.answers.type
    related.ip
    source.address
    source.ip
    source.port
    dns.question.class
    dns.question.name
    dns.question.type
    dns.response_code
cloud.instance.id = instance
cloud.region = region
dns.answers.class = answers.0.Class
dns.answers.data = answers.0.Rdata
dns.answers.type = answers.0.Type
related.ip = [${source.ip}]
source.address = ${source.ip}
source.ip = srcaddr
source.port = srcport
dns.question.name = query_name
dns.question.type = query_type
dns.question.class = query_class
dns.response_code = rcode

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = dns.answers.data dns.question.name

ioc_domain = dns.question.name
ioc_ip = dns.answers.data


[clientvpn]
s3_key = /ClientVPN/
index_name = log-aws-clientvpn
file_format = json
timestamp_key = connection-last-update-time
via_cwl = True

ecs = related.ip
    related.user
    source.ip
    source.port
    user.id
    user.name

related.ip = [device-ip, client-ip]
related.user = username
source.ip = device-ip
source.port = port
user.id = username
user.name = username

static_ecs = event.kind event.category
event.kind = event
event.category = network

script_ecs = error.message
    event.outcome
    network.bytes
    network.packets

geoip = source
ioc_ip = source.ip

field_prefix = clientvpn


[rds-postgresql]
s3_key = [Pp]ostgre
index_name = log-aws-rds-postgresql

via_cwl = True
file_format = multiline

multiline_firstline = ^20\d{2}-\d{2}-\d{2}
log_pattern = ^(?P<postgresql_timestamp>[\d-]+\s[\d:.]+)\s(UTC|GMT)(\s|:(|\[local\]|(?P<postgresql_source_address>[^ ]+)\((?P<postgresql_source_port>\d+)\)):(?P<postgresql_user>[^ ]*)@(?P<postgresql_database>[^ ]*):)\[(?P<postgresql_pid>[\d]+)\](:|\s)(?P<postgresql_log_level>[\w]+):\s+(?P<postgresql_message>[\s\S]*)

timestamp_key = postgresql_timestamp

ecs = log.level
    process.pid
    related.ip
    rds.database_name
    rds.message
    source.address
    source.ip
    source.port
    user.name
log.level = postgresql_log_level
process.pid = postgresql_pid
related.ip = ${source.ip}
rds.database_name = postgresql_database
rds.message = postgresql_message
source.address = postgresql_source_address
source.ip = postgresql_source_address
source.port = postgresql_source_port
user.name = postgresql_user

static_ecs = event.category
event.category = database

script_ecs = rds.query rds.query_time rds.cluster_identifier rds.instance_identifier event.category event.action event.outcome


[rds-mysql-audit]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(audit)
index_name = log-aws-rds-mysql

via_cwl = True

file_format = text
# Log Example
# Aurora-MySQL 56
# 1626329400410570,database-20210715-instance-1,rdsadmin,localhost,5,0,CONNECT,,,0
# Aurora-MySQL 57
# 20210715 05:45:55,ip-10-1-2-34,rdsadmin,localhost,36469,0,CONNECT,,,0
log_pattern = ^(?P<mysql_timestamp>[0-9: ]*),(?P<mysql_serverhost>[^,]*),(?P<mysql_username>[^,]*),(?P<mysql_host>[^,]*),(?P<mysql_thread_id>[\d*]*),(?P<mysql_queryid>[\d*]*),(?P<mysql_operation>[^,]*),(?P<mysql_database>[^,]*),(?P<mysql_object>.*),(?P<mysql_retcode>[^,]*)
timestamp_key = cwl_timestamp
timestamp_format = epoch

ecs = user.name
    rds.database_name
    rds.query
    related.ip
    related.user
    source.address
    source.ip
user.name = mysql_username rds.query
related.ip = [mysql_host]
related.user = [mysql_username, rds.query]
rds.database_name = mysql_database
rds.query = mysql_object
source.address = mysql_host
source.ip = mysql_host

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = audit


[rds-mysql-general]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(general)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ^20\d{2}-\d{2}-\d{2}

log_pattern = ^(?P<mysql_timestamp>[0-9-:.TZ]+)\s+(?P<mysql_thread_id>[0-9]+)\s+(?P<mysql_command>[\w]+)\t?(?P<mysql_query>[\s\S]*)?
timestamp_key = cwl_timestamp
timestamp_format = epoch

ecs = rds.query
rds.query = mysql_query

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = general


[rds-mysql-error]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(error)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ${rds-mysql-general:multiline_firstline}

# two log patterns #134
# "2021-02-18T12:30:41.777501Z 0 [Note] Shutting down slave threads"
# "210218 12:30:43 server_audit: STOPPED"
log_pattern = ^(?P<mysql_timestamp>([0-9-]+(\s|T)+[0-9:.]+Z?))\s+((?P<mysql_thread_id>[0-9]+)\s+\[(?P<mysql_log_level>[0-9a-zA-Z]+)\]\s)?(?P<mysql_message>.*)
timestamp_key = cwl_timestamp
timestamp_format = epoch

ecs = log.level rds.message
log.level = mysql_log_level
rds.message = mysql_message mysql_server_audit_message

script_ecs = rds.query

static_ecs = event.category event.module
event.category = database
event.module = error


[rds-mysql-slowquery]
s3_key = (MySQL|mysql|MariaDB|mariadb).*(slowquery)
index_name = log-aws-rds-mysql

via_cwl = True
file_format = multiline

multiline_firstline = ^# Time:

log_pattern = ^(# Time: (?P<mysql_timestamp>[0-9-:.TZ ]+)\n)?# User@Host: (?P<mysql_username>[^\[]*).*\[(?P<mysql_source_ip>[^\]]*).*Id:\s+(?P<mysql_thread_id>[0-9]+)\n# Query_time: (?P<mysql_query_time>[0-9.]+)\s+Lock_time: (?P<mysql_lock_time>[0-9.]+).*\n(?P<mysql_query>[\s\S]*)
# example)
# Aurora MySQL v56
# # Time: 210723 12:33:18
# Aurora MySQL v57
# # Time: 2021-02-22T09:08:46.414775Z
# timestamp_key = mysql_timestamp
# timestamp_format = iso8601
timestamp_key = cwl_timestamp
timestamp_format = epoch

ecs = rds.query
    rds.query_time
    related.ip
    related.user
    source.ip
    user.name
rds.query = mysql_query
rds.query_time = mysql_query_time
related.ip = [mysql_source_ip]
related.user = [mysql_username]
source.ip = mysql_source_ip
user.name = mysql_username

script_ecs = rds.query rds.cluster_identifier rds.instance_identifier

static_ecs = event.category event.module
event.category = database
event.module = slowquery


[elasticache-redis-slowlog]
# https://docs.aws.amazon.com/ja_jp/AmazonElastiCache/latest/red-ug/Log_Delivery.html
s3_key = (redis|Redis).*(slow|SLOW)
index_name = log-aws-elasticache
file_format = json
timestamp_key = Timestamp
timestamp_format = epoch

renamed_newfields = Duration_us
Duration_us = Duration (us)
field_prefix = elasticache

ecs = source.ip
source.ip = ClientAddress

static_ecs = event.category event.module
event.category = database
event.module = redis-slowlog

script_ecs = source.ip source.port related.ip


[msk]
s3_key = KafkaBrokerLogs
index_name = log-aws-msk
file_format = multiline

#[2021-02-25 08:55:07,452]
multiline_firstline = ^\[20\d{2}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
log_pattern = ^\[(?P<msk_timestamp>[0-9 -:,]+)\] (?P<msk_log_level>[^ ]+) (?P<msk_message>[\s\S]*)

timestamp_key = msk_timestamp
timestamp_format = %Y-%m-%d %H:%M:%S,%f

ecs = log.level
log.level = msk_log_level

script_ecs = msk


[cloudhsm]
index_name = log-aws-cloudhsm
s3_key = /CloudHSM/
file_format = multiline

via_cwl = True
multiline_firstline = ^Time: d{2}/\d{2}/\d{2}

log_pattern = .*Time: (?P<timestamp>.*?), usecs:(?P<timestamp_usec>\d*)(
 Version No\s*: (?P<version_no>.*))?
 Sequence No\s*: (?P<sequence_no>0x\w*)
 Reboot counter\s*: (?P<reboot_counter>0x\w*)(
 Opcode\s*: (?P<opcode_v2>\w*) \((?P<opcode_hex_v2>\w*)\))?
 Command Type\(hex\)\s*: (?P<command_type>\w*) \((?P<command_type_hex>\w*)\)(
 Opcode\s*: (?P<opcode>\w*) \((?P<opcode_hex>\w*)\))?(
 User id\s*: (?P<user_id>\w*))?
 Session Handle\s*: (?P<session_handle>\w*)
 Response\s*: (?P<response>\w*):(HSM Return: (?P<hsm_return>\w*))?(HSM Error: (?P<hsm_error>.*))?
 Log type\s*: (?P<log_type>\w*) \((?P<log_type_code>\d*)\)(
 Priv/Secret Key Handle\s*: (?P<priv_secret_key_handle>\d*))?(
 Public Key Handle\s*: (?P<public_key_handle>\d*))?(
 Priv/Secret Key Handle\s*: (?P<priv_secret_key_handle_v2>\d*))?(
 Key Handle\s*: (?P<key_handle>\d*))?(
 local time\s*: (?P<local_time>.*), usec:(?P<local_time_usec>\d*)
 host time\s*: (?P<host_time>.*) usec:(?P<host_time_usec>\d*))?(
 propagation delay\s*: usec:(?P<propagation_delay>\d*))?(
 User Name\s*: (?P<user_name>\w*)
 User Type\s*: (?P<user_type>\w*) \((?P<user_type_code>\w*)\))?(
 Mechanism\s*: (?P<mechanism>\d*))?(
 Target User Name\s*: (?P<target_user_name>\w*)
 Target User Type\s*: (?P<target_user_type>\w*) \((?P<target_user_type_code>\w*)\)
 Target User id\s*: (?P<target_user_id>\w*))?(
 Mfa enabled\s*: (?P<mfa_enabled>\w*))?(
 Operation\s*: (?P<operation>\w*) \((?P<operation_code>\d*)\)
 Number of Users\s*: (?P<number_of_users>\d*)
 UserIDs\s*: (?P<user_ids>\w*))?(
 (?P<unknown_field_value>[\s\S]*))?

timestamp_key = timestamp
timestamp_format = %m/%d/%y %H:%M:%S.%f
# 04/23/22 14:55:37.147946
field_prefix = cloudhsm

ecs = user.id
user.id = user_id

script_ecs = cloud.instance.id user.name

# Log type : MGMT_KEY_DETAILS_LOG (1)
# Priv/Secret Key Handle : 131079
# Public Key Handle : 0
# Key Handle : 2490398

# Log type : MGMT_USER_DETAILS_LOG (2)
# User Name	: kmsuser
# User Type	: CN_CRYPTO_USER (1)

# Log type : TIME_INFO_LOG (16)
# local time : 04/30/22 20:13:18.189504, usec:1650658398189504
# host time : 04/30/22 20:13:18.191729 usec:1650658398191729
# propagation delay	: usec:39


[opensearch-audit]
s3_key = (OpenSearch|opensearch).*(Audit|audit)
index_name = log-aws-opensearch
file_format = json
timestamp_key = timestamp
via_cwl = True

renamed_newfields = timestamp
timestamp = @timestamp
field_prefix = opensearch

ecs = related.hosts
    related.ip
    related.user
    rule.name
    source.ip
    user_agent.original
    user.name
related.hosts = [audit_rest_request_headers.Host]
related.ip = [audit_request_remote_address, audit_rest_request_headers.Host]
related.user = [audit_request_effective_user, audit_request_initiating_user]
rule.name = audit_transport_request_type audit_rest_request_method audit_category
source.ip = audit_request_remote_address
user_agent.original = audit_rest_request_headers.User-Agent
user.name = audit_request_effective_user, audit_request_initiating_user

static_ecs = event.kind
    event.category
    event.type
event.kind = event
event.category = [database]
event.type = [info]

script_ecs = event.category event.outcome

geoip = source
ioc_ip = source.ip
user_agent_enrichment_field = user_agent


[workspaces-event]
s3_key = (WorkSpaces|workspaces).*(Event|event)
index_name = log-aws-workspaces
file_format = json
#"loginTime": "2018-07-01T17:52:51.595Z",
timestamp_key = loginTime
timestamp_format = iso8601

renamed_newfields = WorkspaceId
WorkspaceId = workspaceId

ecs = host.id
    related.ip
    related.hosts
    source.ip
host.id = workspaceId
related.ip = [clientIpAddress]
related.hosts = [workspaceId]
source.ip = clientIpAddress

static_ecs = event.kind
    event.category
    event.type
    event.outcome
event.kind = event
event.category = [authentication, host]
event.type = [info]
event.outcome = success

geoip = source
ioc_ip = source.ip


[workspaces-inventory]
s3_key = (WorkSpaces|workspaces).*(Inventory|inventory)
index_name = log-aws-workspaces
file_format = json
json_delimiter = Workspaces
#"2021-07-04T09:58:54Z"
timestamp_key = cwe_timestamp
timestamp_format = iso8601

ecs = host.id
    host.ip
    host.name
    host.hostname
    related.hosts
    related.ip
    related.user
    user.id
    user.name
host.id = WorkspaceId
host.ip = IpAddress
host.name = ComputerName
host.hostname = ComputerName
related.hosts = [ComputerName, WorkspaceId]
related.ip = [IpAddress]
related.user = [UserName]
user.id = UserName
user.name = UserName

static_ecs = event.kind event.category event.type
event.kind = state
event.category = [host]
event.type = [info]


[trustedadvisor]
s3_key = /TrustedAdvisor/
index_name = log-aws-trustedadvisor
file_format = json
timestamp_key = creation_date
timestamp_format = iso8601

ecs = cloud.region
    rule.name

cloud.region = result.flaggedResource.region
rule.name = check.name

static_ecs = event.kind event.category event.type event.module
event.kind = event
event.category = configuration
event.type = info
event.module = trustedadvisor

script_ecs = event.category event.kind

field_prefix = trustedadvisor


[directory-service]
s3_key = /DirectoryService/MicrosoftAD/
index_name = log-aws-directory-service
file_format = winevtxml
via_cwl = True

timestamp_key = ${windows-event:timestamp_key}
timestamp_nano = ${windows-event:timestamp_nano}
timestamp_format = ${windows-event:timestamp_format}

ecs = destination.ip
    destination.port
    error.code
    event.code
    event.module
    host.name
    related.ip
    related.user
    source.ip
    source.port
    user.domain
    user.id
    user.name
destination.ip = ${windows-event:destination.ip}
destination.port = ${windows-event:destination.port}
error.code = ${windows-event:error.code}
event.code = ${windows-event:event.code}
event.module = ${windows-event:event.module}
host.name = ${windows-event:host.name}
related.ip = ${windows-event:related.ip}
related.user = ${windows-event:related.user}
source.ip = ${windows-event:source.ip}
source.port = ${windows-event:source.port}
user.domain = ${windows-event:user.domain}
user.id = ${windows-event:user.id}
user.name = ${windows-event:user.name}

static_ecs = event.kind
event.kind = event

script_ecs = event.action event.outcome

geoip = ${windows-event:geoip}
ioc_ip = ${windows-event:ioc_ip}


[fsx-win]
s3_key = /aws-fsx-
index_name = log-aws-fsx-win
file_format = winevtxml
via_cwl = False

timestamp_key = ${windows-event:timestamp_key}
timestamp_nano = ${windows-event:timestamp_nano}
timestamp_format = ${windows-event:timestamp_format}

ecs = destination.ip
    destination.port
    error.code
    event.code
    event.module
    host.name
    related.ip
    related.user
    source.ip
    source.port
    user.domain
    user.id
    user.name
destination.ip = ${windows-event:destination.ip}
destination.port = ${windows-event:destination.port}
error.code = ${windows-event:error.code}
event.code = ${windows-event:event.code}
event.module = ${windows-event:event.module}
host.name = ${windows-event:host.name}
related.ip = ${windows-event:related.ip}
related.user = ${windows-event:related.user}
source.ip = ${windows-event:source.ip}
source.port = ${windows-event:source.port}
user.domain = ${windows-event:user.domain}
user.id = ${windows-event:user.id}
user.name = ${windows-event:user.name}

static_ecs = event.kind
event.kind = event

script_ecs = event.action event.outcome

geoip = ${windows-event:geoip}
ioc_ip = ${windows-event:ioc_ip}


[windows-event]
s3_key = /[Ww]indows.*[Ee]vent
index_name = log-win-event
file_format = winevtxml
via_cwl = True

timestamp_key = Event.System.TimeCreated.SystemTime
timestamp_nano = True
timestamp_format = iso8601

ecs = destination.ip
    destination.port
    error.code
    event.code
    event.module
    host.name
    related.ip
    related.user
    source.ip
    source.port
    user.domain
    user.id
    user.name
destination.ip = Event.EventData.Data.DestAddress
destination.port = Event.EventData.Data.DestPort
error.code = Event.System.Status
event.code = Event.System.EventID
event.module = Event.System.Channel
host.name = Event.System.Computer
related.ip = [Event.EventData.Data.DestAddress, Event.EventData.Data.IpAddress, Event.EventData.Data.SourceAddress]
related.user = [Event.EventData.Data.SubjectUserName, Event.EventData.Data.TargetUserName]
source.ip = Event.EventData.Data.IpAddress Event.EventData.Data.SourceAddress
source.port = Event.EventData.Data.IpPort Event.EventData.Data.SourcePort
user.domain = Event.EventData.Data.SubjectDomainName Event.EventData.Data.TargetDomainName
user.id = Event.EventData.Data.SubjectUserSid Event.EventData.Data.TargetUserSid
user.name = Event.EventData.Data.SubjectUserName Event.EventData.Data.TargetUserName

static_ecs = event.kind
event.kind = event

script_ecs = event.action event.outcome cloud.instance.id

geoip = source destination
ioc_ip = source.ip


[linux-secure]
# Inheritance from linux-os-syslog
s3_key = /[Ll]inux.*[Ss]ecure
index_name = log-linux-secure

via_cwl = ${linux-os-syslog:via_cwl}
# loggroup = secure
file_format = ${linux-os-syslog:file_format}

timestamp_key = ${linux-os-syslog:timestamp_key}
timestamp_format = ${linux-os-syslog:timestamp_format}
log_pattern = ${linux-os-syslog:log_pattern}

ecs = ${linux-os-syslog:ecs}
host.hostname = ${linux-os-syslog:host.hostname}
process.name = ${linux-os-syslog:process.name}
process.pid = ${linux-os-syslog:process.pid}
related.hosts = ${linux-os-syslog:related.hosts}

static_ecs = ${linux-os-syslog:static_ecs}
event.kind = ${linux-os-syslog:event.kind}

script_ecs = ${linux-os-syslog:script_ecs}

geoip = ${linux-os-syslog:geoip}
ioc_ip = ${linux-os-syslog:ioc_ip}


[linux-os-syslog]
# overwrite index_name according to proc
s3_key = /[Ll]inux/
index_name = log-linux-os
via_cwl = True
# loggroup = secure
file_format = text
timestamp_key = syslog_timestamp
timestamp_format = syslog
log_pattern = (?P<syslog_timestamp>[^ ]*\s*[^ ]*\s[^ ]*) (?P<hostname>[^ ]*) (?P<proc>[a-zA-Z0-9_\/\.\-]*)(?:\[(?P<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?P<syslog_message>.*)

ecs = host.hostname process.name process.pid related.hosts
host.hostname = hostname
process.name = proc
process.pid = pid
related.hosts = hostname

static_ecs = event.kind
event.kind = event

script_ecs = cloud.instance.id event.action event.category event.outcome related.ip related.hosts related.user source.ip source.port user.name user.id

geoip = source
ioc_ip = source.ip


[index-metrics]
index_name = metrics-opensearch-index
index_rotation = monthly
s3_key = /OpenSearch/metrics/
file_format = json