{{- if .Values.monitoring.enabled -}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
namespace: amazon-cloudwatch
labels:
app: fluent-bit
component: metric-collector
part-of: logging-monitoring
data:
fluent-bit.conf: |
[SERVICE]
Flush 5
Log_Level Debug
Daemon off
Parsers_File parsers.conf
storage.path /var/fluent-bit/state/flb-storage/
storage.sync normal
storage.checksum off
storage.backlog.mem_limit 5M
@INCLUDE dataplane-log.conf
@INCLUDE host-log.conf
dataplane-log.conf: |
[INPUT]
Name systemd
Tag dataplane.systemd.*
Systemd_Filter _SYSTEMD_UNIT=docker.service
DB /var/fluent-bit/state/systemd.db
Path /var/log/journal
Read_From_Tail ${READ_FROM_TAIL}
[INPUT]
Name tail
Tag dataplane.tail.*
Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser container_firstline
Parser docker
DB /var/fluent-bit/state/flb_dataplane_tail.db
Mem_Buf_Limit 50MB
Skip_Long_Lines On
Refresh_Interval 10
Rotate_Wait 30
storage.type filesystem
Read_from_Head ${READ_FROM_HEAD}
[FILTER]
Name modify
Match dataplane.systemd.*
Rename _HOSTNAME hostname
Rename _SYSTEMD_UNIT systemd_unit
Rename MESSAGE message
Remove_regex ^((?!hostname|systemd_unit|message).)*$
[FILTER]
Name aws
Match dataplane.*
imds_version v1
[OUTPUT]
Name cloudwatch_logs
Match dataplane.*
region ${AWS_REGION}
log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane
log_stream_prefix ${HOST_NAME}-
auto_create_group true
extra_user_agent container-insights
{{- if .Values.fluentbit.output.elasticsearch }}
[OUTPUT]
Name es
Match dataplane.*
Host {{ .Values.fluentbit.output.elasticsearch.host }}
Port {{ .Values.fluentbit.output.elasticsearch.port }}
Index dataplane.fluent.${CLUSTER_NAME}
Generate_ID On
AWS_Auth On
AWS_Region ${AWS_REGION}
{{- end }}
{{- if .Values.fluentbit.output.kinesis.dataplane }}
[OUTPUT]
Name kinesis
Match dataplane.*
region ${AWS_REGION}
stream {{ .Values.fluentbit.output.kinesis.dataplane.stream }}
{{- end }}
{{- if .Values.fluentbit.output.s3.dataplane }}
[OUTPUT]
Name s3
Match dataplane.*
region ${AWS_REGION}
bucket {{ .Values.fluentbit.output.s3.dataplane.bucket }}
{{- end }}
host-log.conf: |
[INPUT]
Name tail
Tag host.dmesg
Path /var/log/dmesg
Parser syslog
DB /var/fluent-bit/state/flb_dmesg.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head ${READ_FROM_HEAD}
[INPUT]
Name tail
Tag host.messages
Path /var/log/messages
Parser syslog
DB /var/fluent-bit/state/flb_messages.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head ${READ_FROM_HEAD}
[INPUT]
Name tail
Tag host.secure
Path /var/log/secure
Parser syslog
DB /var/fluent-bit/state/flb_secure.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head ${READ_FROM_HEAD}
[FILTER]
Name aws
Match host.*
imds_version v1
[OUTPUT]
Name cloudwatch_logs
Match host.*
region ${AWS_REGION}
log_group_name /aws/containerinsights/${CLUSTER_NAME}/host
log_stream_prefix ${HOST_NAME}.
auto_create_group true
extra_user_agent container-insights
{{- if .Values.fluentbit.output.elasticsearch }}
[OUTPUT]
Name es
Match host.*
Host {{ .Values.fluentbit.output.elasticsearch.host }}
Port {{ .Values.fluentbit.output.elasticsearch.port }}
Index host.fluent.${CLUSTER_NAME}
Generate_ID On
AWS_Auth On
AWS_Region ${AWS_REGION}
{{- end }}
{{- if .Values.fluentbit.output.kinesis.host }}
[OUTPUT]
Name kinesis
Match host.*
region ${AWS_REGION}
stream {{ .Values.fluentbit.output.kinesis.host.stream }}
{{- end }}
{{- if .Values.fluentbit.output.s3.host }}
[OUTPUT]
Name s3
Match host.*
region ${AWS_REGION}
bucket {{ .Values.fluentbit.output.s3.host.bucket }}
{{- end}}
application-log.conf: |
[INPUT]
Name tail
Tag application.*
Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
Path /var/log/containers/*.log
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser container_firstline
Parser docker
DB /var/fluent-bit/state/flb_container.db
Mem_Buf_Limit 50MB
Skip_Long_Lines On
Refresh_Interval 10
Rotate_Wait 30
storage.type filesystem
Read_from_Head ${READ_FROM_HEAD}
[INPUT]
Name tail
Tag application.*
Path /var/log/containers/fluent-bit*
Parser docker
DB /var/fluent-bit/state/flb_log.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head ${READ_FROM_HEAD}
[INPUT]
Name tail
Tag application.*
Path /var/log/containers/cloudwatch-agent*
Docker_Mode On
Docker_Mode_Flush 5
Docker_Mode_Parser cwagent_firstline
Parser docker
DB /var/fluent-bit/state/flb_cwagent.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
Read_from_Head ${READ_FROM_HEAD}
[FILTER]
Name kubernetes
Match application.*
Kube_URL https://kubernetes.default.svc:443
Kube_Tag_Prefix application.var.log.containers.
Merge_Log On
Merge_Log_Key log_processed
K8S-Logging.Parser On
K8S-Logging.Exclude Off
Labels Off
Annotations Off
[FILTER]
Name aws
Match application.*
imds_version v1
[OUTPUT]
Name cloudwatch_logs
Match host.*
region ${AWS_REGION}
log_group_name /aws/containerinsights/${CLUSTER_NAME}/application
log_stream_prefix ${HOST_NAME}.
auto_create_group true
extra_user_agent container-insights
{{- if .Values.fluentbit.output.elasticsearch }}
[OUTPUT]
Name es
Match application.*
Host {{ .Values.fluentbit.output.elasticsearch.host }}
Port {{ .Values.fluentbit.output.elasticsearch.port }}
Index application.fluent.${CLUSTER_NAME}
Generate_ID On
AWS_Auth On
AWS_Region ${AWS_REGION}
{{- end }}
{{- if .Values.fluentbit.output.kinesis.application }}
[OUTPUT]
Name kinesis
Match application.*
region ${AWS_REGION}
stream {{ .Values.fluentbit.output.kinesis.application.stream }}
{{- end }}
{{- if .Values.fluentbit.output.s3.application }}
[OUTPUT]
Name s3
Match application.*
region ${AWS_REGION}
bucket {{ .Values.fluentbit.output.s3.application.bucket }}
{{- end }}
parsers.conf: |
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
[PARSER]
Name syslog
Format regex
Regex ^(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S
[PARSER]
Name container_firstline
Format regex
Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=})
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-cluster-info
namespace: amazon-cloudwatch
labels:
app: fluent-bit
component: metric-collector
part-of: logging-monitoring
data:
cluster.name: "{{ .Values.global.clusterName }}"
logs.region: "{{ .Values.global.region }}"
read.head: "True"
read.tail: "On"
{{- end -}}