--- apiVersion: helm.toolkit.fluxcd.io/v2beta1 kind: HelmRelease metadata: name: fluent-bit namespace: flux-system spec: releaseName: fluent-bit targetNamespace: aws-system storageNamespace: aws-system interval: 10m0s chart: spec: chart: fluent-bit version: 0.16.2 sourceRef: kind: HelmRepository name: fluent values: clusterName: sandbox image: repository: public.ecr.aws/aws-observability/aws-for-fluent-bit tag: latest serviceAccount: create: false name: aws-fluent-bit env: - name: HOST_NAME valueFrom: fieldRef: fieldPath: spec.nodeName http: port: 2020 server: "On" tailReadFromHead: "On" systemdReadFromTail: "Off" region: us-west-2 #! https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file config: service: | [SERVICE] Flush 5 Log_Level {{ .Values.logLevel }} Daemon off Parsers_File parsers.conf Parsers_File custom_parsers.conf HTTP_Server {{ .Values.http.server }} HTTP_Listen 0.0.0.0 HTTP_Port {{ .Values.service.port }} storage.path /var/fluent-bit/state/flb-storage/ storage.sync normal storage.checksum off storage.backlog.mem_limit 5M #! https://docs.fluentbit.io/manual/pipeline/inputs inputs: | [INPUT] Name tail Tag application.* Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* Path /var/log/containers/*.log Docker_Mode On Docker_Mode_Flush 5 Docker_Mode_Parser container_firstline Parser docker_cwci DB /var/fluent-bit/state/flb_container.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 Rotate_Wait 30 storage.type filesystem Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name tail Tag application.* Path /var/log/containers/fluent-bit* Parser docker_cwci DB /var/fluent-bit/state/flb_log.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name tail Tag application.* Path /var/log/containers/cloudwatch-agent* Docker_Mode On Docker_Mode_Flush 5 Docker_Mode_Parser cwagent_firstline Parser docker_cwci DB /var/fluent-bit/state/flb_cwagent.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name systemd Tag dataplane.systemd.* Systemd_Filter _SYSTEMD_UNIT=docker.service Systemd_Filter _SYSTEMD_UNIT=kubelet.service DB /var/fluent-bit/state/systemd.db Path /var/log/journal Read_From_Tail {{ .Values.systemdReadFromTail }} [INPUT] Name tail Tag dataplane.tail.* Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* Docker_Mode On Docker_Mode_Flush 5 Docker_Mode_Parser container_firstline Parser docker_cwci DB /var/fluent-bit/state/flb_dataplane_tail.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 Rotate_Wait 30 storage.type filesystem Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name tail Tag host.dmesg Path /var/log/dmesg Parser syslog DB /var/fluent-bit/state/flb_dmesg.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name tail Tag host.messages Path /var/log/messages Parser syslog DB /var/fluent-bit/state/flb_messages.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 Read_from_Head {{ .Values.tailReadFromHead }} [INPUT] Name tail Tag host.secure Path /var/log/secure Parser syslog DB /var/fluent-bit/state/flb_secure.db Mem_Buf_Limit 5MB Skip_Long_Lines On Refresh_Interval 10 Read_from_Head {{ .Values.tailReadFromHead }} #! https://docs.fluentbit.io/manual/pipeline/filters filters: | [FILTER] Name kubernetes Match application.* Kube_URL https://kubernetes.default.svc:443 Kube_Tag_Prefix application.var.log.containers. Merge_Log On Merge_Log_Key log_processed K8S-Logging.Parser On K8S-Logging.Exclude Off Labels Off Annotations Off [FILTER] Name modify Match dataplane.systemd.* Rename MESSAGE message Rename _HOSTNAME hostname Rename _SYSTEMD_UNIT systemd_unit Remove_regex ^((?!hostname|systemd_unit|message).)*$ [FILTER] Name aws Match dataplane.* imds_version v1 [FILTER] Name aws Match host.* imds_version v1 #! https://docs.fluentbit.io/manual/pipeline/outputs outputs: | [OUTPUT] Name cloudwatch_logs Match application.* region {{ .Values.region }} log_group_name /aws/containerinsights/{{ .Values.clusterName }}/application log_stream_prefix ${HOST_NAME}- auto_create_group true extra_user_agent container-insights [OUTPUT] Name cloudwatch_logs Match dataplane.* region {{ .Values.region }} log_group_name /aws/containerinsights/{{ .Values.clusterName }}/dataplane log_stream_prefix ${HOST_NAME}- auto_create_group true extra_user_agent container-insights [OUTPUT] Name cloudwatch_logs Match host.* region {{ .Values.region }} log_group_name /aws/containerinsights/{{ .Values.clusterName }}/host log_stream_prefix ${HOST_NAME}. auto_create_group true extra_user_agent container-insights #! https://docs.fluentbit.io/manual/pipeline/parsers customParsers: | [PARSER] Name docker_cwci Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%LZ [PARSER] Name syslog Format regex Regex ^(?