[SERVICE] HTTP_Server On HTTP_Listen 0.0.0.0 HTTP_PORT 2020 Parsers_File /fb_metrics_parser.conf storage.metrics On # Configure FB to scrape its own prom metrics [INPUT] Name exec Command curl -s http://127.0.0.1:2020/api/v1/metrics/prometheus Interval_Sec 30 Tag fb_metrics # Filter out everything except output metrics # TODO: REMOVE this filter if you want INPUT PLUGIN METRICS [FILTER] Name grep Match fb_metrics Regex exec output # Filter out the HELP and TYPE fields which aren't parseable by the cw metric filter [FILTER] Name grep Match fb_metrics Exclude exec HELP [FILTER] Name grep Match fb_metrics Exclude exec TYPE # Parse the metrics to json for easy parsing in CW Log Group Metrics filter [FILTER] Name parser Match fb_metrics Key_Name exec Parser fluentbit_prom_metrics_to_json Reserve_Data True # Un-comment this section if you want storage metrics # Unfortunately, Fluent Bit publishes storage metrics via # a different HTTP path which vends metrics in JSON instead of Prometheus # These metrics can be sent by the same output as the prom metrics #[INPUT] # Name exec # Command curl -s http://127.0.0.1:2020/api/v1/storage && echo "" # Interval_Sec 5 # Tag fb_metrics-storage # Parse the JSON metrics #[FILTER] # Name parser # Match fb_metrics-storage # Key_Name exec # Parser json # Add hostname to metrics to identify them # [FILTER] # Name record_modifier # Match * # Record hostname ${HOSTNAME} # Send the metrics as CW Logs # The CW Metrics filter on the log group will turn them into metrics # Use hostname in logs to differentiate log streams per task in Fargate # Alternative is to use: https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit#templating-log-group-and-stream-names [OUTPUT] Name cloudwatch_logs Match fb_metrics* region ${FLUENT_BIT_METRICS_LOG_REGION} log_group_name ${FLUENT_BIT_METRICS_LOG_GROUP} log_stream_name ${HOSTNAME}-fb-internal-metrics auto_create_group On retry_limit 2 workers 1