Commit 154327e3 authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Merge branch 'dev' into 'master' 

Introduce label para nivel de log, reestructura

See merge request redmic-project/log/fluent-bit!10
parents bd50fc5f cdf199b0

README.md

+28 −0
Original line number Diff line number Diff line 
# Fluent Bit



Configure and deploy Fluent Bit, a lightweight service to ingest logs and feed Loki



## Log level recognition



There are 6 log levels to be recognised (inspired by [Grafana](https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level)): `critical`, `error`, `warning`, `info`, `debug` and `trace`.



If none of them is recognised, then no `level` label is applied to log message. This way, another tool (like Grafana Logs Panel) may infer it's level later.



Detection is based on finding a **field** with level value or a level value between **delimiters**.



You can customise detected log level values setting these environment variables:



| Variable name | Default value |

| - | - |

| LOG_LEVEL_CRITICAL_VALUES | emerg\|fatal\|alert\|crit\|critical\|Emerg\|Fatal\|Alert\|Crit\|Critical\|EMERG\|FATAL\|ALERT\|CRIT\|CRITICAL |

| LOG_LEVEL_ERROR_VALUES | eror\|err\|error\|Eror\|Err\|Error\|EROR\|ERR\|ERROR |

| LOG_LEVEL_WARNING_VALUES | warn\|warning\|Warn\|Warning\|WARN\|WARNING |

| LOG_LEVEL_INFO_VALUES | info\|information\|informational\|notice\|Info\|Information\|Informational\|Notice\|INFO\|INFORMATION\|INFORMATIONAL\|NOTICE |

| LOG_LEVEL_DEBUG_VALUES | dbug\|debug\|Dbug\|Debug\|DBUG\|DEBUG |

| LOG_LEVEL_TRACE_VALUES | trace\|Trace\|TRACE |



Also, you can change detected field names and delimeters using these environment variables:



| Variable name | Default value |

| - | - |

| LOG_LEVEL_FIELDS | level=\|lvl=\|Level=\|Lvl=\|LEVEL=\|LVL= |

| LOG_LEVEL_DELIMITERS | \d\|\W\|\dm |



> Value `\dm` used at `${LOG_LEVEL_DELIMITERS}` allows recognising coloured logs with ANSI codes (ending with an integer followed by `m`).

deploy/.env

+11 −1
Original line number Diff line number Diff line 
LOG_LEVEL=info

LOG_LEVEL=warn

INPUT_TAG_PREFIX=docker.

LOKI_HOST=loki

EMPTY_LABEL_VALUE=none



LOG_LEVEL_FIELDS=level=|lvl=|Level=|Lvl=|LEVEL=|LVL=

LOG_LEVEL_DELIMITERS=\d|\W|\dm



LOG_LEVEL_CRITICAL_VALUES=emerg|fatal|alert|crit|critical|Emerg|Fatal|Alert|Crit|Critical|EMERG|FATAL|ALERT|CRIT|CRITICAL

LOG_LEVEL_ERROR_VALUES=eror|err|error|Eror|Err|Error|EROR|ERR|ERROR

LOG_LEVEL_WARNING_VALUES=warn|warning|Warn|Warning|WARN|WARNING

LOG_LEVEL_INFO_VALUES=info|information|informational|notice|log|Info|Information|Informational|Notice|Log|INFO|INFORMATION|INFORMATIONAL|NOTICE|LOG

LOG_LEVEL_DEBUG_VALUES=dbug|debug|Dbug|Debug|DBUG|DEBUG

LOG_LEVEL_TRACE_VALUES=trace|Trace|TRACE



PORT=24224

deploy/conf/fluent-bit.conf

+87 −4
Original line number Diff line number Diff line 
[SERVICE]

	flush 1

	log_level ${LOG_LEVEL}

	http_server on



[INPUT]

	name forward



[FILTER]

	name rewrite_tag

	alias tag_level_critical_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $container_name ^\/([^_]+)_([^.]+)\.([^.]+)\.(.*)$ docker-swarm.$1.$2.$3.$4 false

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_CRITICAL_VALUES}).*$ log-level.critical false



[FILTER]

	name rewrite_tag

	alias tag_level_error_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $container_name ^\/(.*)$ docker-container.$1 false

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_ERROR_VALUES}).*$ log-level.error false



[FILTER]

	name rewrite_tag

	alias tag_level_warning_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_WARNING_VALUES}).*$ log-level.warning false



[FILTER]

	name rewrite_tag

	alias tag_level_info_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_INFO_VALUES}).*$ log-level.info false



[FILTER]

	name rewrite_tag

	alias tag_level_debug_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_DEBUG_VALUES}).*$ log-level.debug false



[FILTER]

	name rewrite_tag

	alias tag_level_trace_from_field

	match ${INPUT_TAG_PREFIX}*

	rule $log ^.*?(?:${LOG_LEVEL_FIELDS})(${LOG_LEVEL_TRACE_VALUES}).*$ log-level.trace false



[FILTER]

	name rewrite_tag

	alias tag_level_critical_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_CRITICAL_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.critical false



[FILTER]

	name rewrite_tag

	alias tag_level_error_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_ERROR_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.error false



[FILTER]

	name rewrite_tag

	alias tag_level_warning_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_WARNING_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.warning false



[FILTER]

	name rewrite_tag

	alias tag_level_info_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_INFO_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.info false



[FILTER]

	name rewrite_tag

	alias tag_level_debug_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_DEBUG_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.debug false



[FILTER]

	name rewrite_tag

	alias tag_level_trace_from_delimiter

	match ${INPUT_TAG_PREFIX}*

	rule $log ^(?:.*?(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}))?(${LOG_LEVEL_TRACE_VALUES})(?:${LOG_LEVEL_DELIMITERS})(?:${LOG_LEVEL_DELIMITERS}).*$ log-level.trace false



[FILTER]

	name rewrite_tag

	alias tag_level_unknown

	match ${INPUT_TAG_PREFIX}*

	rule $log ^.*$ log-level. false



[FILTER]

	name rewrite_tag

	match log-level.*

	rule $container_name ^\/([^_]+)_([^.]+)\.([^.]+)\.?(.*)?$ docker-swarm.$1.$2.$3.$4.$TAG[1] false



[FILTER]

	name rewrite_tag

	match log-level.*

	rule $container_name ^\/(.*)$ docker-container.$1.$TAG[1] false



[OUTPUT]

	name loki

	match docker-swarm.*

	host ${LOKI_HOST}

	labels stack_name=$TAG[1], service_name=$TAG[1]_$TAG[2], slot=$TAG[3], container_name=$TAG[1]_$TAG[2].$TAG[3].$TAG[4]

	labels stack_name=$TAG[1], service_name=$TAG[1]_$TAG[2], slot=$TAG[3], container_name=$TAG[1]_$TAG[2].$TAG[3].$TAG[4], level=$TAG[5]

	label_keys $container_id

	remove_keys container_id, container_name, source

	drop_single_key on

	line_format key_value



[OUTPUT]

	name loki

	match docker-container.*

	host ${LOKI_HOST}

	labels stack_name=${EMPTY_LABEL_VALUE}, service_name=${EMPTY_LABEL_VALUE}, slot=${EMPTY_LABEL_VALUE}, container_name=$TAG

	labels stack_name=${EMPTY_LABEL_VALUE}, service_name=${EMPTY_LABEL_VALUE}, slot=${EMPTY_LABEL_VALUE}, container_name=$TAG[1], level=$TAG[2]

	label_keys $container_id

	remove_keys container_id, container_name, source

	drop_single_key on

	line_format key_value

deploy/docker-compose.tmpl.yml

+8 −0
Original line number Diff line number Diff line
@@ -6,6 +6,14 @@ services: 
    environment:

      LOG_LEVEL:

      INPUT_TAG_PREFIX:

      LOG_LEVEL_FIELDS:

      LOG_LEVEL_DELIMITERS:

      LOG_LEVEL_CRITICAL_VALUES:

      LOG_LEVEL_ERROR_VALUES:

      LOG_LEVEL_WARNING_VALUES:

      LOG_LEVEL_INFO_VALUES:

      LOG_LEVEL_DEBUG_VALUES:

      LOG_LEVEL_TRACE_VALUES:

      LOKI_HOST:

      EMPTY_LABEL_VALUE:

    networks: