Commit c7976ebd authored by Pedro Eduardo Trujillo's avatar Pedro Eduardo Trujillo
Browse files

Actualiza alertas de disco, soporta múltiples 

Abandona el seteo de rutas fijas dependientes del entorno (sólo mantiene
la de por defecto de instalación de Docker para ignorarla) y soporta la
alerta sobre muchos volúmenes por nodo.
parent 55f26ae3

deploy/rules/swarm_node.rules.dev.yml

+10 −30
Original line number Diff line number Diff line
@@ -22,43 +22,23 @@ groups: 
      summary: Memory alert for Swarm node '{{$labels.node_name}}'



  - alert: node_disk_usage

    expr: ((node_filesystem_size_bytes{mountpoint="/rootfs"} - node_filesystem_free_bytes{mountpoint="/rootfs"})

      * 100 / node_filesystem_size_bytes{mountpoint="/rootfs"}) * ON(instance) GROUP_LEFT(node_name)

      node_meta > 85

    expr: ((node_filesystem_size_bytes{mountpoint!~"/var/lib/docker/.*"} -

      node_filesystem_free_bytes{mountpoint!~"/var/lib/docker/.*"}) * 100 /

      node_filesystem_size_bytes{mountpoint!~"/var/lib/docker/.*"}) *

      ON(instance) GROUP_LEFT(node_name) node_meta > 85

    for: 5m

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} disk usage is at {{$value | printf "%.1f"}}%

      summary: Disk alert for Swarm node '{{$labels.node_name}}'

      description: Swarm node {{$labels.node_name}} disk {{$labels.mountpoint}} usage is at {{$value | printf "%.1f"}}%

      summary: Disk '{{$labels.mountpoint}}' alert for Swarm node '{{$labels.node_name}}'



  - alert: node_disk_fill_rate_6h

    expr: predict_linear(node_filesystem_free_bytes{mountpoint="/rootfs"}[1h], 6 * 3600) * ON(instance)

      GROUP_LEFT(node_name) node_meta < 0

    expr: predict_linear(node_filesystem_free_bytes{mountpoint!~"/var/lib/docker/.*"}[1h], 6 * 3600) *

      ON(instance) GROUP_LEFT(node_name) node_meta < 0

    for: 1h

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} disk is going to fill up in 6h

      summary: Disk fill alert for Swarm node '{{$labels.node_name}}'



  - alert: SwarmNodeDockerDiskUsage

    expr: ((node_filesystem_size_bytes{mountpoint="/rootfs/mnt/data"} - node_filesystem_free_bytes{mountpoint="/rootfs/mnt/data"})

      * 100 / node_filesystem_size_bytes{mountpoint="/rootfs/mnt/data"}) * ON(instance) GROUP_LEFT(node_name)

      node_meta > 85

    for: 5m

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} Docker mountpoint disk usage is at {{$value | printf "%.1f"}}%

      summary: Docker mountpoint disk alert for Swarm node '{{$labels.node_name}}'



  - alert: SwarmNodeDockerDiskFillRate

    expr: predict_linear(node_filesystem_free_bytes{mountpoint="/rootfs/mnt/data"}[1h], 6 * 3600) * ON(instance)

      GROUP_LEFT(node_name) node_meta < 0

    for: 1h

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} Docker mountpoint disk is going to fill up in 6h

      summary: Docker mountpoint disk fill alert for Swarm node '{{$labels.node_name}}'
 
      description: Swarm node {{$labels.node_name}} disk {{$labels.mountpoint}} is going to fill up in 6h

      summary: Disk '{{$labels.mountpoint}}' fill alert for Swarm node '{{$labels.node_name}}'

deploy/rules/swarm_node.rules.prod.yml

+10 −9
Original line number Diff line number Diff line
@@ -22,25 +22,26 @@ groups: 
      summary: Memory alert for Swarm node '{{$labels.node_name}}'



  - alert: node_disk_usage

    expr: ((node_filesystem_size_bytes{mountpoint="/rootfs"} - node_filesystem_free_bytes{mountpoint="/rootfs"})

      * 100 / node_filesystem_size_bytes{mountpoint="/rootfs"}) * ON(instance) GROUP_LEFT(node_name)

      node_meta > 85

    expr: ((node_filesystem_size_bytes{mountpoint!~"/var/lib/docker/.*"} -

      node_filesystem_free_bytes{mountpoint!~"/var/lib/docker/.*"}) * 100 /

      node_filesystem_size_bytes{mountpoint!~"/var/lib/docker/.*"}) *

      ON(instance) GROUP_LEFT(node_name) node_meta > 85

    for: 5m

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} disk usage is at {{$value | printf "%.1f"}}%

      summary: Disk alert for Swarm node '{{$labels.node_name}}'

      description: Swarm node {{$labels.node_name}} disk {{$labels.mountpoint}} usage is at {{$value | printf "%.1f"}}%

      summary: Disk '{{$labels.mountpoint}}' alert for Swarm node '{{$labels.node_name}}'



  - alert: node_disk_fill_rate_6h

    expr: predict_linear(node_filesystem_free_bytes{mountpoint="/rootfs"}[1h], 6 * 3600) * ON(instance)

      GROUP_LEFT(node_name) node_meta < 0

    expr: predict_linear(node_filesystem_free_bytes{mountpoint!~"/var/lib/docker/.*"}[1h], 6 * 3600) *

      ON(instance) GROUP_LEFT(node_name) node_meta < 0

    for: 1h

    labels:

      severity: critical

    annotations:

      description: Swarm node {{$labels.node_name}} disk is going to fill up in 6h

      summary: Disk fill alert for Swarm node '{{$labels.node_name}}'

      description: Swarm node {{$labels.node_name}} disk {{$labels.mountpoint}} is going to fill up in 6h

      summary: Disk '{{$labels.mountpoint}}' fill alert for Swarm node '{{$labels.node_name}}'



  - alert: SwarmNodeEbsDiskUsage

    expr: ((node_filesystem_size_bytes{mountpoint=~".*/ebs/.*"} - node_filesystem_free_bytes{mountpoint=~".*/ebs/.*"})