60 lines
2.6 KiB
YAML
60 lines
2.6 KiB
YAML
groups:
|
|
- name: EC2-alertmanager-Alerts
|
|
rules:
|
|
- alert: alertmanager-CPU-Crossed-80%
|
|
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*alertmanager.*'}[1m])) by (name, private_ip) * 100) > 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: sre
|
|
service: alertmanager
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: alertmanager Instance out of CPU (instance {{ $labels.private_ip }})
|
|
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: alertmanager-Memory-Crossed-90%
|
|
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]))) > 90
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: sre
|
|
service: alertmanager
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: alertmanager Instance out of Memory (instance {{ $labels.private_ip }})
|
|
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: alertmanager-DiskSpace-Crossed-80%
|
|
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'}) > 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: sre
|
|
service: alertmanager
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: alertmanager Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
|
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: alertmanager-Instance-Down
|
|
expr: up{job="node_exporter", type=~'.*alertmanager.*'} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: sre
|
|
service: alertmanager
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: alertmanager Instance is DOWN (instance {{ $labels.private_ip }})
|
|
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|