87 lines
3.8 KiB
YAML
87 lines
3.8 KiB
YAML
groups:
|
|
- name: DBE-Redis-Lab-Alerts
|
|
rules:
|
|
- alert: BDB-Cluster-CPU-Usage-By-Shards-Crossed-Threshold
|
|
expr: ((delta(redis_process_cpu_system_seconds_total{bdb!~"11257076|11648992|11479248|11497326"}[60s]) + delta(redis_process_cpu_user_seconds_total{bdb!~"11257076|11257076|11648992|11479248|11497326"}[60s]))/60*100 or redis_cpu_percent{bdb!~"11257076|11257076|11648992|11479248|11497326"}) >= 70
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - CPU usage by shards crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }} shard: {{ $labels.redis }} role: {{ $labels.role }} node:{{ $labels.node }}, slots:{{ $labels.slots }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - CPU usage by shards crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }} shard: {{ $labels.redis }} role: {{ $labels.role }} node:{{ $labels.node }}, slots:{{ $labels.slots }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: BDB-Cluster-CPU-Usage-Crossed-Threshold
|
|
expr: 100 - node_cpu_idle * 100 >= 75
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - CPU usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - CPU usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: BDB-Cluster-Memory-Usage-Crossed-Threshold
|
|
expr: (bdb_used_memory/bdb_memory_limit{bdb!~"11144376|11075753|11132625|11202564|11144272|11261951|11648992|11479248|11081008"}) * 100 >= 90
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - Memory usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - Memory usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: BDB-Cluster-Avg-Latency-Crossed-Threshold
|
|
expr: bdb_avg_latency >= 10000
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - AVG Latency crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - AVG Latency crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: BDB-Cluster-Ingress-Usage-Crossed-Threshold
|
|
expr: bdb_ingress_bytes >= 400787200
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - Ingress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - Ingress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: BDB-Cluster-Egress-Usage-Crossed-Threshold
|
|
expr: bdb_egress_bytes >= 2684354560
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: dbe
|
|
service: redis
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "BDB Cluster - Egress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
|
description: "BDB Cluster - Egress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|