73 lines
2.9 KiB
YAML
73 lines
2.9 KiB
YAML
groups:
|
|
- name: EKS-Node-Alerts
|
|
rules:
|
|
- alert: Kubernetes-Node-Not-Ready-Generic
|
|
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='Ready', status='true'} == 0
|
|
for: 6m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-node-not-ready-generic
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: URGENT - Kubernetes Node not ready (instance {{ $labels.instance }})
|
|
description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Kubernetes-Node-Not-Ready-NetworkUnavailable
|
|
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='NetworkUnavailable', status='true'} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-node-not-ready-networkunavailable
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: URGENT - Kubernetes Node not ready due to NetworkUnavailable (instance {{ $labels.instance }})
|
|
description: "Node {{ $labels.node }} has been unready due to NetworkUnavailable for a long time\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Kubernetes-Node-Not-Ready-MemoryPressure
|
|
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="MemoryPressure", status="true"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-node-not-ready-memorypressure
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: URGENT - Kubernetes Node not ready due to MemoryPressure (instance {{ $labels.instance }})
|
|
description: "{{ $labels.node }} has been unready due to MemoryPressure for a long time\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Kubernetes-Node-Not-Ready-DiskPressure
|
|
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="DiskPressure", status="true"} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-node-not-ready-diskpressure
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: URGENT - Kubernetes Node not ready due to DiskPressure (instance {{ $labels.instance }})
|
|
description: "{{ $labels.node }} has been unready due to DiskPressure for a long time\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Kubernetes-Node-DiskSpace-Crossed-Threshold
|
|
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"}) > 85
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-node-diskspace-alert
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: Kubernetes Node running out of Disk Space (instance {{ $labels.private_ip }})
|
|
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|