93 lines
4.5 KiB
YAML
93 lines
4.5 KiB
YAML
groups:
|
|
- name: EKS-Common-Alerts
|
|
rules:
|
|
- alert: KubernetesPersistentvolumeclaimPending
|
|
expr: kube_persistentvolumeclaim_status_phase{phase="Pending",namespace!~"int-.*"} == 1
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})
|
|
description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesVolumeOutOfDiskSpace
|
|
expr: kubelet_volume_stats_available_bytes{namespace!~"int-.*"} / kubelet_volume_stats_capacity_bytes{namespace!~"int-.*"} * 100 < 30
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
|
|
description: "Volume is almost full (< 30% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesVolumeFullInFourDays
|
|
expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
|
|
description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesPersistentvolumeError
|
|
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job=~"kube-state-metrics", namespace!~"int-.*"} > 0
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
|
|
description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesStatefulsetDown
|
|
expr: (kube_statefulset_status_replicas_ready{statefulset!="vault",namespace!~"int-.*"} / kube_statefulset_status_replicas_current{statefulset!="vault",namespace!~"int-.*"}) != 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
|
|
description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesStatefulsetDownVault
|
|
expr: (kube_statefulset_status_replicas_ready{statefulset=~"vault",namespace!~"int-.*"} / kube_statefulset_replicas{statefulset=~"vault",namespace!~"int-.*"}) != 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
|
|
description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
# - alert: KubernetesHpaMetricAvailability
|
|
# expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive", namespace!~"argocd|grafana" } == 1
|
|
# for: 0m
|
|
# labels:
|
|
# severity: warning
|
|
# annotations:
|
|
# summary: Kubernetes HPA metric availability (instance {{ $labels.instance }})
|
|
# description: "HPA is not able to collect metrics\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesHpaScaleCapability
|
|
expr: kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"int-.*"} >= kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"int-.*"}
|
|
for: 2m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: Kubernetes HPA scale capability (instance {{ $labels.instance }})
|
|
description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesDeploymentReplicasMismatch
|
|
expr: kube_deployment_spec_replicas{namespace!~"int-.*"} != kube_deployment_status_replicas_available{namespace!~"int-.*"}
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})
|
|
description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: KubernetesStatefulsetReplicasMismatch
|
|
expr: kube_statefulset_status_replicas_ready{namespace!~"int-.*"} != kube_statefulset_status_replicas{namespace!~"int-.*"}
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
|
|
description: "A StatefulSet does not match the expected number of replicas.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|