46 lines
2.6 KiB
YAML
46 lines
2.6 KiB
YAML
groups:
|
|
- name: EKS-dataplatform-Contour-Pod-Alerts
|
|
rules:
|
|
- alert: dataplatform-Contour-Pod-CPU-Crossed-Threshold
|
|
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 85
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-contour
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: dataplatform-Contour-Pod-Not-in-Ready-State
|
|
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} == 1
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-contour
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
|
|
|
- alert: dataplatform-Contour-Pod-Restarted-Multiple-Times
|
|
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-contour
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|