129 lines
6.3 KiB
YAML
129 lines
6.3 KiB
YAML
groups:
|
|
- name: EKS-Airflow KubernetesNodeReady
|
|
rules:
|
|
- alert: KubernetesNodeReady
|
|
expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="Ready",status="true"} == 0
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
service: airflow-new
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: Kubernetes Node ready (instance {{ $labels.instance }})
|
|
description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"
|
|
|
|
- alert: EKS-Airflow KubernetesMemoryPressure
|
|
expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="MemoryPressure",status="true"} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: Kubernetes memory pressure (instance {{ $labels.instance }})
|
|
description: "{{ $labels.node }} has MemoryPressure condition\n LABELS = {{ $labels }}"
|
|
|
|
- alert: EKS-Airflow KubernetesDiskPressure
|
|
expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="DiskPressure",status="true"} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: Kubernetes disk pressure (instance {{ $labels.instance }})
|
|
description: "{{ $labels.node }} has DiskPressure condition\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-CPU-Crossed-80%
|
|
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!="", namespace="prd-di-airflow-new"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-Memory-Crossed-80%
|
|
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",namespace="prd-di-airflow-new"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-in-PodInitializing-State
|
|
expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='PodInitializing',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} is in PodInitializing State
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in PodInitializing state for last 2 mins.\n STATE = PodInitializing\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-in-CrashLoopBackOff-State
|
|
expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CrashLoopBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} is in CrashLoopBackOff State
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff state for last 2 mins.\n STATE = CrashLoopBackOff\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-in-ImagePullBackOff-State
|
|
expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='ImagePullBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} is in ImagePullBackOff State
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in ImagePullBackOff state for last 2 mins.\n STATE = ImagePullBackOff\n LABELS = {{ $labels }}"
|
|
|
|
- alert: Pod-in-CreateContainerConfigError-State
|
|
expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CreateContainerConfigError',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
bu: dataengg
|
|
team: data-intelligence
|
|
env: prd
|
|
priority: p0
|
|
service: airflow-new
|
|
annotations:
|
|
summary: EKS Pod - {{ $labels.pod }} is in CreateContainerConfigError State
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CreateContainerConfigError state for last 2 mins.\n STATE = CreateContainerConfigError\n LABELS = {{ $labels }}"
|