groups:
- name: EKS-Airflow KubernetesNodeReady
  rules:
  - alert: KubernetesNodeReady
    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="Ready",status="true"} == 0
    for: 3m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      service: airflow-new
      env: prd
      priority: p0
    annotations:
      summary: Kubernetes Node ready (instance {{ $labels.instance }})
      description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"

  - alert: EKS-Airflow KubernetesMemoryPressure
    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="MemoryPressure",status="true"} == 1
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: Kubernetes memory pressure (instance {{ $labels.instance }})
      description: "{{ $labels.node }} has MemoryPressure condition\n  LABELS = {{ $labels }}"

  - alert: EKS-Airflow KubernetesDiskPressure
    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="DiskPressure",status="true"} == 1
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: Kubernetes disk pressure (instance {{ $labels.instance }})
      description: "{{ $labels.node }} has DiskPressure condition\n  LABELS = {{ $labels }}"

  - alert: Pod-CPU-Crossed-80%
    expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!="", namespace="prd-di-airflow-new"}[1m]))  / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: Pod-Memory-Crossed-80%
    expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",namespace="prd-di-airflow-new"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: Pod-in-PodInitializing-State
    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='PodInitializing',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: EKS Pod - {{ $labels.pod }} is in PodInitializing State
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in PodInitializing state for last 2 mins.\n  STATE = PodInitializing\n  LABELS = {{ $labels }}"

  - alert: Pod-in-CrashLoopBackOff-State
    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CrashLoopBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: EKS Pod - {{ $labels.pod }} is in CrashLoopBackOff State
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff state for last 2 mins.\n  STATE = CrashLoopBackOff\n  LABELS = {{ $labels }}"

  - alert: Pod-in-ImagePullBackOff-State
    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='ImagePullBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
    for: 1m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: EKS Pod - {{ $labels.pod }} is in ImagePullBackOff State
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in ImagePullBackOff state for last 2 mins.\n  STATE = ImagePullBackOff\n  LABELS = {{ $labels }}"

  - alert: Pod-in-CreateContainerConfigError-State
    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CreateContainerConfigError',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
    for: 2m
    labels:
      severity: critical
      bu: dataengg
      team: data-intelligence
      env: prd
      priority: p0
      service: airflow-new
    annotations:
      summary: EKS Pod - {{ $labels.pod }} is in CreateContainerConfigError State
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CreateContainerConfigError state for last 2 mins.\n  STATE = CreateContainerConfigError\n  LABELS = {{ $labels }}"