groups:
- name: EKS-dataplatform-vmselect-Alerts
  rules:
  - alert: dataplatform-vmselect-Pod-CPU-Crossed-Threshold
    expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m]))  / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: dataplatform-vmselect-Pod-Memory-Crossed-Threshold
    expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: dataplatform-vmselect-Pod-Not-in-Ready-State
    expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} == 1
    for: 3m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: EKS Pod - {{ $labels.pod }} not in Ready State
      description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n  STATE = NotReady\n  LABELS = {{ $labels }}"

  - alert: dataplatform-vmselect-Pod-Restarted-Multiple-Times
    expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} >= 1
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: dataplatform-vmselect-Deployment-Replica-MisMatch
    expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
      description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: dataplatform-vmselect-HPA-Max-Count-Reached
    expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: vmselect
      env: prd
      priority: p0
    annotations:
      summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
      description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"