gitadmin/devops-infra-helm-charts-gcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added repo

Browse Source
This commit is contained in:
Your Name
2026-08-26 03:39:42 +05:30
parent 45c25a95af
commit b8575bb8b9
6889 changed files with 1217125 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-central-pod-alerts.yaml
+38
View File
@@ -0,0 +1,38 @@
groups:
- name: EKS-central-Pod-Alerts
rules:
- alert: Pod-CPU-Crossed-80%
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'central'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} == 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-dataengg-pod-alerts.yaml
+39
View File
@@ -0,0 +1,39 @@
groups:
- name: EKS-dataengg-Pod-Alerts
rules:
- alert: Pod-CPU-Crossed-80%
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'dataengg'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} == 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[3m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-datascience-pod-alerts.yaml
+38
View File
@@ -0,0 +1,38 @@
groups:
- name: EKS-datascience-Pod-Alerts
rules:
- alert: Pod-CPU-Crossed-80%
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'datascience'} > 80
for: 4m
labels:
severity: critical
annotations:
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 80
for: 4m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} == 1
for: 4m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 1
for: 4m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-demand-pod-alerts.yaml
+47
View File
@@ -0,0 +1,47 @@
groups:
- name: EKS-demand-Pod-Alerts
rules:
- alert: Pod-CPU-Crossed-80%
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'demand'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%-New
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80)
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} == 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-infra-pod-alerts.yaml
+20
View File
@@ -0,0 +1,20 @@
groups:
- name: EKS-infra-Pod-Alerts
rules:
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} == 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} > 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
helm-templates/victoria-metrics-alerts-config/configmap/infra/devops/pod/eks-supply-pod-alerts.yaml
+47
View File
@@ -0,0 +1,47 @@
groups:
- name: EKS-supply-Pod-Alerts
rules:
- alert: Pod-CPU-Crossed-80%
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'supply'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Memory-Crossed-80%-New
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80)
for: 1m
labels:
severity: critical
annotations:
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: Pod-Not-in-Ready-State
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} == 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} not in Ready State
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
- alert: Pod-Restarted-Multiple-Times
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 1
for: 3m
labels:
severity: critical
annotations:
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"