added repo
This commit is contained in:
Your Name
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-alertmanager-Alerts
|
||||
rules:
|
||||
- alert: alertmanager-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*alertmanager.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*alertmanager.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EC2-grafana-Alerts
|
||||
rules:
|
||||
- alert: grafana-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*grafana.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*grafana.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*grafana.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*grafana.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Down
|
||||
expr: probe_http_status_code{job="blackbox", instance="https://grafana.meesho.com"} != 200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana is DOWN (URL - {{ $labels.instance }})
|
||||
description: "{{ $labels.instance }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: central-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: datascience-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: demand-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: supply-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-prometheus-Alerts
|
||||
rules:
|
||||
- alert: prometheus-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*prometheus.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*prometheus.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*prometheus.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*prometheus.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
groups:
|
||||
- name: EKS-VM-Stack-Node-Alerts
|
||||
rules:
|
||||
- alert: EKS-VM-Stack-Node-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter_eks', mode="idle", name=~'.*vm.*|.*grafana.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: EKS-VM-Stack-Node-Memory-Crossed-80%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]))) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: EKS-VM-Stack-Node-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks", name=~'.*vm.*|.*grafana.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks", name=~'.*vm.*|.*grafana.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vmagent-Alerts
|
||||
rules:
|
||||
- alert: central-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-central-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-central-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmagent-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dataplatform-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dataplatform-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmagent-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-datascience-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-datascience-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmagent-Alerts
|
||||
rules:
|
||||
- alert: demand-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-demand-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-demand-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmagent-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dp-starburst-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dp-starburst-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmagent-Alerts
|
||||
rules:
|
||||
- alert: supply-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-supply-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-supply-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vmalert-Alerts
|
||||
rules:
|
||||
- alert: central-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmalert-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmalert-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmalert-Alerts
|
||||
rules:
|
||||
- alert: demand-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmalert-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmalert-Alerts
|
||||
rules:
|
||||
- alert: supply-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vminsert-Alerts
|
||||
rules:
|
||||
- alert: central-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vminsert-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vminsert-Alerts
|
||||
rules:
|
||||
- alert: datascience-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} > 250
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vminsert-Alerts
|
||||
rules:
|
||||
- alert: demand-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vminsert-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vminsert-Alerts
|
||||
rules:
|
||||
- alert: supply-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} > 180
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-central-vmselect-Alerts
|
||||
rules:
|
||||
- alert: central-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} > 180
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmselect-Alerts
|
||||
rules:
|
||||
- alert: demand-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmselect-Alerts
|
||||
rules:
|
||||
- alert: supply-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-vmsingle-Alerts
|
||||
rules:
|
||||
- alert: vmsingle-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*vmsingle.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*vmsingle.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*vmsingle.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*vmsingle.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-central-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: central-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dataplatform.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dataplatform.*', persistentvolume=~'pvc-4f013cc3-1692-46c7-9e9c-b638a0d980af|pvc-a8ad12ce-736c-43af-b202-6bbab7d80537|pvc-b0134836-60ab-4867-8f9e-709941f28528'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: demand-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: supply-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
Reference in New Issue
Block a user