added repo
This commit is contained in:
Your Name
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-central-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: central-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dataplatform.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dataplatform.*', persistentvolume=~'pvc-4f013cc3-1692-46c7-9e9c-b638a0d980af|pvc-a8ad12ce-736c-43af-b202-6bbab7d80537|pvc-b0134836-60ab-4867-8f9e-709941f28528'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: demand-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: supply-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
Reference in New Issue
Block a user