added repo
This commit is contained in:
Your Name
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: DBE-Redis-Lab-Alerts
|
||||
rules:
|
||||
- alert: BDB-Cluster-CPU-Usage-By-Shards-Crossed-Threshold
|
||||
expr: ((delta(redis_process_cpu_system_seconds_total{bdb!~"11257076|11648992|11479248|11497326"}[60s]) + delta(redis_process_cpu_user_seconds_total{bdb!~"11257076|11257076|11648992|11479248|11497326"}[60s]))/60*100 or redis_cpu_percent{bdb!~"11257076|11257076|11648992|11479248|11497326"}) >= 70
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - CPU usage by shards crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }} shard: {{ $labels.redis }} role: {{ $labels.role }} node:{{ $labels.node }}, slots:{{ $labels.slots }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - CPU usage by shards crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }} shard: {{ $labels.redis }} role: {{ $labels.role }} node:{{ $labels.node }}, slots:{{ $labels.slots }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: BDB-Cluster-CPU-Usage-Crossed-Threshold
|
||||
expr: 100 - node_cpu_idle * 100 >= 75
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - CPU usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - CPU usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: BDB-Cluster-Memory-Usage-Crossed-Threshold
|
||||
expr: (bdb_used_memory/bdb_memory_limit{bdb!~"11144376|11075753|11132625|11202564|11144272|11261951|11648992|11479248|11081008"}) * 100 >= 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - Memory usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - Memory usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: BDB-Cluster-Avg-Latency-Crossed-Threshold
|
||||
expr: bdb_avg_latency >= 10000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - AVG Latency crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - AVG Latency crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: BDB-Cluster-Ingress-Usage-Crossed-Threshold
|
||||
expr: bdb_ingress_bytes >= 400787200
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - Ingress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - Ingress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: BDB-Cluster-Egress-Usage-Crossed-Threshold
|
||||
expr: bdb_egress_bytes >= 2684354560
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: dbe
|
||||
service: redis
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "BDB Cluster - Egress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}"
|
||||
description: "BDB Cluster - Egress usage crossed Threshold - Cluster ID : {{ $labels.bdb }} and the value is {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-central-Cilium-Alerts
|
||||
rules:
|
||||
- alert: central-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*central.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "central Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "central Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForCentralCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*central.*",subnet_id!~"(subnet-068af921836ca6723|subnet-025917a40be07542c)"}) by (cluster) < 700
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 700. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 700.\n LABELS: {{ $labels }}"
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
groups:
|
||||
- name: EKS-Cilium-Alerts
|
||||
rules:
|
||||
- alert: CiliumClusterMeshFailureAlert
|
||||
expr: sum(cilium_clustermesh_remote_cluster_failures) by (cluster, source_cluster, target_cluster) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium clustermesh failure from {{ $labels.source_cluster }} to {{ $labels.target_cluster }}"
|
||||
description: "Cilium clustermesh failure VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: DroppedEgressPackets
|
||||
expr: sum(rate(cilium_drop_count_total{direction="EGRESS", k8s_app="cilium", reason=~"No mapping for NAT masquerade|FIB lookup failed"}[1m])) by (kubernetes_pod_name, instance, reason, cluster) >= 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
# - alert: SubnetOutofIpsByCilium
|
||||
# expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 300
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: warning
|
||||
# bu: infra
|
||||
# team: devops
|
||||
# service: eks-cilium
|
||||
# env: prd
|
||||
# priority: p0
|
||||
# annotations:
|
||||
# summary: "Subnet {{ $labels.subnet_id }} has ip address less than 300. Please check"
|
||||
# description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}"
|
||||
|
||||
# - alert: SubnetOutofIpsByCilium_100
|
||||
# expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 100
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: critical
|
||||
# bu: infra
|
||||
# team: devops
|
||||
# service: eks-cilium
|
||||
# env: prd
|
||||
# priority: p0
|
||||
# annotations:
|
||||
# summary: "Subnet {{ $labels.subnet_id }} has ip address less than 100. Please check"
|
||||
# description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 100.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-Cilium-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*dataplatform.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "dataplatform Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "dataplatform Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDataplatformCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*dataplatform.*"}) by (cluster) < 1000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 1000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 1000.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Cilium-Alerts
|
||||
rules:
|
||||
- alert: datascience-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*datascience.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDatascienceCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*datascience.*"}) by (cluster) < 6000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 6000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 6000.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-demand-Cilium-Alerts
|
||||
rules:
|
||||
- alert: demand-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*demand.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 30
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "demand Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "demand Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDemandCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*demand.*"}) by (cluster) < 6000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 6000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 6000.\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-Cilium-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*dp-starburst.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "dp-starburst Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "dp-starburst Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
groups:
|
||||
- name: EKS-prod-ops-Cilium-Alerts
|
||||
rules:
|
||||
- alert: prod-ops-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*prod-ops.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: CiliumRemoteClusterCountForProdOpsCluster
|
||||
expr: avg(cilium_clustermesh_remote_clusters{cluster=~"prod-ops-cluster"}) by (cluster, source_cluster, target_cluster) < 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium remote cluster count less than actual remote clusters connected for {{ $labels.cluster }}"
|
||||
description: "Cilium remote cluster count less than actual remote clusters connected. Value = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForOpsCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*ops.*"}) by (cluster) < 500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 500. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-supply-Cilium-Alerts
|
||||
rules:
|
||||
- alert: supply-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*supply.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "supply Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "supply Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForSupplyCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*supply.*"}) by (cluster) < 2500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 2500. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 2500.\n LABELS: {{ $labels }}"
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
groups:
|
||||
- name: EKS-Contour-TG-Alerts
|
||||
rules:
|
||||
- alert: Unhealthy-Host-in-EKSContourTG
|
||||
expr: aws_applicationelb_un_healthy_host_count_maximum{job="aws_applicationelb", target_group =~ ".*cntr.*|.*contour.*"} offset 10m >= 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "Unhealthy Host in EKS Contour TG {{ $labels.target_group }} is greater than 1"
|
||||
description: "Unhealthy Host in EKS Contour TG {{ $labels.target_group }} is :{{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Healthy-Host-in-EKSContourTG-is-VeryLess
|
||||
expr: aws_applicationelb_healthy_host_count_average{job="aws_applicationelb", target_group =~ ".*cntr.*|.*contour.*"} offset 10m < 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "Healthy Host in EKS Contour TG {{ $labels.target_group }} is less than 3"
|
||||
description: "Healthy Host in EKS Contour TG {{ $labels.target_group }} is less than 3, Value - {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-central-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: central-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*central.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*central.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*central.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: datascience-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*datascience.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*datascience.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*datascience.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-demand-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: demand-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*demand.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*demand.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*demand.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-prod-ops-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: prod-ops-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*prod-ops.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prod-ops-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*prod-ops.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prod-ops-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*prod-ops.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-supply-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: supply-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*supply.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*supply.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*supply.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-Karpenter-Alerts
|
||||
rules:
|
||||
- alert: KarpenterProvisionerLimitBreached80Percent
|
||||
expr: karpenter_provisioner_usage_pct > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-karpenter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Karpenter provisioner - 80% limit breached for Provisiosner - {{ $labels.provisioner }} and resource type - {{ $labels.resource_type }} and the value is {{ $value }}"
|
||||
description: "Karpenter provisioner - 80% limit breached for Provisiosner - {{ $labels.provisioner }} and Resource Type - {{ $labels.resource_type }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-Node-Alerts
|
||||
rules:
|
||||
- alert: Kubernetes-Node-Not-Ready-Generic
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='Ready', status='true'} == 0
|
||||
for: 6m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-generic
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready (instance {{ $labels.instance }})
|
||||
description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-NetworkUnavailable
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='NetworkUnavailable', status='true'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-networkunavailable
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to NetworkUnavailable (instance {{ $labels.instance }})
|
||||
description: "Node {{ $labels.node }} has been unready due to NetworkUnavailable for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-MemoryPressure
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="MemoryPressure", status="true"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-memorypressure
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to MemoryPressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has been unready due to MemoryPressure for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-DiskPressure
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="DiskPressure", status="true"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-diskpressure
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to DiskPressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has been unready due to DiskPressure for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-DiskSpace-Crossed-Threshold
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"}) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-diskspace-alert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: EKS-central-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'central'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
groups:
|
||||
- name: EKS-dataengg-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'dataengg'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[3m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'datascience'} > 80
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 80
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} == 1
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 1
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
groups:
|
||||
- name: EKS-demand-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'demand'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%-New
|
||||
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
groups:
|
||||
- name: EKS-infra-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
groups:
|
||||
- name: EKS-supply-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'supply'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%-New
|
||||
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-alertmanager-Alerts
|
||||
rules:
|
||||
- alert: alertmanager-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*alertmanager.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*alertmanager.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*alertmanager.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: alertmanager-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*alertmanager.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: alertmanager
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: alertmanager Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EC2-grafana-Alerts
|
||||
rules:
|
||||
- alert: grafana-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*grafana.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*grafana.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*grafana.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*grafana.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: grafana-Down
|
||||
expr: probe_http_status_code{job="blackbox", instance="https://grafana.meesho.com"} != 200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: grafana
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: grafana is DOWN (URL - {{ $labels.instance }})
|
||||
description: "{{ $labels.instance }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: central-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: datascience-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: demand-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-kube-state-metrics-Alerts
|
||||
rules:
|
||||
- alert: supply-kube-state-metrics-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'kube-system'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'kube-system'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'kube-system'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'kube-system'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-kube-state-metrics-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'kube-system', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'kube-system', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*kube-state-metrics.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: kube-state-metrics
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-prometheus-Alerts
|
||||
rules:
|
||||
- alert: prometheus-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*prometheus.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*prometheus.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*prometheus.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*prometheus.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prometheus-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*prometheus.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: prometheus
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: prometheus Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
groups:
|
||||
- name: EKS-VM-Stack-Node-Alerts
|
||||
rules:
|
||||
- alert: EKS-VM-Stack-Node-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter_eks', mode="idle", name=~'.*vm.*|.*grafana.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: EKS-VM-Stack-Node-Memory-Crossed-80%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter_eks', name=~'.*vm.*|.*grafana.*'}[1m]))) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: EKS-VM-Stack-Node-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks", name=~'.*vm.*|.*grafana.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks", name=~'.*vm.*|.*grafana.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vm-stack-node
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vmagent-Alerts
|
||||
rules:
|
||||
- alert: central-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-central-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-central-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmagent-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dataplatform-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dataplatform-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmagent-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-datascience-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-datascience-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmagent-Alerts
|
||||
rules:
|
||||
- alert: demand-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} > 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-demand-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-demand-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmagent-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dp-starburst-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-dp-starburst-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmagent-Alerts
|
||||
rules:
|
||||
- alert: supply-vmagent-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmagent-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-p-supply-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-p-supply-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmagent.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmagent
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vmalert-Alerts
|
||||
rules:
|
||||
- alert: central-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmalert-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmalert-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmalert-Alerts
|
||||
rules:
|
||||
- alert: demand-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmalert-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmalert-Alerts
|
||||
rules:
|
||||
- alert: supply-vmalert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmalert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmalert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmalert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-central-vminsert-Alerts
|
||||
rules:
|
||||
- alert: central-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vminsert-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vminsert-Alerts
|
||||
rules:
|
||||
- alert: datascience-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} > 250
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-demand-vminsert-Alerts
|
||||
rules:
|
||||
- alert: demand-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vminsert-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-supply-vminsert-Alerts
|
||||
rules:
|
||||
- alert: supply-vminsert-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} > 180
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vminsert-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vminsert.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vminsert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-central-vmselect-Alerts
|
||||
rules:
|
||||
- alert: central-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} > 180
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmselect-Alerts
|
||||
rules:
|
||||
- alert: demand-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmselect-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmselect-Alerts
|
||||
rules:
|
||||
- alert: supply-vmselect-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-Deployment-Replica-MisMatch
|
||||
expr: (kube_deployment_spec_replicas{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'} != kube_deployment_status_replicas_ready{namespace=~'victoriametrics', job=~'kube-state-metrics-.*'}) * on(deployment) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_deployment_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Deployment - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmselect-HPA-Max-Count-Reached
|
||||
expr: (kube_horizontalpodautoscaler_status_desired_replicas{namespace=~"victoriametrics"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace=~"victoriametrics"}) * on(horizontalpodautoscaler) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_horizontalpodautoscaler_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmselect.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmselect
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS HPA - {{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its Max Replica Count {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
groups:
|
||||
- name: EC2-vmsingle-Alerts
|
||||
rules:
|
||||
- alert: vmsingle-CPU-Crossed-80%
|
||||
expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*vmsingle.*'}[1m])) by (name, private_ip) * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance out of CPU (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-Memory-Crossed-90%
|
||||
expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*vmsingle.*'}[1m]))) > 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance out of Memory (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-DiskSpace-Crossed-80%
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*vmsingle.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter", type=~'.*vmsingle.*'}) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: vmsingle-Instance-Down
|
||||
expr: up{job="node_exporter", type=~'.*vmsingle.*'} == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmsingle
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: vmsingle Instance is DOWN (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-central-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: central-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*central.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*central.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*central.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dataplatform.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dataplatform.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dataplatform.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dataplatform.*', persistentvolume=~'pvc-4f013cc3-1692-46c7-9e9c-b638a0d980af|pvc-a8ad12ce-736c-43af-b202-6bbab7d80537|pvc-b0134836-60ab-4867-8f9e-709941f28528'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-datascience-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: datascience-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*datascience.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*datascience.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*datascience.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-demand-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: demand-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*demand.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*demand.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*demand.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*dp-starburst.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*dp-starburst.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*dp-starburst.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
groups:
|
||||
- name: EKS-supply-vmstorage-Alerts
|
||||
rules:
|
||||
- alert: supply-vmstorage-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", namespace=~'victoriametrics'}[1m])) / sum by (pod) (kube_pod_container_resource_requests{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Memory-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="", namespace=~'victoriametrics'}) / sum by (pod) (kube_pod_container_resource_requests{resource="memory", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} > 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', job=~"kube-state-metrics-.*", namespace=~'victoriametrics'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", namespace=~'victoriametrics'}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'} >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-STS-Replica-MisMatch
|
||||
expr: (kube_statefulset_replicas{job=~'kube-state-metrics.*', namespace=~"victoriametrics"} != kube_statefulset_status_replicas_ready{job=~'kube-state-metrics.*', namespace=~"victoriametrics"}) * on(statefulset) group_left(label_bu, label_team, label_service, label_env, label_priority, label_type, namespace, cluster) kube_statefulset_labels{label_env=~'prod|prd', cluster=~'.*supply.*', label_type=~'.*vmstorage.*'}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS STS - {{ $labels.deployment }} Replicas Mis-Match
|
||||
description: "STS {{ $labels.namespace }}/{{ $labels.statefulset }} has Replica Mis-Match {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-Storage-Crossed-80%
|
||||
expr: sum by (persistentvolumeclaim) ((kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} - kubelet_volume_stats_available_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'}) / kubelet_volume_stats_capacity_bytes{namespace=~"victoriametrics", cluster=~'.*supply.*'} * 100) > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} usage is over 80%
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} usage is {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PVC-not-Bound
|
||||
expr: kube_persistentvolumeclaim_status_phase{namespace=~'victoriametrics', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PVC - {{ $labels.persistentvolumeclaim }} is not Bound
|
||||
description: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-vmstorage-PV-not-Bound
|
||||
expr: kube_persistentvolume_status_phase{job=~'kube-state-metrics-.*', phase!~'Bound', cluster=~'.*supply.*'} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: sre
|
||||
service: vmstorage
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS PV - {{ $labels.persistentvolume }} is not Bound
|
||||
description: "PV {{ $labels.namespace }}/{{ $labels.persistentvolume }} is not Bound {{ $value }} for last 1 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
Reference in New Issue
Block a user