added repo
This commit is contained in:
Your Name
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-central-Cilium-Alerts
|
||||
rules:
|
||||
- alert: central-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*central.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "central Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "central Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForCentralCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*central.*",subnet_id!~"(subnet-068af921836ca6723|subnet-025917a40be07542c)"}) by (cluster) < 700
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 700. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 700.\n LABELS: {{ $labels }}"
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
groups:
|
||||
- name: EKS-Cilium-Alerts
|
||||
rules:
|
||||
- alert: CiliumClusterMeshFailureAlert
|
||||
expr: sum(cilium_clustermesh_remote_cluster_failures) by (cluster, source_cluster, target_cluster) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium clustermesh failure from {{ $labels.source_cluster }} to {{ $labels.target_cluster }}"
|
||||
description: "Cilium clustermesh failure VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: DroppedEgressPackets
|
||||
expr: sum(rate(cilium_drop_count_total{direction="EGRESS", k8s_app="cilium", reason=~"No mapping for NAT masquerade|FIB lookup failed"}[1m])) by (kubernetes_pod_name, instance, reason, cluster) >= 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
# - alert: SubnetOutofIpsByCilium
|
||||
# expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 300
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: warning
|
||||
# bu: infra
|
||||
# team: devops
|
||||
# service: eks-cilium
|
||||
# env: prd
|
||||
# priority: p0
|
||||
# annotations:
|
||||
# summary: "Subnet {{ $labels.subnet_id }} has ip address less than 300. Please check"
|
||||
# description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}"
|
||||
|
||||
# - alert: SubnetOutofIpsByCilium_100
|
||||
# expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 100
|
||||
# for: 1m
|
||||
# labels:
|
||||
# severity: critical
|
||||
# bu: infra
|
||||
# team: devops
|
||||
# service: eks-cilium
|
||||
# env: prd
|
||||
# priority: p0
|
||||
# annotations:
|
||||
# summary: "Subnet {{ $labels.subnet_id }} has ip address less than 100. Please check"
|
||||
# description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 100.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-Cilium-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*dataplatform.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "dataplatform Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "dataplatform Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDataplatformCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*dataplatform.*"}) by (cluster) < 1000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 1000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 1000.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Cilium-Alerts
|
||||
rules:
|
||||
- alert: datascience-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*datascience.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDatascienceCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*datascience.*"}) by (cluster) < 6000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 6000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 6000.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-demand-Cilium-Alerts
|
||||
rules:
|
||||
- alert: demand-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*demand.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 30
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "demand Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "demand Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForDemandCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*demand.*"}) by (cluster) < 6000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 6000. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 6000.\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-Cilium-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*dp-starburst.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "dp-starburst Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "dp-starburst Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
groups:
|
||||
- name: EKS-prod-ops-Cilium-Alerts
|
||||
rules:
|
||||
- alert: prod-ops-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*prod-ops.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: CiliumRemoteClusterCountForProdOpsCluster
|
||||
expr: avg(cilium_clustermesh_remote_clusters{cluster=~"prod-ops-cluster"}) by (cluster, source_cluster, target_cluster) < 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cilium remote cluster count less than actual remote clusters connected for {{ $labels.cluster }}"
|
||||
description: "Cilium remote cluster count less than actual remote clusters connected. Value = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForOpsCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*ops.*"}) by (cluster) < 500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 500. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}"
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
groups:
|
||||
- name: EKS-supply-Cilium-Alerts
|
||||
rules:
|
||||
- alert: supply-MapOps-Crossed-Threshold
|
||||
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*supply.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "supply Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
||||
description: "supply Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
|
||||
- alert: SubnetOutofIpsForSupplyCluster
|
||||
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*supply.*"}) by (cluster) < 2500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-cilium
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Cluster {{ $labels.cluster }} has ip address less than 2500. Please check"
|
||||
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 2500.\n LABELS: {{ $labels }}"
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
groups:
|
||||
- name: EKS-Contour-TG-Alerts
|
||||
rules:
|
||||
- alert: Unhealthy-Host-in-EKSContourTG
|
||||
expr: aws_applicationelb_un_healthy_host_count_maximum{job="aws_applicationelb", target_group =~ ".*cntr.*|.*contour.*"} offset 10m >= 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "Unhealthy Host in EKS Contour TG {{ $labels.target_group }} is greater than 1"
|
||||
description: "Unhealthy Host in EKS Contour TG {{ $labels.target_group }} is :{{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Healthy-Host-in-EKSContourTG-is-VeryLess
|
||||
expr: aws_applicationelb_healthy_host_count_average{job="aws_applicationelb", target_group =~ ".*cntr.*|.*contour.*"} offset 10m < 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "Healthy Host in EKS Contour TG {{ $labels.target_group }} is less than 3"
|
||||
description: "Healthy Host in EKS Contour TG {{ $labels.target_group }} is less than 3, Value - {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-central-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: central-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*central.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*central.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: central-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*central.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-dataplatform-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: dataplatform-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dataplatform-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dataplatform.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: datascience-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*datascience.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*datascience.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: datascience-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*datascience.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-demand-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: demand-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*demand.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*demand.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: demand-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*demand.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-dp-starburst-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: dp-starburst-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: dp-starburst-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*dp-starburst.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-prod-ops-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: prod-ops-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*prod-ops.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prod-ops-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*prod-ops.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: prod-ops-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*prod-ops.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
groups:
|
||||
- name: EKS-supply-Contour-Pod-Alerts
|
||||
rules:
|
||||
- alert: supply-Contour-Pod-CPU-Crossed-Threshold
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~"prod|prd", cluster=~".*supply.*"} > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-Contour-Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", phase!="Pending", job=~"kube-state-metrics-p-.*", pod=~".*contour.*", pod!~".*default-backend.*"} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*supply.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.reason }} state for last 2 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: supply-Contour-Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*", pod=~".*contour.*", pod!~".*default-backend.*"}[1m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~"prod|prd", cluster=~".*supply.*"} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-contour
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 2 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-Karpenter-Alerts
|
||||
rules:
|
||||
- alert: KarpenterProvisionerLimitBreached80Percent
|
||||
expr: karpenter_provisioner_usage_pct > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-karpenter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Karpenter provisioner - 80% limit breached for Provisiosner - {{ $labels.provisioner }} and resource type - {{ $labels.resource_type }} and the value is {{ $value }}"
|
||||
description: "Karpenter provisioner - 80% limit breached for Provisiosner - {{ $labels.provisioner }} and Resource Type - {{ $labels.resource_type }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-Node-Alerts
|
||||
rules:
|
||||
- alert: Kubernetes-Node-Not-Ready-Generic
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='Ready', status='true'} == 0
|
||||
for: 6m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-generic
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready (instance {{ $labels.instance }})
|
||||
description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-NetworkUnavailable
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition='NetworkUnavailable', status='true'} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-networkunavailable
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to NetworkUnavailable (instance {{ $labels.instance }})
|
||||
description: "Node {{ $labels.node }} has been unready due to NetworkUnavailable for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-MemoryPressure
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="MemoryPressure", status="true"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-memorypressure
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to MemoryPressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has been unready due to MemoryPressure for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-Not-Ready-DiskPressure
|
||||
expr: kube_node_status_condition{job=~'kube-state-metrics-p-.*', condition="DiskPressure", status="true"} == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-not-ready-diskpressure
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: URGENT - Kubernetes Node not ready due to DiskPressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has been unready due to DiskPressure for a long time\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Kubernetes-Node-DiskSpace-Crossed-Threshold
|
||||
expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter_eks"}) > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: eks-node-diskspace-alert
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: Kubernetes Node running out of Disk Space (instance {{ $labels.private_ip }})
|
||||
description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: EKS-central-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'central'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'central'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
groups:
|
||||
- name: EKS-dataengg-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'dataengg'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[3m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'dataengg'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
groups:
|
||||
- name: EKS-datascience-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'datascience'} > 80
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!=""}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 80
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} == 1
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'datascience'} > 1
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
groups:
|
||||
- name: EKS-demand-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'demand'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%-New
|
||||
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-merch-merchandising-admin.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 80)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'demand'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
groups:
|
||||
- name: EKS-infra-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'infra'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
groups:
|
||||
- name: EKS-supply-Pod-Alerts
|
||||
rules:
|
||||
- alert: Pod-CPU-Crossed-80%
|
||||
expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!=""}[1m])) / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd', label_bu=~'supply'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%
|
||||
expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",pod!~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Memory-Crossed-80%-New
|
||||
expr: (100 * sum by (pod) (container_memory_rss{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80) and (100 * sum by (pod) (container_memory_working_set_bytes{image!="", pod=~"prd-pdf.*|prd-invoice.*"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 80)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Not-in-Ready-State
|
||||
expr: kube_pod_status_phase{phase!='Running', phase!='Succeeded', phase!='Pending', job=~'kube-state-metrics-p-.*'} * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} not in Ready State
|
||||
description: "Please connect with DevOps-POC of this service for this Issue. Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for last 3 mins.\n STATE = NotReady\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: Pod-Restarted-Multiple-Times
|
||||
expr: sum by (pod,namespace) (increase(kube_pod_container_status_restarts_total{job=~"kube-state-metrics-.*"}[2m])) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{job=~"kube-state-metrics-.*", label_env=~'prod|prd',label_bu=~'supply'} > 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: EKS Pod - {{ $labels.pod }} Restarted Multiple Times
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has Restarted for more than {{ $value }} times in last 3 mins.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
Reference in New Issue
Block a user