31 lines
1.6 KiB
YAML
31 lines
1.6 KiB
YAML
groups:
|
|
- name: EKS-datascience-Cilium-Alerts
|
|
rules:
|
|
- alert: datascience-MapOps-Crossed-Threshold
|
|
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*datascience.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 50
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-cilium
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
|
description: "datascience Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
|
|
|
- alert: SubnetOutofIpsForDatascienceCluster
|
|
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*datascience.*"}) by (cluster) < 6000
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-cilium
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "Cluster {{ $labels.cluster }} has ip address less than 6000. Please check"
|
|
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 6000.\n LABELS: {{ $labels }}"
|