45 lines
2.1 KiB
YAML
45 lines
2.1 KiB
YAML
groups:
|
|
- name: EKS-prod-ops-Cilium-Alerts
|
|
rules:
|
|
- alert: prod-ops-MapOps-Crossed-Threshold
|
|
expr: topk(5, avg(rate(cilium_bpf_map_ops_total{k8s_app="cilium", map_name=~"snat_v4_external", operation=~"delete", cluster=~".*prod-ops.*"}[5m])) by (kubernetes_pod_name, instance, map_name, operation, cluster)) <= 40
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-cilium
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}"
|
|
description: "prod-ops Cilium Map Ops has crossed 40 Ops per Second for Map Name - snat_v4_external, operation - delete, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}"
|
|
|
|
- alert: CiliumRemoteClusterCountForProdOpsCluster
|
|
expr: avg(cilium_clustermesh_remote_clusters{cluster=~"prod-ops-cluster"}) by (cluster, source_cluster, target_cluster) < 5
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-cilium
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "Cilium remote cluster count less than actual remote clusters connected for {{ $labels.cluster }}"
|
|
description: "Cilium remote cluster count less than actual remote clusters connected. Value = {{ $value }}\n LABELS: {{ $labels }}"
|
|
|
|
- alert: SubnetOutofIpsForOpsCluster
|
|
expr: sum(cilium_operator_ipam_available_ips_per_subnet{cluster=~".*ops.*"}) by (cluster) < 500
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
bu: infra
|
|
team: devops
|
|
service: eks-cilium
|
|
env: prd
|
|
priority: p0
|
|
annotations:
|
|
summary: "Cluster {{ $labels.cluster }} has ip address less than 500. Please check"
|
|
description: "Cluster {{ $labels.cluster }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}"
|