groups: - name: EKS-Cilium-Alerts rules: - alert: CiliumClusterMeshFailureAlert expr: sum(cilium_clustermesh_remote_cluster_failures) by (cluster, source_cluster, target_cluster) > 0 for: 5m labels: severity: critical bu: infra team: devops service: eks-cilium env: prd priority: p0 annotations: summary: "Cilium clustermesh failure from {{ $labels.source_cluster }} to {{ $labels.target_cluster }}" description: "Cilium clustermesh failure VALUE = {{ $value }}\n LABELS: {{ $labels }}" - alert: DroppedEgressPackets expr: sum(rate(cilium_drop_count_total{direction="EGRESS", k8s_app="cilium", reason=~"No mapping for NAT masquerade|FIB lookup failed"}[1m])) by (kubernetes_pod_name, instance, reason, cluster) >= 5 for: 5m labels: severity: critical bu: infra team: devops service: eks-cilium env: prd priority: p0 annotations: summary: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}" description: "Cilium Dropped Egress Packets has crossed 5 Ops per Second for Reason - {{ $labels.reason }}, Pod - {{ $labels.kubernetes_pod_name }}, Instance - {{ $labels.instance }}, cluster - {{ $labels.cluster }} and the value is {{ $value }}\n Labels - {{ $labels }}" # - alert: SubnetOutofIpsByCilium # expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 300 # for: 1m # labels: # severity: warning # bu: infra # team: devops # service: eks-cilium # env: prd # priority: p0 # annotations: # summary: "Subnet {{ $labels.subnet_id }} has ip address less than 300. Please check" # description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 500.\n LABELS: {{ $labels }}" # - alert: SubnetOutofIpsByCilium_100 # expr: sum(cilium_operator_ipam_available_ips_per_subnet) by (cluster, subnet_id) < 100 # for: 1m # labels: # severity: critical # bu: infra # team: devops # service: eks-cilium # env: prd # priority: p0 # annotations: # summary: "Subnet {{ $labels.subnet_id }} has ip address less than 100. Please check" # description: "Subnet {{ $labels.subnet_id }} is left with ip address of VALUE = {{ $value }} which is less than 100.\n LABELS: {{ $labels }}"