added repo
This commit is contained in:
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"rd-cms-consumer_prd-cms-consumer_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-cms-consumer_prd-cms-consumer_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-cms-consumer_prd-cms-consumer_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: cms.catalog_trendmap_Consumer_Lag_Exceeded_10_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"cms.catalog_trendmap", groupId=~"catalogTrendMappingUpdateConsumer"})by(groupId,topic) > 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS cms.catalog_trendmap_Consumer_Lag_Exceeded_10 on {{ $labels.service }}"
|
||||
description: "EKS cms.catalog_trendmap_Consumer_Lag_Exceeded_10 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: cms.social_proofing_aggregate_Consumer_Lag_Exceeded_400_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"cms.social_proofing_aggregate", groupId=~"socialProofingAggregateConsumer"})by(groupId,topic) > 400
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS cms.social_proofing_aggregate_Consumer_Lag_Exceeded_400 on {{ $labels.service }}"
|
||||
description: "EKS cms.social_proofing_aggregate_Consumer_Lag_Exceeded_400 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-cms-scheduler_prd-cms-scheduler_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-cms-scheduler_prd-cms-scheduler_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-cms-scheduler_prd-cms-scheduler_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-cms-web_prd-cms-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 30
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-cms-web_prd-cms-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-cms-web_prd-cms-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: cms-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-media_prd-media-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: media
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-media_prd-media-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 15
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: media
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-media_prd-media-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 30000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: media
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-post-order-communication-review", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: post-order-communication-review
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
|
||||
description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_EKS
|
||||
expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-post-order-communication-review", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: post-order-communication-review
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
|
||||
description: "EKS HTTP 4XX Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_EKS
|
||||
expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-post-order-communication-review", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: post-order-communication-review
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
|
||||
description: "EKS HTTP 5XX Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"rd-review_prd-review-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 200
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review_prd-review-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-review_prd-review-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 600
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-admin_prd-review-admin-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-admin
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-admin_prd-review-admin-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-admin
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-review-admin_prd-review-admin-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-admin
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-consumer_prd-review-consumer_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-consumer_prd-review-consumer_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-review-consumer_prd-review-consumer_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: offers.deal_mappings_Consumer_Lag_Exceeded_100_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"offers.deal_mappings", groupId=~"review-deals-consumer-group"})by(groupId,topic) > 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS offers.deal_mappings_Consumer_Lag_Exceeded_100 on {{ $labels.service }}"
|
||||
description: "EKS offers.deal_mappings_Consumer_Lag_Exceeded_100 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: reviews.review_auto_moderation_status_Consumer_Lag_Exceeded_1000_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"reviews.review_auto_moderation_status", groupId=~"review-auto-moderation-consumer-group"})by(groupId,topic) > 1000
|
||||
for: 30m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS reviews.review_auto_moderation_status_Consumer_Lag_Exceeded_1000 on {{ $labels.service }}"
|
||||
description: "EKS reviews.review_auto_moderation_status_Consumer_Lag_Exceeded_1000 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: rvw.change_data_Consumer_Lag_Exceeded_1_Million_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"rvw.change_data", groupId=~"rvw.change_data_consumer_group"})by(groupId,topic) > 1000000
|
||||
for: 8h
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS rvw.change_data_Consumer_Lag_Exceeded_1_Million on {{ $labels.service }}"
|
||||
description: "EKS rvw.change_data_Consumer_Lag_Exceeded_1_Million on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: rvw.crud.backfill_Consumer_Lag_Exceeded_100_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"rvw.crud.backfill", groupId=~"review.crudBackFill"})by(groupId,topic) > 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS rvw.crud.backfill_Consumer_Lag_Exceeded_100 on {{ $labels.service }}"
|
||||
description: "EKS rvw.crud.backfill_Consumer_Lag_Exceeded_100 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: rvw.crud_Consumer_Lag_Exceeded_50_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"rvw.crud", groupId=~"rvw-crud-mongo-consumer"})by(groupId,topic) > 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS rvw.crud_Consumer_Lag_Exceeded_50 on {{ $labels.service }}"
|
||||
description: "EKS rvw.crud_Consumer_Lag_Exceeded_50 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: rvw.helpful.crud_Consumer_Lag_Exceeded_10_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"rvw.helpful.crud", groupId=~"review_helpful_crud_consumer_group"})by(groupId,topic) > 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS rvw.helpful.crud_Consumer_Lag_Exceeded_10 on {{ $labels.service }}"
|
||||
description: "EKS rvw.helpful.crud_Consumer_Lag_Exceeded_10 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: rvw.reprocess_v2_Consumer_Group_Lag_Exceeded_600_EKS
|
||||
expr: sum(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-shared-kafk.rzorfs.*", name="OffsetLag",topic=~"rvw.reprocess_v2", groupId=~"rvw-reprocess-mongo-consumer"})by(groupId,topic) > 600
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS rvw.reprocess_v2_Consumer_Group_Lag_Exceeded_300 on {{ $labels.service }}"
|
||||
description: "EKS rvw.reprocess_v2_Consumer_Group_Lag_Exceeded_300 on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-scheduler_prd-review-scheduler_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-review-scheduler_prd-review-scheduler_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-review-scheduler_prd-review-scheduler_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: review-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_EKS
|
||||
expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-supplier-sf", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
|
||||
description: "EKS HTTP 4XX Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-supplier-sf_prd-supplier-sf-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_EKS
|
||||
expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-supplier-sf", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 35
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
|
||||
description: "EKS HTTP 5XX Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-supplier-sf_prd-supplier-sf-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 35
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-supplier-sf", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 5000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
|
||||
description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-supplier-sf_prd-supplier-sf-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 4000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: supplier-sf
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-watcher-consumer_prd-watcher-consumer_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-watcher-consumer_prd-watcher-consumer_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-watcher-consumer_prd-watcher-consumer_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-consumer
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-watcher-web_prd-watcher-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-watcher-web_prd-watcher-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-watcher-web_prd-watcher-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 300
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: product-meta
|
||||
service: watcher-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
Reference in New Issue
Block a user