added repo
This commit is contained in:
Your Name
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest_prd-qwest-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest_prd-qwest-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest_prd-qwest-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-internal_prd-qwest-internal-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-internal
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-internal_prd-qwest-internal-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-internal
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-internal_prd-qwest-internal-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 750
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-internal
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-product-sf_prd-qwest-product-sf-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-sf
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-product-sf_prd-qwest-product-sf-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-sf
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-product-sf_prd-qwest-product-sf-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 4000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-sf
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-product-ts_prd-qwest-product-ts-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-ts
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-product-ts_prd-qwest-product-ts-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-ts
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-product-ts_prd-qwest-product-ts-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-product-ts
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-sf-old_prd-qwest-sf-old-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sf-old
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-sf-old_prd-qwest-sf-old-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sf-old
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-sf-old_prd-qwest-sf-old-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sf-old
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-sort-filter_prd-qwest-sort-filter-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-sort-filter_prd-qwest-sort-filter-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-sort-filter_prd-qwest-sort-filter-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-text-search_prd-qwest-text-search-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-text-search
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-text-search_prd-qwest-text-search-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-text-search
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-text-search_prd-qwest-text-search-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 400
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-text-search
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-vs_prd-qwest-vs_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-vs_prd-qwest-vs_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-vs_prd-qwest-vs_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 800
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-vs-sort-filter_prd-qwest-vs-sort-filter-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-qwest-vs-sort-filter_prd-qwest-vs-sort-filter-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-qwest-vs-sort-filter_prd-qwest-vs-sort-filter-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 2000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: qwest-vs-sort-filter
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-autocomplete_prd-search-autocomplete-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-autocomplete
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-autocomplete_prd-search-autocomplete-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-autocomplete
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-search-autocomplete_prd-search-autocomplete-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-autocomplete
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-orchestrator_prd-search-orchestrator-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-orchestrator
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-orchestrator_prd-search-orchestrator-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-orchestrator
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-search-orchestrator_prd-search-orchestrator-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 400
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-orchestrator
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-preprocessor-api_prd-search-preprocessor-api-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-api
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-preprocessor-api_prd-search-preprocessor-api-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-api
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: P99_Latency_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-search-preprocessor-api_prd-search-preprocessor-api-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 120
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-api
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-abg_map_builder', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-abg_map_builder-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-abg_map_builder', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-abg_map_builder-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-abg_map_builder-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-abg_map_builder-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-cache_translate_term', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-cache_translate_term-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-cache_translate_term', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-cache_translate_term-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-cache_translate_term-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-cache_translate_term-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-pop_except_norm', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-pop_except_norm-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-pop_except_norm', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-pop_except_norm-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-pop_except_norm-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-pop_except_norm-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-pop_exp_cache', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-pop_exp_cache-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-pop_exp_cache', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-pop_exp_cache-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-pop_exp_cache-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-pop_exp_cache-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-pop_query_blacklist', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-pop_query_blacklist-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-pop_query_blacklist', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-pop_query_blacklist-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-pop_query_blacklist-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-pop_query_blacklist-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-search-preprocessor-cron-stop_words_loader', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-search-preprocessor-cron-stop_words_loader-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-search-preprocessor-cron-stop_words_loader', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-search-preprocessor-cron-stop_words_loader-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-search-preprocessor-cron-stop_words_loader-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-search-preprocessor-cron-stop_words_loader-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-cron
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-preprocessor-internal_prd-search-preprocessor-internal-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-internal
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-preprocessor-internal_prd-search-preprocessor-internal-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-preprocessor-internal
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}
|
||||
LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-rule-engine_prd-search-rule-engine-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-rule-engine
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-search-rule-engine_prd-search-rule-engine-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-rule-engine
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-search-rule-engine_prd-search-rule-engine-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 750
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: demand
|
||||
team: search
|
||||
service: search-rule-engine
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
Reference in New Issue
Block a user