added repo
This commit is contained in:
Your Name
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-aggtxnsstatusupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-aggtxnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-aggtxnsstatusupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-aggtxnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-aggtxnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-aggtxnsstatusupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-balanceupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-balanceupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-balanceupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-balanceupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-balanceupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-balanceupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-failedtxnssupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-failedtxnssupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-failedtxnssupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-failedtxnssupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-failedtxnssupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-failedtxnssupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-invalidbenetxnsupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-invalidbenetxnsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-invalidbenetxnsupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-invalidbenetxnsupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-invalidbenetxnsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-invalidbenetxnsupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-noninittransactionsupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-noninittransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-noninittransactionsupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-noninittransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-noninittransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-noninittransactionsupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-pendingtransactionsupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-pendingtransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-pendingtransactionsupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-pendingtransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-pendingtransactionsupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-pendingtransactionsupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-txncountupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-txncountupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-txncountupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-txncountupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-txncountupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-txncountupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-txnsstatusupdate', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-txnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-txnsstatusupdate', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-txnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-txnsstatusupdate-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-txnsstatusupdate-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-api-payout-scheduler-userbankdetailsretry', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-api-payout-scheduler-userbankdetailsretry-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-api-payout-scheduler-userbankdetailsretry', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-api-payout-scheduler-userbankdetailsretry-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-api-payout-scheduler-userbankdetailsretry-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-api-payout-scheduler-userbankdetailsretry-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-scheduler
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_asyncValidation_bankDetails_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/asyncValidation/bankDetails"}) >= 200
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_asyncValidation_bankDetails on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_asyncValidation_bankDetails on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_directTransfer_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/directTransfer"}) >= 1300
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_directTransfer on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_directTransfer on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_directTransfer_4XX_Alert_EKS
|
||||
expr: sum by (cashFreeSubCode,uri,cluster) (CASHFREE_METRICS_value_count{cashFreeSubCode=~"4[0-9][0-9]",uri="/payout/v1/directTransfer"} ) >= 40
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_directTransfer_4XX_Alert on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_directTransfer_4XX_Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_getTransferStatus_4XX_Alert_EKS
|
||||
expr: sum by (cashFreeSubCode,uri,cluster) (CASHFREE_METRICS_value_count{cashFreeSubCode=~"4[0-9][0-9]",uri="/payout/v1.2/getTransferStatus"} ) >= 150
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_getTransferStatus_4XX_Alert on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_getTransferStatus_4XX_Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_getValidationStatus_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/getValidationStatus/bank"}) >= 100
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_getValidationStatus on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_getValidationStatus on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_ifsc_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/ifsc/{ifsc}"}) >= 150
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_ifsc on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_ifsc on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_v12_validation_bankDetails_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1.2/validation/bankDetails"}) >= 200
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_v12_validation_bankDetails on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_v12_validation_bankDetails on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_v12_validation_bankDetails_4XX_Alert_EKS
|
||||
expr: sum by (cashFreeSubCode,uri,cluster) (CASHFREE_METRICS_value_count{cashFreeSubCode=~"4[0-9][0-9]",uri="/payout/v1.2/validation/bankDetails"} ) >= 120
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_v12_validation_bankDetails_4XX_Alert on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_v12_validation_bankDetails_4XX_Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_v1_validation_bankDetails_EKS
|
||||
expr: (sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/validation/bankDetails"})) >= 50
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_v1_validation_bankDetails on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_v1_validation_bankDetails on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: CASHFREE_METRICS_value_count_v1_validation_upiDetails_EKS
|
||||
expr: sum by (uri,cluster) (CASHFREE_METRICS_value_count{uri="/payout/v1/validation/upiDetails"}) >= 400
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS CASHFREE_METRICS_value_count_v1_validation_upiDetails on {{ $labels.service }}"
|
||||
description: "EKS CASHFREE_METRICS_value_count_v1_validation_upiDetails on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Per_API_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Per_API_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Per_API_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Error_Percentage_Alert_Contour
|
||||
expr: ((sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80",envoy_response_code=~"[4].*"}[2m])) by (cluster, envoy_cluster_name, job) * 60 ) / (sum(rate(envoy_cluster_upstream_rq_total{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80"}[2m]))*60) )*100 >= 2
|
||||
for: 1s
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Error_Percentage_Alert on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Error_Percentage_Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Per_API_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Per_API_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Per_API_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Error_Percentage_Alert_Contour_EKS
|
||||
expr: ((sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80",envoy_response_code=~"[5].*"}[2m])) by (cluster, envoy_cluster_name, job) * 60 ) / (sum(rate(envoy_cluster_upstream_rq_total{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80"}[2m]))*60) )*100 >= 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Error_Percentage_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Error_Percentage_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-api-payout-web_prd-api-payout-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 9000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: WALLET_TRANSACTION_RESPONSE_STATUS_value_count_EKS
|
||||
expr: (sum(WALLET_TRANSACTION_RESPONSE_STATUS_value_count{status!="success"}) by (status,cluster)) >= 0
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: api-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS WALLET_TRANSACTION_RESPONSE_STATUS_value_count_Alert on {{ $labels.service }}"
|
||||
description: "EKS WALLET_TRANSACTION_RESPONSE_STATUS_value_count_Alert on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-file-payout-web_prd-file-payout-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: file-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-file-payout-web_prd-file-payout-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: file-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-file-payout-web_prd-file-payout-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 75
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: file-payout-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-invoice-web_prd-invoice-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: invoice-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-invoice-web_prd-invoice-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 15
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: invoice-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-invoice-web_prd-invoice-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 5000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: invoice-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-payment-aggregator-web_prd-payment-aggregator-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 250
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-aggregator-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-payment-aggregator-web_prd-payment-aggregator-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-aggregator-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-payment-aggregator-web_prd-payment-aggregator-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-aggregator-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour_EKS on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour_EKS on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-payment-communication-web_prd-payment-communication-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-communication-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-payment-communication-web_prd-payment-communication-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-communication-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-payment-communication-web_prd-payment-communication-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 750
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: payment-communication-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-reseller-payment-scheduler-aggregatepayments', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-reseller-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-reseller-payment-scheduler-aggregatepayments', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-reseller-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-reseller-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-reseller-payment-scheduler-aggregatepayments-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-reseller-payment-scheduler-expiredrequestlogs', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-reseller-payment-scheduler-expiredrequestlogs-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-reseller-payment-scheduler-expiredrequestlogs', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-reseller-payment-scheduler-expiredrequestlogs-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-reseller-payment-scheduler-expiredrequestlogs-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-reseller-payment-scheduler-expiredrequestlogs-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-reseller-payment-scheduler-failedmanualrefund', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-reseller-payment-scheduler-failedmanualrefund-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-reseller-payment-scheduler-failedmanualrefund', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-reseller-payment-scheduler-failedmanualrefund-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-reseller-payment-scheduler-failedmanualrefund-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-reseller-payment-scheduler-failedmanualrefund-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-reseller-payment-web_prd-reseller-payment-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-reseller-payment-web_prd-reseller-payment-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-reseller-payment-web_prd-reseller-payment-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 5000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: reseller-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-aggearlypayments', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-aggearlypayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-aggearlypayments', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-aggearlypayments-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-aggearlypayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-aggearlypayments-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-aggregatepayments', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-aggregatepayments', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-aggregatepayments-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-aggregatepayments-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-dualwritecor', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-dualwritecor-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-dualwritecor', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-dualwritecor-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-dualwritecor-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-dualwritecor-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-failedpaymentpro', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-failedpaymentpro-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-failedpaymentpro', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-failedpaymentpro-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-failedpaymentpro-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-failedpaymentpro-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-gstreportsproc', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-gstreportsproc-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-gstreportsproc', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-gstreportsproc-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-gstreportsproc-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-gstreportsproc-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-migrateclaimsdata', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-migrateclaimsdata-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-migrateclaimsdata', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-migrateclaimsdata-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-migrateclaimsdata-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-migrateclaimsdata-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-migratesupplydata', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-migratesupplydata-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-migratesupplydata', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-migratesupplydata-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-migratesupplydata-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-migratesupplydata-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-monthlyaggearlypay', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-monthlyaggearlypay-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-monthlyaggearlypay', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-monthlyaggearlypay-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-monthlyaggearlypay-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-monthlyaggearlypay-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
groups:
|
||||
- name: EKS-CronJob-Alerts
|
||||
rules:
|
||||
- alert: CronJob-failedJobsHistoryLimit-GreaterThan-1
|
||||
expr: kube_cronjob_spec_failed_job_history_limit{cronjob=~'prd-supplier-payment-scheduler-requestlogscleanup', job=~'kube-state-metrics-p-.*'} > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }}"
|
||||
description: "CronJob - {{ $labels.cronjob }} has failedJobsHistoryLimit Greater than {{ $value }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Failed
|
||||
expr: kube_job_status_failed{job_name=~'prd-supplier-payment-scheduler-requestlogscleanup-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob has FAILED"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob has FAILED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Running-MoreThan-60
|
||||
expr: time() - kube_cronjob_next_schedule_time{cronjob=~'prd-supplier-payment-scheduler-requestlogscleanup', job=~'kube-state-metrics-p-.*'} > 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "CronJob - {{ $labels.cronjob }} is running for more than 60"
|
||||
description: "CronJob - {{ $labels.cronjob }} is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Job-Running-MoreThan-60
|
||||
expr: kube_job_spec_completions{job_name=~'prd-supplier-payment-scheduler-requestlogscleanup-.*', job=~'kube-state-metrics-p-.*'} - kube_job_status_succeeded{job_name=~'prd-supplier-payment-scheduler-requestlogscleanup-.*', job=~'kube-state-metrics-p-.*'} > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60"
|
||||
description: "Job - {{ $labels.job_name }}, of the CronJob is running for more than 60 VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: CronJob-Pod-Created
|
||||
expr: increase(kube_pod_info{job=~'kube-state-metrics-p-.*', created_by_name=~'prd-supplier-payment-scheduler-requestlogscleanup-.*'}[1m]) > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-scheduler
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "Pod - {{ $labels.pod }}, of the CronJob is CREATED"
|
||||
description: "Pod - {{ $labels.pod }}, of the CronJob is CREATED VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-supplier-payment-web_prd-supplier-payment-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 250
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-supplier-payment-web_prd-supplier-payment-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-supplier-payment-web_prd-supplier-payment-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 5000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: supplier-payment-web
|
||||
env: prd
|
||||
priority: p1
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_4XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-taxation-web_prd-taxation-web-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: taxation-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: HTTP_5XX_Alert_Contour_EKS
|
||||
expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-taxation-web_prd-taxation-web-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: taxation-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
|
||||
description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-taxation-web_prd-taxation-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 75
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: taxation-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour_EKS on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour_EKS on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
groups:
|
||||
- name: EKS-App-Alerts
|
||||
rules:
|
||||
- alert: Latency_P99_Alert_Contour_EKS_EKS_EKS
|
||||
expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-taxation-web_prd-taxation-web-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 150
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: supply
|
||||
team: payout
|
||||
service: taxation-web
|
||||
env: prd
|
||||
priority: p2
|
||||
annotations:
|
||||
summary: "EKS Latency_P99_Alert_Contour_EKS_EKS on {{ $labels.service }}"
|
||||
description: "EKS Latency_P99_Alert_Contour_EKS_EKS on {{ $labels.service }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
Reference in New Issue
Block a user