added repo
This commit is contained in:
Your Name
+11
@@ -0,0 +1,11 @@
|
||||
groups:
|
||||
- name: URL-Healthcheck
|
||||
rules:
|
||||
- alert: HealcheckFailing
|
||||
expr: up{job="blackbox"} == 0 or probe_success{job="blackbox"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: "critical"
|
||||
annotations:
|
||||
summary: "Endpoint {{ $labels.instance }} down"
|
||||
description: "Healthcheck Endpoint: {{ $labels.instance }} is down. Please check."
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
groups:
|
||||
- name: confluent-cloud-alerts
|
||||
rules:
|
||||
- alert: ConsumerLag
|
||||
expr: (confluent_kafka_server_consumer_lag_offsets{job="Confluent_Cloud"}) > 15000000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ConsumerLag Offsets for (Kafka {{ $labels.kafka_id }}) and for the topic (Topic Name : {{ $labels.topic }}) , (Consumer : {{ $labels.consumer_group_id}}) is greater than 15000"
|
||||
description: "ConsumerLag Offsets for (Kafka {{ $labels.kafka_id }}) and for the topic (Topic Name : {{ $labels.topic }}) , (Consumer : {{ $labels.consumer_group_id}}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
groups:
|
||||
- name: Kafka-lag-Alerts
|
||||
rules:
|
||||
- alert: displayAdsCampaignReactivate_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"display_ads.campaign_reactivate", groupId=~"displayAdsCampaignReactivate"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu : supply
|
||||
team : display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: displayAdsBudgetExhausted_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"display_ads.budget_exhausted", groupId=~"displayAdsBudgetExhausted"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu : supply
|
||||
team : display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: displayAdsRedisRetry_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"display_ads.redis_retry", groupId=~"displayAdsRedisRetry"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: displayAdsWidgetView_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"test_topic", groupId=~"displayAdsWidgetView"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: displayAdsDSFeed_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"display_ads.ds_feed", groupId=~"displayAdsDSFeed"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: displayAdsBannerView_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"test_topic", groupId=~"displayAdsBannerView"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: adWidgetViewEventConsumerGroup_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"rt_ad_widget_views", groupId=~"adWidgetViewEventConsumerGroup"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: adBannerViewEventConsumer_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"rt_banner_viewed", groupId=~"adBannerViewEventConsumer"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: adWidgetClickEventConsumer_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"rt_widget_clicked", groupId=~"adWidgetClickEventConsumer"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: adBannerClickEventConsumer_greater_than_1000
|
||||
expr: sum by (groupId,topic)(kafka_consumer_group_ConsumerLagMetrics_Value{instance=~".*bac-p-display-ads-kafka.*", name="OffsetLag", topic=~"rt_banner_clicked", groupId=~"adBannerClickEventConsumer"}) > 1000
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
env: prd
|
||||
bu: supply
|
||||
team: display-ads
|
||||
metric_type: consumer_lag_metric
|
||||
annotations:
|
||||
summary: "Consumer Lag on {{ $labels.groupId }}"
|
||||
description: "Consumer Lag on {{ $labels.groupId }} VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,92 @@
|
||||
groups:
|
||||
- name: EKS-Common-Alerts
|
||||
rules:
|
||||
- alert: KubernetesPersistentvolumeclaimPending
|
||||
expr: kube_persistentvolumeclaim_status_phase{phase="Pending",namespace!~"int-.*"} == 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})
|
||||
description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesVolumeOutOfDiskSpace
|
||||
expr: kubelet_volume_stats_available_bytes{namespace!~"int-.*"} / kubelet_volume_stats_capacity_bytes{namespace!~"int-.*"} * 100 < 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
|
||||
description: "Volume is almost full (< 30% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesVolumeFullInFourDays
|
||||
expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesPersistentvolumeError
|
||||
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job=~"kube-state-metrics", namespace!~"int-.*"} > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
|
||||
description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetDown
|
||||
expr: (kube_statefulset_status_replicas_ready{statefulset!="vault",namespace!~"int-.*"} / kube_statefulset_status_replicas_current{statefulset!="vault",namespace!~"int-.*"}) != 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetDownVault
|
||||
expr: (kube_statefulset_status_replicas_ready{statefulset=~"vault",namespace!~"int-.*"} / kube_statefulset_replicas{statefulset=~"vault",namespace!~"int-.*"}) != 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
# - alert: KubernetesHpaMetricAvailability
|
||||
# expr: kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive", namespace!~"argocd|grafana" } == 1
|
||||
# for: 0m
|
||||
# labels:
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: Kubernetes HPA metric availability (instance {{ $labels.instance }})
|
||||
# description: "HPA is not able to collect metrics\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesHpaScaleCapability
|
||||
expr: kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"int-.*"} >= kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"int-.*"}
|
||||
for: 2m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: Kubernetes HPA scale capability (instance {{ $labels.instance }})
|
||||
description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDeploymentReplicasMismatch
|
||||
expr: kube_deployment_spec_replicas{namespace!~"int-.*"} != kube_deployment_status_replicas_available{namespace!~"int-.*"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})
|
||||
description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetReplicasMismatch
|
||||
expr: kube_statefulset_status_replicas_ready{namespace!~"int-.*"} != kube_statefulset_status_replicas{namespace!~"int-.*"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet does not match the expected number of replicas.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
groups:
|
||||
- name: Elastic-HA-Proxy-Alerts
|
||||
rules:
|
||||
- alert: haproxy_backend_status
|
||||
expr: avg by(proxy)(haproxy_backend_status{job=~"elastic-fallback-haproxy.*", state="UP"}) < 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: elastic-ha-proxy
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "haproxy backend status is down"
|
||||
description: "haproxy backend status is down VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: Frontend_in_state_Elastic_HAProxy
|
||||
expr: sum(rate(haproxy_frontend_bytes_in_total{job=~"elastic-fallback-haproxy.*"}[1m])) by (proxy) >= 650
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: elastic-ha-proxy
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "Frontend in state Elastic HAProxy"
|
||||
description: "Frontend_in_state_Elastic_HAProxy is greater than VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: haproxy_backend_5xx_total
|
||||
expr: sum(rate(haproxy_backend_http_responses_total{job="elastic-fallback-haproxy.*",code=~"5xx|other"}[1m])) by (proxy,code) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: elastic-ha-proxy
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "haproxy backend 5xx is high"
|
||||
description: "haproxy backend 5xx is greater VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: haproxy_backend_4xx_total
|
||||
expr: sum(rate(haproxy_backend_http_responses_total{job="elastic-fallback-haproxy.*",code=~"4xx|other"}[1m])) by (proxy,code) * 60 >= 20
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: elastic-ha-proxy
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "haproxy backend 4xx status is down"
|
||||
description: "haproxy backend 4xx is down VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: haproxy_frontend_bytes_out_total
|
||||
expr: sum(rate(haproxy_frontend_bytes_out_total{job="elastic-fallback-haproxy.*"}[1m])) by (proxy) >= 650
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
bu: infra
|
||||
team: devops
|
||||
service: elastic-ha-proxy
|
||||
env: prd
|
||||
priority: p0
|
||||
annotations:
|
||||
summary: "haproxy frontend bytes out total is high"
|
||||
description: "haproxy_frontend_bytes_out_total is greater than VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
@@ -0,0 +1,812 @@
|
||||
groups:
|
||||
- name: Elastic-Search-Alerts
|
||||
rules:
|
||||
- alert: FreeStorageSpace-100G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name !~ 'bac-p-advertisement|bac-p-supplier-orders|bac-p-orders|bac-p-supplier-search|bac-p-advertisement-search|platform-p-qwest-analytics|bac-p-address-search|bac-p-sx-support|bac-p-review2'} offset 10m / 1024 < 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 100G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: FreeStorageSpace-3G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-sx-support'} offset 10m / 1024 < 3
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 3G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: FreeStorageSpace-25G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-orders'} offset 10m / 1024 < 25
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 25G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-80G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'platform-p-qwest-analytics'} offset 10m / 1024 < 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 80G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-90G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-supplier-search|bac-p-advertisement'} offset 10m / 1024 < 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 90G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-85G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-review2'} offset 10m / 1024 < 85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 85G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-60G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-advertisement-search'} offset 10m / 1024 < 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 60G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-50G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-address-search'} offset 10m / 1024 < 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 50G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeStorageSpace-40G
|
||||
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-supplier-orders'} offset 10m / 1024 < 40
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 40G"
|
||||
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: CPUUtilization_80+
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name !~ 'platform-p-garylog|bac-p-scraper|platform-p-qwest-analytics|platform-p-qwest'} offset 10m > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
||||
description: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: CPUUtilization_70+
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name =~ 'platform-p-qwest'} offset 10m > 70
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 70%"
|
||||
|
||||
|
||||
- alert: CPUUtilization_90+
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name =~ 'bac-p-scraper|platform-p-qwest-analytics'} offset 10m > 90
|
||||
for: 4m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 90%"
|
||||
|
||||
|
||||
- alert: JVMMemoryPressure
|
||||
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name !~ 'platform-p-garylog|bac-p-orders|platform-p-ts-products|platform-p-qwest-analytics|platform-p-qwest-sf|platform-p-qwest-ts-catalogs|platform-p-ofs-catalogs|platform-p-ofs-products'} offset 10m >= 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
||||
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
||||
|
||||
|
||||
- alert: JVMMemoryPressure
|
||||
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'platform-p-qwest-analytics'} offset 10m >= 80
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
||||
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: JVMMemoryPressure
|
||||
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'bac-p-orders'} offset 10m >= 95
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 95%"
|
||||
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: JVMMemoryPressure
|
||||
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'platform-p-qwest-sf|platform-p-ofs-catalogs|platform-p-qwest-ts-catalogs|platform-p-ts-products|platform-p-ofs-products'} offset 10m >= 90
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 90%"
|
||||
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: ClusterIndexWritesBlocked
|
||||
expr: aws_es_cluster_index_writes_blocked_maximum{job="aws_es"} offset 10m >= 1
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "ClusterIndexWritesBlocked in Elastic Search {{ $labels.domain_name }} is blocked for more than 1 mins"
|
||||
description: "ClusterIndexWritesBlocked in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: ClusterStatus.red
|
||||
expr: aws_es_cluster_status_red_maximum{job="aws_es"} offset 10m >= 1
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "ClusterStatus.red for Elastic Search {{ $labels.domain_name }}."
|
||||
description: "ClusterStatus.red for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: Nodes
|
||||
expr: aws_es_nodes_minimum{job="aws_es", domain_name=~ "bac-p-review2|platform-p-qwest" } < 9 or aws_es_nodes_minimum{job="aws_es", domain_name="bac-p-scraper" } < 6 or aws_es_nodes_minimum{job="aws_es", domain_name="bac-p-supplier-search" } < 2
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Nodes for Elastic Search {{ $labels.domain_name }} is less than mentioned number in expression."
|
||||
description: "Cluster ES node for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: KibanaHealthyNodes
|
||||
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es"} offset 10m != 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
||||
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: KibanaHealthyNodes
|
||||
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name!~"platform-p-qwest-sort-filter|platform-p-qwest-analytics|platform-p-qwest-text-search" } offset 10m < 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
||||
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: KibanaHealthyNodes
|
||||
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name=~"platform-p-qwest-sort-filter|platform-p-qwest-text-search" } offset 10m < 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
||||
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: KibanaHealthyNodes
|
||||
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name=~"platform-p-qwest-analytics" } offset 10m < 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
||||
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_4xx_20+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name!~"platform-p-graylog|campaign-performance|platform-p-ts-products|platform-p-qwest-sf|platform-p-garylog|platform-p-qwest-sort-filter|bac-p-advertisement|bac-p-supplier-search|platform-p-ofs|bac-p-advertisement-ts|platform-p-qwest|platform-p-qwest-text-search|bac-p-scraper|platform-p-qwest-analytics|platform-p-ofs-catalogs|platform-p-sf-catalogs"} offset 10m >= 20
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_4xx_50+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-ofs|platform-p-qwest-sf"} offset 10m >= 50
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_4xx_100+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"bac-p-scraper|campaign-performance|platform-p-qwest-analytics|platform-p-qwest-sort-filter|platform-p-qwest-text-search|platform-p-ofs-catalogs|bac-p-advertisement|bac-p-advertisement-ts"} offset 10m >= 100
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_4xx_200+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-sf-catalogs"} offset 10m >= 200
|
||||
for: 300s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
- alert: ES_4xx_170+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 170
|
||||
for: 300s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
- alert: ES_4xx_1000+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-ts-products"} offset 10m >= 1000
|
||||
for: 300s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_4xx_50+
|
||||
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-qwest"} offset 10m >= 50
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_5xx_5+
|
||||
expr: aws_es_5xx_sum{job="aws_es", domain_name!~"platform-p-garylog"} offset 10m >= 5
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "5xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
description: "5xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# NODE WISE ES RULES
|
||||
|
||||
|
||||
|
||||
- alert: ES_Master_CPU_Utilization_60%
|
||||
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name!~"bac-p-recommendation|bac-p-scraper|platform-p-garylog"} offset 10m >= 60
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60%. Please check."
|
||||
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Master_CPU_Utilization_60%
|
||||
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name=~"bac-p-recommendation|bac-p-recommendation-price"} offset 10m >= 60
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60%. Please check."
|
||||
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Master_CPU_Utilization_30%
|
||||
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name=~"bac-p-scraper"} offset 10m >= 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 30%. Please check."
|
||||
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Node_CPU_Utilization_65%
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name!~"platform-p-qwest-analytics|platform-p-garylog|bac-p-scraper|platform-p-qwest-ts-catalogs"} offset 10m >= 65
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 65%. Please check."
|
||||
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Node_CPU_Utilization_80%
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"bac-p-scraper"} offset 10m >= 80
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 80%. Please check."
|
||||
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Node_CPU_Utilization_90%
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"platform-p-qwest-analytics"} offset 10m >= 90
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 90%. Please check."
|
||||
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: ES_Node_CPU_Utilization_70%
|
||||
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"platform-p-qwest-ts-catalogs"} offset 10m >= 70
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 70%. Please check."
|
||||
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_100k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name!~"bac-p-recommendation|platform-p-ofs-catalogs|bac-p-advertisement-search|campaign-performance|platform-p-ts-products|bac-p-supplier-orders|bac-p-merchandising|bac-p-advertisement|platform-p-garylog|platform-p-ofs|bac-p-supplier-search|platform-p-qwest-sort-filter|bac-p-review2|platform-p-qwest|platform-p-qwest-analytics|bac-p-recommendation-price|platform-p-qwest-text-search|platform-p-ofs-products|bac-p-orders"} offset 10m >= 100000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_100k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"campaign-performance"} offset 10m >= 100000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_200k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-advertisement-search|bac-p-orders"} offset 10m >= 200000
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 200k. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_250k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"platform-p-ofs"} offset 10m >= 250000
|
||||
for: 20s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 250k. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_150k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-merchandising|platform-p-ofs-catalogs|platform-p-qwest-text-search|platform-p-qwest-sort-filter|bac-p-advertisement|platform-p-qwest-analytics"} offset 10m >= 150000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 150K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_170k
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"platform-p-ts-products"} offset 10m >= 170000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 170K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_1.2M
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-recommendation"} offset 10m >= 1200000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 1.2M. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_120K
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-review2|platform-p-qwest"} offset 10m >= 120000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 120K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_700K
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 700000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 700K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Indexing_Rate_400K
|
||||
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-recommendation-price"} offset 10m >= 400000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 400K. Please check."
|
||||
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_60ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name!~"bac-p-address-search|bac-p-recommendation|platform-p-ofs|platform-p-qwest-ts-catalogs|bac-p-recommendation-price|platform-p-garylog|bac-p-scraper|platform-p-qwest-sort-filter|bac-p-supplier-order-search|platform-p-qwest|bac-p-supplier-search|platform-p-qwest-analytics|platform-p-qwest-text-search|bac-p-review2|bac-p-orders|platform-p-qwest-sf|bac-p-merchandising|platform-p-ofs-products|platform-p-ts-products"} offset 10m >= 60
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 60ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_50ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-supplier-order-search|bac-p-orders"} offset 10m >= 50
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 50ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_500ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-sort-filter|platform-p-ts-products|bac-p-address-search"} offset 10m >= 500
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 500ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_80ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest"} offset 10m >= 80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 80ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_1000ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-text-search"} offset 10m >= 1000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 1000ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_75ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-ofs"} offset 10m >= 75
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 75ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
- alert: ES_Search_Latency_100ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-analytics|platform-p-qwest-ts-catalogs"} offset 10m >= 100
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_8000ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-merchandising"} offset 10m >= 8000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 8000ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_300ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-review2|platform-p-qwest-sf"} offset 10m >= 300
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 300ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_200ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-ofs-products"} offset 10m >= 200
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 200ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
- alert: ES_Search_Latency_250ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-supplier-search"} offset 10m >= 250
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 250ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_450ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-scraper"} offset 10m >= 450
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 450ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Latency_100ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-recommendation-price"} offset 10m >= 100
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
- alert: ES_Search_Latency_150ms
|
||||
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-recommendation"} offset 10m >= 150
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 150ms. Please check."
|
||||
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_20K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name!~"platform-p-ofs-catalogs|platform-p-qwest-ts-catalogs|bac-p-orders|bac-p-review2|bac-p-supplier-orders|platform-p-ofs|platform-p-garylog|bac-p-advertisement-ts|bac-p-supplier-order-search|bac-p-supplier-search|bac-p-advertisement|platform-p-qwest-text-search|platform-p-qwest-analytics|bac-p-community|bac-p-advertisement-search"} offset 10m >= 20000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 20K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
- alert: ES_Search_Rate_30K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-orders"} offset 10m >= 30000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 30K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_40K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-advertisement-ts|bac-p-advertisement-search|platform-p-ofs|platform-p-ofs-catalogs"} offset 10m >= 40000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 40K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_120K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-order-search"} offset 10m >= 120000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 120K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_65K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"platform-p-qwest-analytics|platform-p-qwest-ts-catalogs"} offset 10m >= 65000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 65K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
- alert: ES_Search_Rate_60K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-orders"} offset 10m >= 60000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_60K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-community"} offset 10m >= 60000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_55K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"platform-p-qwest-text-search"} offset 10m >= 55000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 55K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_80K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-advertisement|bac-p-review2"} offset 10m >= 80000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 80K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: ES_Search_Rate_200K
|
||||
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 200000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Elastic Search
|
||||
annotations:
|
||||
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 200K. Please check."
|
||||
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
||||
@@ -0,0 +1,189 @@
|
||||
groups:
|
||||
- name: Kafka-Alerts
|
||||
rules:
|
||||
- alert: GlobalTopicCount_200+
|
||||
expr: aws_kafka_global_topic_count_average{job="aws_kafka"} offset 5m > 200
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is greater than 200+"
|
||||
description: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: OfflinePartitionsCount_0+
|
||||
expr: aws_kafka_offline_partitions_count_average{job="aws_kafka"} offset 5m > 0
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is greater than 0"
|
||||
description: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: KafkaDataLogsDiskUsed
|
||||
expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name!~"bac-p-communicator.*|bac-p-datahub.*|bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 80
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 80"
|
||||
description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: KafkaDataLogsDiskUsed
|
||||
expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-communicator.*"} offset 5m > 60
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
|
||||
description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: KafkaDataLogsDiskUsed
|
||||
expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 60
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
|
||||
description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
- alert: KafkaAppLogsDiskUsed
|
||||
expr: aws_kafka_kafka_app_logs_disk_used_average{job="aws_kafka"} offset 10m > 50
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 50"
|
||||
description: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: KafkaDataLogsDiskUsed
|
||||
expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-datahub.*"} offset 5m > 90
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 90"
|
||||
description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: RootDiskUsed
|
||||
expr: aws_kafka_root_disk_used_average{job="aws_kafka"} offset 10m > 60
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
|
||||
description: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: NetworkRxDropped
|
||||
expr: aws_kafka_network_rx_dropped_average{job="aws_kafka"} offset 10m > 0
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is greater than 0"
|
||||
description: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: NetworkTxDropped
|
||||
expr: aws_kafka_network_tx_dropped_average{job="aws_kafka"} offset 10m > 0
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is greater than 0"
|
||||
description: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: CpuUser
|
||||
expr: aws_kafka_cpu_user_average{job="aws_kafka"} offset 10m > 85
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "CpuUser in Kafka {{ $labels.cluster_name }} is greater than 85"
|
||||
description: "CpuUser in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: ZooKeeperSessionState
|
||||
expr: aws_kafka_zoo_keeper_session_state_sample_count{job="aws_kafka"} offset 10m != 1
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is not equal to 1"
|
||||
description: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: ActiveControllerCount
|
||||
expr: aws_kafka_active_controller_count_sample_count{cluster_name =~ "data-platform-p-kafka|data-platform-p-prism-kafka|bac-p-communicator-kafka", job="aws_kafka"} offset 10m < 3
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 3"
|
||||
description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: Kafka_BytesInPerSec
|
||||
expr: aws_kafka_bytes_in_per_sec_average{job="aws_kafka"} offset 10m == 0
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is 0"
|
||||
description: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
- alert: Kafka_BytesOutPerSec
|
||||
expr: aws_kafka_bytes_out_per_sec_average{job="aws_kafka"} offset 10m == 0
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is not equal to 3"
|
||||
description: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: Kafka_ReplicationBytes_plus_bytes_in_and_out_200MB+
|
||||
expr: (aws_kafka_bytes_in_per_sec_average offset 10m + aws_kafka_bytes_out_per_sec_average offset 10m + aws_kafka_replication_bytes_in_per_sec_average offset 10m + aws_kafka_replication_bytes_out_per_sec_average offset 10m ) > 200000000
|
||||
for: 60s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is not equal to 200MB"
|
||||
description: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
|
||||
- alert: ActiveControllerCount_IngestionKafka
|
||||
expr: aws_kafka_active_controller_count_sample_count{cluster_name="bac-p-ingestion-kafka",job="aws_kafka"} offset 10m != 4
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Kafka
|
||||
annotations:
|
||||
summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 4"
|
||||
description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
groups:
|
||||
- name: kubernetes-cost-report.rules
|
||||
interval: 5m
|
||||
rules:
|
||||
- expr: "(\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (kube_node_labels{job=\"kube-state-metrics\"})
|
||||
\n * on (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_cost_all{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance:spot_instance_cost:cost
|
||||
- expr: |-
|
||||
sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(
|
||||
zone_capacity_instance:spot_instance_cost:cost
|
||||
)
|
||||
record: capacity_instance:spot_instance_cost:cost
|
||||
- expr: "(\n sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(kube_node_labels{job=\"kube-state-metrics\"}) \n * on (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cost_all{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance:on_demand_instance_cost:cost
|
||||
- expr: "(\n (\n (sum by(namespace, node, pod) (cluster:namespace:pod_memory:active:kube_pod_container_resource_requests)
|
||||
/1024/1024/1024) \n * on (node) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n )\n\n * ignoring(namespace,
|
||||
node, pod) group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_mem_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:spot_pod_mem_requests_cost
|
||||
- expr: "(\n (\n (sum by(namespace, node, pod) (cluster:namespace:pod_memory:active:kube_pod_container_resource_requests)
|
||||
/1024/1024/1024) \n * on (node) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n )\n\n * ignoring(namespace,
|
||||
node, pod) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_mem_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_namespace_node_pod:pod_memory_requests_instance_mem_price:on_demand_pod_mem_requests_cost
|
||||
- expr: "(\n (\n sum by(namespace, node, pod) (cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests)
|
||||
\n * on (node) group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n
|
||||
\ )\n\n * ignoring(namespace, node, pod) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:spot_pod_cpu_requests_cost
|
||||
- expr: "(\n (\n sum by(namespace, node, pod) (cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests)
|
||||
\n * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n
|
||||
\ )\n\n * ignoring(namespace, node, pod) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_namespace_node_pod:pod_cpu_requests_instance_cpu_price:on_demand_pod_cpu_requests_cost
|
||||
- expr: "(\n (\n (sum by (namespace, node, pod) (container_memory_working_set_bytes{name!=\"\"})
|
||||
/1024/1024/1024)\n * on (node) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n )\n\n * ignoring(namespace,
|
||||
node, pod) group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_mem_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:spot_pod_mem_usage_cost
|
||||
- expr: "(\n (\n (sum by (namespace, node, pod) (container_memory_working_set_bytes{name!=\"\"})
|
||||
/1024/1024/1024)\n * on (node) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n )\n\n * ignoring(namespace,
|
||||
node, pod) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_mem_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_namespace_node_pod:pod_memory_usage_instance_mem_price:on_demand_pod_mem_usage_cost
|
||||
- expr: "(\n (\n sum by(namespace, node, pod) (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate)
|
||||
\n * on (node) group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{job=\"kube-state-metrics\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n
|
||||
\ )\n\n * ignoring(namespace, node, pod) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:spot_pod_cpu_usage_cost
|
||||
- expr: "(\n (\n sum by(namespace, node, pod) (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate)
|
||||
\n * on (node) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{job=\"kube-state-metrics\", label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n
|
||||
\ )\n\n * ignoring(namespace, node, pod) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_namespace_node_pod:pod_cpu_usage_instance_cpu_price:on_demand_pod_cpu_usage_cost
|
||||
- expr: "(\n (\n sum by (node, resource) (\n (\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"}\n - on (node) group_left\n sum by (node)
|
||||
(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate)\n
|
||||
\ )\n )\n\n * on (node) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ \n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n )\n\n * ignoring (node,
|
||||
resource) group_left\n\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n\n)"
|
||||
record: zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:spot_idle_cpu_cost
|
||||
- expr: "(\n (\n sum by (node, resource) (\n (\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"}\n - on (node) group_left\n sum by (node)
|
||||
(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate)\n
|
||||
\ )\n )\n\n * on (node) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n \n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n )\n\n * ignoring
|
||||
(node, resource) group_left\n\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_cpu_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n\n)"
|
||||
record: capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_cpu_price:on_demand_idle_cpu_cost
|
||||
- expr: "(\n (\n (\n sum by (node, resource) (\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"}\n - on (node) group_left\n sum by (node)
|
||||
(container_memory_working_set_bytes{job=\"kubelet\", name!=\"\"})\n )\n
|
||||
\ /1024/1024/1024\n )\n\n * on (node) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ \n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n )\n\n * ignoring (node,
|
||||
resource) group_left\n\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_mem_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n\n)"
|
||||
record: zone_capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:spot_idle_mem_cost
|
||||
- expr: "(\n (\n (\n sum by (node, resource) (\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"}\n - on (node) group_left\n sum by (node)
|
||||
(container_memory_working_set_bytes{job=\"kubelet\", name!=\"\"})\n )\n
|
||||
\ /1024/1024/1024\n )\n\n * on (node) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n \n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n )\n\n * ignoring
|
||||
(node, resource) group_left\n\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_mem_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n\n)"
|
||||
record: capacity_instance_node_resource:kube_node_status_allocatable_idle_instance_mem_price:on_demand_idle_mem_cost
|
||||
- expr: "(\n (\n (\n sum by(node, resource) (\n ceil(kube_node_status_capacity{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"}) \n -\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"}\n )\n ) \n \n * on (node) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ \n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n )\n\n * ignoring(node,
|
||||
resource) group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_cpu_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:spot_shared_cpu_cost
|
||||
- expr: "(\n (\n (\n sum by(node, resource) (\n kube_node_status_capacity{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"} \n -\n kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"cpu\"}\n )\n ) \n \n * on (node) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n \n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type, node) (kube_node_labels{job=\"kube-state-metrics\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n )\n\n * ignoring(node,
|
||||
resource) group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
(instance_cpu_price{job=\"kubernetes-cost-report\", label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_node_resource:kube_node_status_shared_instance_cpu_price:on_demand_shared_cpu_cost
|
||||
- expr: "(\n (\n (\n sum by(node, resource) (\n ceil(kube_node_status_capacity{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"} /1024/1024/1024)\n -\n (kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"} /1024/1024/1024)\n )\n ) \n \n * on (node)
|
||||
group_left(label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type)\n \n sum by (label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{job=\"kube-state-metrics\", label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n
|
||||
\ )\n\n * ignoring(node, resource) group_left(label_topology_kubernetes_io_zone,
|
||||
label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)
|
||||
\n\n sum by (label_topology_kubernetes_io_zone, label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_mem_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"SPOT\"})\n)"
|
||||
record: zone_capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:spot_shared_mem_cost
|
||||
- expr: "(\n (\n (\n sum by(node, resource) (\n ceil(kube_node_status_capacity{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"} /1024/1024/1024)\n -\n (kube_node_status_allocatable{job=\"kube-state-metrics\",
|
||||
resource=\"memory\"} /1024/1024/1024)\n )\n )\n \n * on (node)
|
||||
group_left(label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type)\n
|
||||
\ \n sum by (label_eks_amazonaws_com_capacity_type, label_beta_kubernetes_io_instance_type,
|
||||
node) (kube_node_labels{job=\"kube-state-metrics\", label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n
|
||||
\ )\n\n * ignoring(node, resource) group_left(label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) \n\n sum by (label_eks_amazonaws_com_capacity_type,
|
||||
label_beta_kubernetes_io_instance_type) (instance_mem_price{job=\"kubernetes-cost-report\",
|
||||
label_eks_amazonaws_com_capacity_type=\"ON_DEMAND\"})\n)"
|
||||
record: capacity_instance_node_resource:kube_node_status_shared_instance_mem_price:on_demand_shared_mem_cost
|
||||
@@ -0,0 +1,52 @@
|
||||
groups:
|
||||
- name: MongoDB Alerts
|
||||
rules:
|
||||
- alert: mongodb_Down
|
||||
expr: mongodb_up{job="Mongodb-Exporter"} !=1
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Mongo node is down(instance {{ $labels.instance }})"
|
||||
description: "MongoDB Node :{{ $labels.instance }} down"
|
||||
|
||||
|
||||
- alert: mongodb_ReplicaLag
|
||||
expr: mongodb_mongod_replset_member_replication_lag{job="Mongodb-Exporter"} > 20
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MongoDB replication lag for (instance {{ $labels.instance }})"
|
||||
description: "MongoDB replication lag for {{ $labels.instance }} is :{{ $value }}"
|
||||
|
||||
|
||||
- alert: mongodb_CurrentConnection
|
||||
expr: mongodb_connections{state="current",job="Mongodb-Exporter"} > 750
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MongoDB current connection for (instance {{ $labels.instance }}) is high."
|
||||
description: "MongoDB Current Connections for {{ $labels.instance }} is :{{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: mongodb_Members
|
||||
expr: mongodb_mongod_replset_number_of_members{job="Mongodb-Exporter"} < 3
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MongoDB replica member count (instance {{ $labels.instance }})"
|
||||
description: "MongoDB replica members are less than 3 for the service"
|
||||
|
||||
|
||||
- alert: mongodb_UsedMemory_Invoice
|
||||
expr: (mongodb_memory{type="resident"} / 1024 ) > 75
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MongoDB UsedMemory is High (instance {{ $labels.instance }})"
|
||||
description: "MongoDB UsedMemory for {{ $labels.instance }} is :{{ $value }}"
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
groups:
|
||||
- name: Process_Alert
|
||||
rules:
|
||||
- alert: bluedart-shipment-queue-processor
|
||||
expr: bluedart_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "bluedart-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
description: "bluedart-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
# - alert: delhivery-shipment-queue-processor
|
||||
# expr: delhivery_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
# for: 5s
|
||||
# labels:
|
||||
# severity: critical
|
||||
#annotations:
|
||||
# summary: "delhivery-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
# description: "delhivery-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: xpressbees-shipment-queue-processor
|
||||
expr: xpressbees_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "xpressbees-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
description: "xpressbees-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: track-shipment-delay-processor
|
||||
expr: track_shipment_delay_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "track-shipment-delay-processor process is down {{ $labels.host }}"
|
||||
description: "track-shipment-delay-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: jusda-shipment-queue-processor
|
||||
expr: jusda_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "jusda-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
description: "jusda-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: ecom-shipment-queue-processor
|
||||
expr: ecom_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "ecom-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
description: "ecom-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: shadowfax-shipment-queue-processor
|
||||
expr: 'shadowfax_shipment_queue_processor_value{host="bac-p-manifest-worker-01b-cicd",job="telegraf-exporter",name="bac-p-worker-manifest-01b"} < 1'
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "shadowfax-shipment-queue-processor process is down {{ $labels.host }}"
|
||||
description: "shadowfax-shipment-queue-processor process is down {{ $labels.host }}. Please check."
|
||||
|
||||
#- alert: order_event_worker
|
||||
# expr: order_event_worker_value{host="bac-p-supplyapi-01b",job="telegraf-exporter",name="bac-p-supplyapi-01b"} < 4
|
||||
# for: 5s
|
||||
# labels:
|
||||
# severity: critical
|
||||
# annotations:
|
||||
# summary: "order_event_worker process is less than 9 {{ $labels.host }}"
|
||||
# description: "order_event_worker process is less than 9 {{ $labels.host }}. Please check."
|
||||
|
||||
- alert: user_payment_done_worker
|
||||
expr: user_payment_done_worker_value{host="bac-p-supplyapi-01b",job="telegraf-exporter",name="bac-p-supplyapi-01b"} < 2
|
||||
for: 5s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "user_payment_done_worker process is less than two {{ $labels.host }}"
|
||||
description: "user_payment_done_worker process is less than two {{ $labels.host }}. Please check."
|
||||
@@ -0,0 +1,961 @@
|
||||
groups:
|
||||
- name: RDS-Alerts
|
||||
rules:
|
||||
- alert: cpuUsage70%
|
||||
expr: aws_rds_cpuutilization_maximum{job="aws_rds", dbinstance_identifier !~ "bac-p-payments-slave-02|data-p-emr-airflow-prism-new-upgraded|data-p-metabase-presto-ext-rds|bac-p-live-commerce|bac-p-data-platform-presto-gateway|supply-prod-admin-slave|bac-p-cis|data-p-metabase-rds|supply-prod"} offset 10m > 70
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "CPU utilization of the RDS {{ $labels.dbinstance_identifier }} is greater than 70"
|
||||
description: "CPU Utilization of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: cpuUsage50%
|
||||
expr: aws_rds_cpuutilization_maximum{job="aws_rds", dbinstance_identifier =~ "supply-prod"} offset 10m > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "CPU utilization of the RDS {{ $labels.dbinstance_identifier }} is greater than 50"
|
||||
description: "CPU Utilization of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: cpuUsage85%
|
||||
expr: aws_rds_cpuutilization_maximum{job="aws_rds", dbinstance_identifier =~ "bac-p-data-platform-presto-gateway|data-p-metabase-presto-ext-rds"} offset 10m > 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "CPU utilization of the RDS {{ $labels.dbinstance_identifier }} is greater than 85"
|
||||
description: "CPU Utilization of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
- alert: cpuUsage80%
|
||||
expr: aws_rds_cpuutilization_maximum{job="aws_rds", dbinstance_identifier =~ "data-p-metabase-rds|bac-p-data-platform-presto-gateway|bac-p-live-commerce"} offset 10m > 80
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "CPU utilization of the RDS {{ $labels.dbinstance_identifier }} is greater than 80"
|
||||
description: "CPU Utilization of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }}"
|
||||
|
||||
|
||||
- alert: cpuUsage90%
|
||||
expr: aws_rds_cpuutilization_maximum{job="aws_rds", dbinstance_identifier =~ "supply-prod-admin-slave|bac-p-cis|bac-p-payments-slave-02"} offset 10m > 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "CPU utilization of the RDS {{ $labels.dbinstance_identifier }} is greater than 90"
|
||||
description: "CPU Utilization of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }}"
|
||||
|
||||
|
||||
|
||||
################################## RDS FreeStorage #################################
|
||||
|
||||
|
||||
- alert: freeStorage_300G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier !~ 'data-p-metabase-presto-rds-perf-cmr-1909|metabase-presto-rds-perf-cmr-1909|bac-p-ab-service|bac-p-content-new|bac-p-price-recommendation-new|bac-p-price-recommendation-data-slave-new|bac-p-grafana-dp|bac-p-price-data-slave-new|bac-p-tracking-new|bac-p-tracking-new-slave|bac-preprod-ab-rds|bac-p-whatsapp-client-eks|bac-p-content-data-slave-new|bac-p-eks-presto-resource-group|data-p-metabase-presto-rds-perf-cmr-1208|data-p-metabase-presto-ext-rds|bac-p-transact-slave-02|bac-p-supplier-return-assurance |bac-p-supplier-return-assurance-slave|bac-p-meesholink|bac-p-price-new|bac-p-supplier-detail-new|bac-p-ab-service-pre-prod|bac-p-inventory-data-slave-new|bac-p-inventory-new|bac-p-transact|bac-p-sx-support.*|data-p-metabase-presto-nrt-rds|data-p-metabase-presto-rds-perf-dataops-756|data-p-rds-airflow-etl-2-4|data-p-superset-dashboard|bac-p-supplier-incentives-data.*|bac-p-supplier-detail-data.*|bac-p-producer-fulfilment.*|bac-p-growth.*|bac-p-farmiso-gcp.*|bac-p-display-ads.*|bac-p-di-metastore|bac-p-app-dbs-metabase|bac-p-farmiso-relay-new|bac-p-supplier-optin-slave|bac-p-fraud-supplier-platform|bac-p-fraud-supplier-platform-slave|bac-p-supplier-optin-data-slave|bac-p-relay|bac-p-offer-platform-data-slave|bac-p-relay-data-slave1|bac-p-supplier-optin|bac-p-loyalty-earn-engine|bac-p-loyalty-earn-engine-slave.*|bac-p-loyalty-earn-engine-data-slave|bac-p-offer-platform|bac-p-offer-platform-slave|bac-p-datahub.*|bac-p-supplier-data-platform|bac-p-taxonomy-new-data-slave|bac-p-vmextractor.*|bac-p-live-commerce.*|data-p-sb-presto-rds-slave.*|bac-p-listing-go-live.*|bac-p-review|bac-p-shipment-service.*|data-p-hive-metastore-slave.*|bac-p-ct-qct-v2.*|bac-p-payments.*|bac-p-address.*|bac-p-data-platform-presto-gateway.*|bac-p-community.*|bac-p-community-data-slave.*|bac-p-pow.*|bac-p-supplier-orders-data-slave.*|bac-p-supplier-business-partner.*|data-p-sb-presto-rds|bac-p-supplier-payment-dummy-new|bac-p-order-service.*|bac-p-review-slave-02|data-p-metabase-rds-temp|data-p-metabase-presto-rds|bac-p-wallet|data-p-emr-airflow-prism|data-p-emr-airflow-prism-new|bac-p-payments-slave-01|bac-p-wallet-slave-01|bac-p-supplier-ums-slave|bac-p-cis-slave-01|bac-p-review-data-slave-01|bac-p-cis|bac-p-supplier-ums|supply-prod|bac-p-whatsapp-client|bac-p-cdp-slave-01|bac-p-cdp|bac-p-tracking-slave-01|front-p-catalog|bac-p-supply-order-slave|bac-p-tracking-01|bac-p-supply-payment-slave|supply-prod-admin-slave|supply-prod-heevo-slave-01|data-p-hive-metastore|front-p-webviews|bac-p-supplier-orders-slave|data-p-metabase-rds|data-p-metabase-main-rds|hive-metastore|data-p-emr-airflow|data-p-metabase-presto-rds-restored-perf-dataops-195|bac-p-supplier-incentives|bac-p-supplier-incentives-slave.*|bac-p-supplier-orders|bac-p-merchandising.*|devops-p-grafana|devops-prod-vault|bac-p-fraud-supplier-platform-data-slave'} offset 10m / 1073741824 < 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 300G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_200G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-sx-support.*|bac-p-transact-slave-02|bac-p-relay|bac-p-address*|supply-prod|bac-p-payments.*|data-p-emr-airflow-prism-new|bac-p-review|supply-prod-admin-slave|bac-p-supplier-orders|bac-p-taxonomy-new-data-slave'} offset 10m / 1073741824 < 200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 200G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_70G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-content-new|bac-p-content-data-slave-new|data-p-sb-presto-rds|bac-p-whatsapp-client|bac-p-supplier-ums-slave|bac-p-data-platform-presto-gateway|bac-p-supplier-ums'} offset 10m / 1073741824 < 70
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 70G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
- alert: freeStorage_50G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-pow.*|bac-p-farmiso-gcp.*|bac-p-grafana-dp|bac-p-price-data-slave-new|bac-p-supplier-optin-data-slave|bac-p-supplier-optin-slave|bac-p-data-platform-presto-gateway|bac-p-loyalty-earn-engine-slave.*|bac-p-loyalty-earn-engine-data-slave|data-p-sb-presto-rds-slave|bac-p-shipment-service|data-p-hive-metastore-slave|bac-p-data-platform-presto-gateway|bac-p-loyalty-earn-engine||bac-p-supplier-incentives-slave.*|bac-p-producer-fulfilment.*'} offset 10m / 1073741824 < 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 50G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
- alert: freeStorage_100G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-supplier-incentives-data.*|bac-p-price-new|bac-p-supplier-detail-data.*|bac-p-inventory-data-slave-new|bac-p-inventory-new|bac-p-tracking-new|bac-p-tracking-new-slave|front-p-catalog|bac-p-supplier-business-partner|bac-p-cdp|data-p-emr-airflow-prism|bac-p-cdp-slave-01|bac-p-community|bac-p-community-data-slave.*|data-p-metabase-presto-rds-restored-perf-dataops-195|bac-p-relay-data-slave1|bac-p-fraud-supplier-platform-slave|bac-p-fraud-supplier-platform'} offset 10m / 1073741824 < 100
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 100G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_150G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'data-p-metabase-presto-rds-perf-cmr-1909|bac-p-offer-platform-data-slave|bac-p-order-service.*|bac-p-review-slave-02|bac-p-supply-payment-slave|bac-p-supply-order-slave|bac-p-cis-slave-01|supply-prod-heevo-slave-01|bac-p-cis|bac-p-supplier-incentives|bac-p-review-data-slave-01|bac-p-fraud-supplier-platform-data-slave'} offset 10m / 1073741824 < 150
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 150G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_130G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-growth.*'} offset 10m / 1073741824 < 130
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 130G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_140G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-supplier-orders-data-slave|bac-p-transact|bac-p-supplier-orders-slave'} offset 10m / 1073741824 < 140
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 140G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_145G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-wallet|bac-p-wallet-slave-01'} offset 10m / 1073741824 < 145
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 145G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
- alert: freeStorage_180G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'front-p-webviews'} offset 10m / 1073741824 < 180
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 180G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_75G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'data-p-metabase-rds-temp|bac-p-supplier-detail-new|bac-p-offer-platform|data-p-hive-metastore|bac-p-tracking-slave-01|bac-p-tracking-01|data-p-metabase-rds|data-p-metabase-main-rds|hive-metastore|data-p-emr-airflow|bac-p-offer-platform-slave|devops-p-grafana|devops-prod-vault'} offset 10m / 1073741824 < 75
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 75G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
- alert: freeStorage_5G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-supplier-data-platform'} offset 10m / 1073741824 < 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 5G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
- alert: freeStorage_40G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'metabase-presto-rds-perf-cmr-1909|data-p-superset-dashboard'} offset 10m / 1073741824 < 40
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 40G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
- alert: freeStorage_35G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'data-p-metabase-presto-rds|data-p-metabase-presto-ext-rds|data-p-metabase-presto-nrt-rds|data-p-metabase-presto-rds-perf-dataops-756'} offset 10m / 1073741824 <35
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 35G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
|
||||
- alert: freeStorage_10G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-display-ads.*|bac-p-di-metastore|bac-p-ct-qct-v2|bac-p-vmextractor.*'} offset 10m / 1073741824 < 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 10G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
- alert: freeStorage_15G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-preprod-ab-rds'} offset 10m / 1073741824 < 15
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 15G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} GB"
|
||||
|
||||
|
||||
- alert: freeStorage_30G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-ab-service|bac-p-eks-presto-resource-group|bac-p-price-recommendation-data-slave-new|bac-p-price-recommendation-new|bac-p-supplier-optin|bac-p-merchandising.*|bac-p-listing-go-live.*|bac-p-datahub.*'} offset 10m / 1073741824 < 30
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 30G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
- alert: freeStorage_25G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-farmiso-relay-new|bac-p-supplier-return-assurance|bac-p-supplier-return-assurance-slave'} offset 10m / 1073741824 < 25
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 25G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
- alert: freeStorage_20G
|
||||
expr: aws_rds_free_storage_space_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-live-commerce|bac-p-meesholink|bac-p-app-dbs-metabase|bac-p-ab-service-pre-prod|bac-p-whatsapp-client-eks|data-p-rds-airflow-etl-2-4'} offset 10m / 1073741824 < 20
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is less than 20G"
|
||||
description: "FreeStorage of the RDS {{ $labels.dbinstance_identifier }} is {{ $value }} bytes"
|
||||
|
||||
|
||||
|
||||
################################## RDS connections #################################
|
||||
|
||||
|
||||
- alert: RDS_Connections_150+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier !~ "bac-p-pow-rds-new|front-p-catalog-new|bac-p-supply-payment-new-slave|bac-p-post-order-new-slave|bac-p-feed-slave-01|bac-p-sx-support.*|bac-p-feed-data-slave|bac-p-taxonomy-new-slave2|bac-p-taxonomy-new-data-slave|bac-p-taxonomy-new-slave04|bac-p-supplier-data-platform|bac-p-fraud-supplier-platform|bac-p-order-service-new-data-slave-01|bac-p-post-order-slave|data-p-sb-presto-rds|bac-p-loyalty-earn-engine|bac-p-order-service-new-slave-01|bac-p-communications|bac-p-feed-slave-02|bac-p-taxonomy-new-slave03|bac-p-data-platform-presto-gateway|bac-p-supplier-optin|bac-p-support|bac-p-wallet-new-rds|bac-p-wallet-v2|bac-p-price.*|bac-p-offer-platform|bac-p-taxonomy-new-slave01|bac-p-merchandising-data-slave1|bac-p-order-service-new|bac-p-taxonomy-new-slave02|bac-p-merchandising|bac-p-merchandising-data-slave|bac-p-taxonomy-new|bac-p-offer-service|bac-p-advertisement-new.*|bac-p-shipment-service.*|bac-p-supplier-detail.*|bac-p-ab-service.*|bac-p-inventory.*|bac-p-address.*|bac-p-cis|bac-p-community.*|bac-p-content|bac-p-ct-qct-v2|bac-p-discovery|bac-p-discovery-slave|bac-p-farmiso|bac-p-feed|bac-p-recommendation-service|bac-p-order-service|bac-p-ors.*|bac-p-payments.*|bac-p-pow-rds|bac-p-referral-earnings|bac-p-review.*|bac-p-supplier-ums|bac-p-taxonomy|bac-p-tracking-.*|bac-p-tracking-slave-01|bac-p-transact|bac-p-ums|bac-p-user-profile.*|bac-p-wallet|bac-p-whatsapp-client|data-p-emr-airflow-prism|data-p-metabase-rds|data-p-rds-airflow-etl|front-p-catalog|front-p-webviews|supply-prod|supply-prod-heevo-slave-01|bac-p-supply-order-slave|bac-p-supply-payment-slave|bac-p-taxonomy-slave2|bac-p-growth|bac-p-live-commerce|data-p-emr-airflow-prism-new|bac-p-wallet-new-updated-rds"} offset 10m > 150
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (150+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_200+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-ct-qct-v2|bac-p-ors|bac-p-ors-slave|bac-p-support|bac-p-data-platform-presto-gateway|bac-p-tracking-slave-01|bac-p-whatsapp-client|bac-p-recommendation-service"} offset 10m > 200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (200+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_1500+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-farmiso"} offset 10m > 1500
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1500+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_1200+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-merchandising|bac-p-merchandising-data-slave1"} offset 10m > 1200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1200+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_250+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-pow-rds-new|data-p-metabase-rds|bac-p-supplier-detail|bac-p-offer-service"} offset 10m > 250
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (250+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_400+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-pow-rds|bac-p-fraud-supplier-platform|bac-p-feed-data-slave|bac-p-order-service-new-slave-01"} offset 10m > 400
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (400+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_350+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-loyalty-earn-engine"} offset 10m > 350
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (350+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_450+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-supplier-ums"} offset 10m > 450
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (450+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_550+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-order-service-new"} offset 10m > 550
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (550+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_900+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-supply-payment-new-slave|bac-p-post-order-new-slave"} offset 10m > 900
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (900+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_300+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-sx-support.*|front-p-catalog-new|bac-p-supplier-data-platform|bac-p-ab-service-slave-01|data-p-emr-airflow-prism-new|bac-p-communications|bac-p-feed-slave-02|data-p-emr-airflow-prism|data-p-rds-airflow-etl|bac-p-merchandising-data-slave"} offset 10m > 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (300+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_500+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-feed-slave-01|bac-p-address|bac-p-supplier-optin|bac-p-address-slave|bac-p-community-slave-01|bac-p-order-service-new-data-slave-01|bac-p-referral-earnings|bac-p-tracking-01|front-p-webviews"} offset 10m > 500
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (500+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_600+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-inventory|data-p-sb-presto-rds"} offset 10m > 600
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (600+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_1800+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-live-commerce"} offset 10m > 1800
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1800+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_700+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "supply-prod-heevo-slave-01"} offset 10m > 700
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (700+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_1100+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-wallet-new-rds"} offset 10m > 1100
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1100+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_2000+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-price|bac-p-payments-slave-01"} offset 10m > 2000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_1000+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-cis|bac-p-taxonomy-new|bac-p-offer-platform|bac-p-taxonomy-new-slave04|bac-p-advertisement-new|bac-p-wallet-v2|bac-p-order-service|bac-p-review-slave-02|bac-p-growth|bac-p-post-order-slave"} offset 10m > 1000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_1600+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "front-p-catalog"} offset 10m > 1600
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1600+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
- alert: RDS_Connections_1300+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-taxonomy-new-slave03|bac-p-taxonomy-new-slave02|bac-p-taxonomy-new-slave2|bac-p-discovery"} offset 10m > 1300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (1300+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_2200+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-discovery-slave"} offset 10m > 2200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_2000+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-price|bac-p-community"} offset 10m > 2000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_2100+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-feed"} offset 10m > 2100
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2100+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_2200+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-shipment-service"} offset 10m > 2200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2200+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_2700+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-wallet"} offset 10m > 2700
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2700+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_2500+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-ums|supply-prod|bac-p-taxonomy-new-data-slave|bac-p-wallet-new-updated-rds"} offset 10m > 2500
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (2500+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_3000+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-ab-service|bac-p-taxonomy-slave2"} offset 10m > 3000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (3000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_3200+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-review"} offset 10m > 3200
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (3200+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_3500+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-taxonomy|bac-p-transact|bac-p-taxonomy-new-slave01"} offset 10m > 3500
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (3500+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: RDS_Connections_3800+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-user-profile-slave|bac-p-supply-order-slave"} offset 10m > 3800
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (3800+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_5000+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-supply-payment-slave|bac-p-payments"} offset 10m > 5000
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (5000+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_4300+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-user-profile"} offset 10m > 4300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (4300+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: RDS_Connections_7500+
|
||||
expr: aws_rds_database_connections_maximum {job="aws_rds",dbinstance_identifier =~ "bac-p-content"} offset 10m > 7500
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Number of connections of the RDS {{ $labels.dbinstance_identifier }} High (7500+)."
|
||||
description: "High Number of connections in the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
################################## RDS IOPS #################################
|
||||
|
||||
- alert: IOPS_700+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"data-p-metabase-rds|bac-p-data-platform-presto-gateway|data-p-hive-metastore"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"data-p-metabase-rds|bac-p-data-platform-presto-gateway|data-p-hive-metastore"} offset 10m ) > 700
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 700."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_2500+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-ab-service|bac-p-ab-service-slave-01|bac-p-review-slave-01|data-p-emr-airflow"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-ab-service|bac-p-ab-service-slave-01|bac-p-review-slave-01|data-p-emr-airflow"} offset 10m ) > 2500
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 2500."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_4000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-cis"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-cis"} offset 10m ) > 4000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 4000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_12000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-payments-slave-01"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-payments-slave-01"} offset 10m ) > 12000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 12000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_5000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~""} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~""} offset 10m ) > 5000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 5000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_4000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-cis-slave-01"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-cis-slave-01"} offset 10m ) > 4000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 4000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_12000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-payments"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-payments"} offset 10m ) > 12000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 12000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_6000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-tracking-01"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-tracking-01"} offset 10m ) > 6000
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 6000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_7000+
|
||||
expr: ( aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-tracking-slave-01|bac-p-content-slv-01|front-p-catalog|front-p-catalog-slave02"} offset 10m + aws_rds_read_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-tracking-slave-01|bac-p-content-slv-01|front-p-catalog"} offset 10m ) > 7000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 7000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_8000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"front-p-catalog-slave02"} offset 10m + aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier=~"front-p-catalog-slave02"} offset 10m ) > 8000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 8000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_15000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-wallet"} offset 10m + aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier=~"bac-p-wallet"} offset 10m ) > 15000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 15000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_9000+
|
||||
expr: ( aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-content|bac-p-wallet-slave-01"} offset 10m + aws_rds_read_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-content|bac-p-wallet-slave-01"} offset 10m ) > 9000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 9000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_9600+
|
||||
expr: ( aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-ums"} offset 10m + aws_rds_read_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-ums"} offset 10m ) > 9600
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 9000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_3500+
|
||||
expr: ( aws_rds_write_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-review"} offset 10m + aws_rds_read_iops_average{job="aws_rds",dbinstance_identifier =~ "bac-p-review"} offset 10m ) > 3500
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 3500."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_12000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-ums-slave|supply-prod-admin-slave"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-ums-slave|supply-prod-admin-slave"} offset 10m ) > 12000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 12000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_28000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-supply-order-slave"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-supply-order-slave"} offset 10m ) > 28000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 28000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: IOPS_45000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"bac-p-supply-payment-slave"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"bac-p-supply-payment-slave"} offset 10m ) > 45000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 45000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_13000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"supply-prod-heevo-slave-01"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"supply-prod-heevo-slave-01"} offset 10m ) > 13000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 13000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: IOPS_14000+
|
||||
expr: (aws_rds_read_iops_average{job="aws_rds", dbinstance_identifier =~"supply-prod"} offset 10m + aws_rds_write_iops_average{job="aws_rds", dbinstance_identifier=~"supply-prod"} offset 10m ) > 14000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "IOPS of the RDS {{ $labels.dbinstance_identifier }} is greater than 14000."
|
||||
description: "IOPS of the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
|
||||
################################## RDS read replica #################################
|
||||
|
||||
- alert: Replication_Error
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds"} offset 10m == -1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication error for RDS {{ $labels.dbinstance_identifier }}."
|
||||
description: "Replication error for RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: ReplicaLagAlert_5mins
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds", dbinstance_identifier !~ 'bac-p-supply-payment-slave|supply-prod-heevo-slave-01|bac-p-data-platform-presto-gateway|bac-p-wallet-slave-01|front-p-catalog-slave02|bac-p-ums-slave' } offset 10m > 600
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication lag of the RDS {{ $labels.dbinstance_identifier }} is greater than 300 seconds."
|
||||
description: "Replication Lag for the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
- alert: ReplicaAlert_10mins
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds", dbinstance_identifier =~ 'supply-prod-heevo-slave-01'} offset 10m > 1000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication lag of the RDS {{ $labels.dbinstance_identifier }} is greater than 1000 seconds."
|
||||
description: "Replication Lag for the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: ReplicaAlert_20mins
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-ums-slave'} offset 10m > 2000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication lag of the RDS {{ $labels.dbinstance_identifier }} is greater than 2000 seconds."
|
||||
description: "Replication Lag for the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
- alert: ReplicaAlert_60mins
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds", dbinstance_identifier =~ 'front-p-catalog-slave02'} offset 10m > 3600
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication lag of the RDS {{ $labels.dbinstance_identifier }} is greater than 3600 seconds."
|
||||
description: "Replication Lag for the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: ReplicaAlert_30mins
|
||||
expr: aws_rds_replica_lag_average{job="aws_rds", dbinstance_identifier =~ 'bac-p-wallet-slave-01'} offset 10m > 2000
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS RDS
|
||||
annotations:
|
||||
summary: "Replication lag of the RDS {{ $labels.dbinstance_identifier }} is greater than 2000 seconds."
|
||||
description: "Replication Lag for the RDS {{ $labels.dbinstance_identifier }} : {{ $value }}."
|
||||
@@ -0,0 +1,340 @@
|
||||
groups:
|
||||
- name: Redis-Alerts
|
||||
rules:
|
||||
- alert: FreeableMemory-For-t3.medium
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-community.*|.*bac-p-farmiso.*|.*front-p-pow-int.*"} offset 10m / 1073741824 < 1
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 1GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-bulletin.*|bac-p-campaign-performance.*|bac-p-communicator-new.*|bac-p-datascience-relay.*|bac-p-feed-cluster.*|bac-p-manifest.*|bac-p-manifest-new.*|bac-p-ofs.*|bac-p-ofs-queue.*|bac-p-order-service.*|bac-p-vision-redis.*|bac-p-pricing-new.*|bac-p-supplier-ums.*|front-p-pow.*"} offset 10m / 1073741824 < 5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 5GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-ctqct.*"} offset 10m / 1073741824 < 1.2
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 1.2GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.xLarge
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-advertisement-serving.*|bac-p-audience-new.*|bac-p-bulletin-new.*|bac-p-cdp-cluster.*|bac-p-inventory.*|bac-p-merchandising-cluster.*|bac-p-notification-store.*|bac-p-payments.*|bac-p-qwest-cluster-sf.*|bac-p-qwest-cluster-text-search.*|bac-p-qwest-cluster-vs.*|bac-p-qwest-sq.*|bac-p-search-preprocessor.*|bac-p-supply.*|bac-p-ums-clustered-users-profile.*|data-science-p-fy-exp.*"} offset 10m / 1073741824 < 8
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 8GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.2x.Large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-cis.*|bac-p-ct-qct-v2-newer.*|bac-p-datascience-relay-2.*|bac-p-organic-fy.*|bac-p-qwest-celery-text-search.*|bac-p-qwest-celery-text-search-2.*|bac-p-supplier-store-cluster.*|bac-p-taxonomy-monster.*|bac-p-transact.*|data-science-clp-exp.*"} offset 10m / 1073741824 < 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 10GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.4x.Large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-advertisement-views.*|bac-p-cart.*|bac-p-growth.*|bac-p-journey-new.*|bac-p-qwest-celery.*|bac-p-qwest-celery-vs-1.*|bac-p-qwest-cluster.*|bac-p-review-new.*"} offset 10m / 1073741824 < 14
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 14GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.8x.Large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-review.*"} offset 10m / 1073741824 < 18
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 18GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-r5-and-r6g.12x.Large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-review.*"} offset 10m / 1073741824 < 25
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 25GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-m5.large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-mini-url.*|bac-p-qwest-celery-sf.*|data-p-airflow.*"} offset 10m / 1073741824 < 3
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 3GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-m5.large
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-opp.*"} offset 10m / 1073741824 < 2.5
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 2.5GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: FreeableMemory-For-m5.xlarge
|
||||
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-deduplication-search.*|front-p-redis.*"} offset 10m / 1073741824 < 4
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 4GB"
|
||||
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: EngineCPUUtilization
|
||||
expr: aws_elasticache_engine_cpuutilization_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-journey-new-001"} offset 10m >= 70
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "EngineCPUUtilization in Redis {{ $labels.cache_cluster_id }} is greater than 70"
|
||||
description: "EngineCPUUtilization in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: DatabaseMemoryUsagePercentage
|
||||
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id !~ "bac-p-organic-fy.*|bac-p-manifest-002|bac-p-manifest-001|bac-p-datascience-relay-2.*|bac-p-ctqct.*|bac-p-review-new-001|bac-p-review-new-002"} offset 10m >= 70
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 70"
|
||||
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: DatabaseMemoryUsagePercentage
|
||||
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-datascience-relay-2.*|bac-p-ctqct.*"} offset 10m >= 85
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 85"
|
||||
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: DatabaseMemoryUsagePercentage
|
||||
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-review-new-001|bac-p-manifest-001|bac-p-manifest-002|bac-p-review-new-002"} offset 10m >= 80
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 80"
|
||||
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: DatabaseMemoryUsagePercentage
|
||||
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-organic-fy.*"} offset 10m >= 90
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 90"
|
||||
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: SwapUsage
|
||||
expr: aws_elasticache_swap_usage_average{job="aws_elasticache"} offset 10m / 1000000 > 50
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "SwapUsage in Redis {{ $labels.cache_cluster_id }} is greater than 50MB"
|
||||
description: "SwapUsage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: ReplicationLag
|
||||
expr: aws_elasticache_replication_lag_average{job="aws_elasticache"} offset 10m > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "ReplicationLag in Redis {{ $labels.cache_cluster_id }} is greater than 10sec"
|
||||
description: "ReplicationLag in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_70MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id !~ ".*bac-p-supplier-store-cluster.*|.*bac-p-manifest-serving.*|.*bac-p-farmiso.*|.*bac-p-organic-fy.*|.*bac-p-ums-users-profile-new.*|.*data-science-clp-exp.*|.*bac-p-merchandising.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*bac-p-feed.*|.*bac-p-notification-store.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*|.*bac-p-qwest-celery.*|.*bac-p-supplier-store-new.*|.*bac-p-ums-users-profile.*|bac-p-qwest-001|bac-p-qwest-002"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id !~ ".*bac-p-edge-proxy.*|.*bac-p-supplier-store-cluster.*|.*bac-p-manifest-serving.*|.*bac-p-farmiso.*|.*bac-p-organic-fy.*|.*bac-p-ums-users-profile-new.*|.*data-science-clp-exp.*|.*bac-p-merchandising.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*bac-p-feed.*|.*bac-p-notification-store.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*|.*bac-p-qwest-celery.*|.*bac-p-supplier-store-new.*|.*bac-p-ums-users-profile.*|bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000 ) > 70
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 70MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_110MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-manifest-serving.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-manifest-serving.*"} offset 10m / 60000000) > 110
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 110MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_120MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-notification-store.*|.*bac-p-farmiso.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-edge-proxy.*|.*bac-p-farmiso.*|.*bac-p-notification-store.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*platform-p-qwest-2.*|.*data-science-p-fy-exp.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*"} offset 10m / 60000000) > 120
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 120MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_250MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-organic-fy.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-organic-fy.*"} offset 10m / 60000000) > 250
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 250MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_90MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*data-science-clp-exp.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*data-science-clp-exp.*"} offset 10m / 60000000) > 90
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 90MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_156MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile.*"} offset 10m / 60000000) > 156
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 156MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_200MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-merchandising.*|.*bac-p-feed.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-merchandising.*|.*bac-p-feed.*"} offset 10m / 60000000) > 200
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 200MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_525MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-qwest-celery.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-qwest-celery.*"} offset 10m / 60000000) > 525
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 525MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_300MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile-new.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile-new.*"} offset 10m / 60000000) > 300
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 300MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_215MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000) > 215
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 215MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: NByteIn+NByteOut_250MB+
|
||||
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-supplier-store-cluster.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-supplier-store-cluster.*"} offset 10m / 60000000) > 250
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Redis
|
||||
annotations:
|
||||
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 250MB"
|
||||
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
groups:
|
||||
- name: redis-labs-alerts
|
||||
rules:
|
||||
- alert: Throughput Greater Than 15k
|
||||
expr: (bdb_total_req{bdb="10801408"}) offset 10m > 15000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 15000"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: Throughput Greater Than 25k
|
||||
expr: (bdb_total_req{bdb=~"10810299|10810324|10621595|11094293"}) offset 10m > 25000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 25000"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: Throughput Greater Than 50k
|
||||
expr: (bdb_total_req{bdb=~"11029271|10772171"}) offset 10m > 50000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 50000"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: Throughput Greater Than 89k
|
||||
expr: (bdb_total_req{bdb=~"10790599"}) offset 10m > 89000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 89000"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: Throughput Greater Than 200k
|
||||
expr: (bdb_total_req{bdb=~"11029236"}) offset 10m > 200000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 200000"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: bdb_avg_latency_greather_than_1ms_Default
|
||||
expr: (bdb_avg_latency) offset 10m > 0.1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of Cluster {{ $labels.cluster }}, (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 100 Milli Second"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: bdb_avg_latency_greather_than_10ms
|
||||
expr: (bdb_avg_latency{bdb=~"11029271|11094293"}) offset 10m > 0.01
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of Cluster {{ $labels.cluster }}, (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 10 Milli Second"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: bdb_avg_latency_greather_than_50ms
|
||||
expr: (bdb_avg_latency{bdb=~"11029236|10621595|10790599"}) offset 10m > 0.05
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of Cluster {{ $labels.cluster }}, (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 50 Milli Second"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
- alert: bdb_avg_latency_greather_than_100ms
|
||||
expr: (bdb_avg_latency{bdb=~"10772171|10810299|10810324"}) offset 10m > 0.1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of Cluster {{ $labels.cluster }}, (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 100 Milli Second"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: bdb_memory_usage_greaterthan_80%
|
||||
expr: avg by (cluster,instance)((bdb_used_memory{} / bdb_memory_limit{}) )* 100 offset 10m > 80
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Throughput of Cluster {{ $labels.cluster }}, (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 100 Milli Second"
|
||||
description: "Throughput of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: Node Free Memory Less than 2GB
|
||||
expr: (node_free_memory / 1073741824) offset 10m < 2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "node_free_memory of Cluster {{ $labels.cluster}}, (instance {{ $labels.instance }}) and for the node ( Node ID : {{ $labels.node }}) is less than 2GB"
|
||||
description: "node_free_memory of (Node ID : {{ $labels.node }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
- alert: Redis Shard CPU usage percent greater than 70%_Default
|
||||
expr: (redis_process_cpu_usage_percent) offset 10m > 70
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Redis Shard CPU Usage of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 70%"
|
||||
description: "Redis Shard CPU Usage of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
|
||||
|
||||
- alert: redis shard connections greater than 6000_Default
|
||||
expr: (redis_connected_clients) offset 10m > 6000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "redis shard connections of (instance {{ $labels.instance }}) and for the database (bdb ID : {{ $labels.bdb }}) is greater than 6000 connections"
|
||||
description: "redis shard connections of (bdb ID : {{ $labels.bdb }}) ( VALUE = {{ $value }}\n LABELS: {{ $labels }}"
|
||||
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
groups:
|
||||
- name: scylla.rules
|
||||
rules:
|
||||
- record: cql:all_shardrate1m
|
||||
expr: sum(rate(scylla_cql_reads[60s])) by (cluster, dc, instance, shard) + sum(rate(scylla_cql_inserts[60s]) ) by (cluster, dc, instance, shard) + sum( rate(scylla_cql_updates[60s]) ) by (cluster, dc, instance, shard) + sum( rate(scylla_cql_deletes[60s])) by (cluster, dc, instance, shard)
|
||||
- record: cql:all_system_shardrate1m
|
||||
expr: sum(rate(scylla_cql_reads_per_ks{ks="system"}[60s])) by (cluster, dc, instance, shard) + sum(rate(scylla_cql_inserts_per_ks{ks="system"}[60s]) ) by (cluster, dc, instance, shard) + sum( rate(scylla_cql_updates_per_ks{ks="system"}[60s]) ) by (cluster, dc, instance, shard) + sum( rate(scylla_cql_deletes_per_ks{ks="system"}[60s])) by (cluster, dc, instance, shard)
|
||||
- record: cql:local_shardrate1m
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_reads_local_node[60s])) by (cluster, dc, instance, shard) + sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node[60s]) ) by (cluster, dc, instance, shard)
|
||||
- record: cql:all_rate1m
|
||||
expr: sum(cql:all_shardrate1m) by (cluster, dc, instance)
|
||||
- record: cql:non_token_aware
|
||||
expr: (sum(cql:all_rate1m) by (cluster) >bool 100) * clamp_min(1-(sum(cql:local_shardrate1m) by (cluster) / sum(cql:all_rate1m) by (cluster)), 0)
|
||||
- record: cql:non_system_prepared1m
|
||||
expr: clamp_min(sum(rate(scylla_query_processor_statements_prepared[1m])) by (cluster, dc, instance, shard) - cql:all_system_shardrate1m, 0)
|
||||
- record: cql:non_prepared
|
||||
expr: (sum(cql:non_system_prepared1m) by (cluster) >bool 100) * (sum(cql:non_system_prepared1m) by (cluster) / clamp_min(sum(cql:all_rate1m) by (cluster)- sum(cql:all_system_shardrate1m) by (cluster), 0.001))
|
||||
- record: cql:non_paged_no_system1m
|
||||
expr: clamp_min(sum(rate(scylla_cql_unpaged_select_queries[60s])) by (cluster, dc, instance) - sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks="system"}[60s])) by (cluster, dc, instance), 0)
|
||||
- record: cql:non_paged_no_system
|
||||
expr: (sum(cql:non_paged_no_system1m) by (cluster, dc, instance) >bool 100) * sum(cql:non_paged_no_system) by (cluster, dc, instance)/clamp_min(sum(rate(scylla_cql_reads[60s]))by (cluster, dc, instance) - sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks="system"}[60s])) by (cluster, dc, instance), 0.01)
|
||||
- record: cql:non_paged
|
||||
expr: (sum(cql:non_paged_no_system1m) by (cluster) >bool 100) * sum(cql:non_paged_no_system1m) by (cluster)/clamp_min(sum(rate(scylla_cql_reads[60s]))by (cluster) - sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks="system"}[60s])) by (cluster), 0.01)
|
||||
- record: cql:reverse_queries
|
||||
expr: sum(rate(scylla_cql_reverse_queries[60s])) by (cluster)/ sum(rate(scylla_cql_reads[60s])) by (cluster)
|
||||
- record: cql:allow_filtering
|
||||
expr: sum(rate(scylla_cql_filtered_read_requests[60s])) by (cluster)/ sum(rate(scylla_cql_reads[60s])) by (cluster)
|
||||
- record: cql:any_queries
|
||||
expr: sum(rate(scylla_query_processor_queries{consistency_level="ANY"}[60s])) by (cluster) >bool 0
|
||||
- record: cql:all_queries
|
||||
expr: sum(rate(scylla_query_processor_queries{consistency_level="ALL"}[60s])) by (cluster) >bool 0
|
||||
- record: errors:operation_unavailable
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_unavailable[60s])) by (cluster, dc, instance) + sum(rate(scylla_storage_proxy_coordinator_write_unavailable[60s])) by (cluster, dc, instance) + sum(rate(scylla_storage_proxy_coordinator_range_unavailable[60s])) by (cluster, dc, instance)
|
||||
- record: errors:local_failed
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_errors_local_node[60s])) by (cluster, dc, instance) + sum(rate(scylla_storage_proxy_coordinator_write_errors_local_node[60s])) by (cluster, dc, instance)
|
||||
- record: errors:nodes_total
|
||||
expr: errors:local_failed + errors:operation_unavailable
|
||||
- record: manager:repair_done_ts
|
||||
expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="repair"}[60s])) by (cluster) > 0) or on(cluster) manager:repair_done_ts
|
||||
- record: manager:backup_done_ts
|
||||
expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="DONE",type="backup"}[60s])) by (cluster) > 0) or on(cluster) manager:backup_done_ts
|
||||
- record: manager:repair_fail_ts
|
||||
expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="repair"}[60s])) by (cluster) > 0) or on(cluster) manager:repair_fail_ts
|
||||
- record: manager:backup_fail_ts
|
||||
expr: timestamp(sum(changes(scylla_manager_scheduler_run_total{status="ERROR",type="backup"}[60s])) by (cluster) > 0) or on(cluster) manager:backup_fail_ts
|
||||
- record: manager:repair_progress
|
||||
expr: (max(scylla_manager_scheduler_run_indicator{type="repair"}) by (cluster) >bool 0)*((max(scylla_manager_repair_token_ranges_total) by(cluster)<= 0)*0 or on(cluster) (sum(scylla_manager_repair_token_ranges_success>=0) by (cluster) + sum(scylla_manager_repair_token_ranges_error>=0) by (cluster))/sum(scylla_manager_repair_token_ranges_total>=0) by (cluster))
|
||||
- record: scylla_manager_repair_progress
|
||||
expr: sum(manager:repair_progress) by (cluster)
|
||||
labels:
|
||||
level: "1"
|
||||
by: "cluster"
|
||||
- record: manager:backup_progress
|
||||
expr: (max(scylla_manager_scheduler_run_indicator{type="backup"}) by (cluster) >bool 0)*((max(scylla_manager_backup_files_size_bytes) by(cluster)<= 0)*0 or on(cluster) (sum(scylla_manager_backup_files_uploaded_bytes) by (cluster) + sum(scylla_manager_backup_files_skipped_bytes) by (cluster) + sum(scylla_manager_backup_files_failed_bytes)by(cluster))/sum(scylla_manager_backup_files_size_bytes>=0) by (cluster))
|
||||
- record: scylla_manager_backup_progress
|
||||
expr: sum(manager:backup_progress) by (cluster)
|
||||
labels:
|
||||
level: "1"
|
||||
by: "cluster"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name, shard)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, shard)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, instance)
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, dc)
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name)
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name)
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name)
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name)
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencyp99
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, le))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name, shard)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name, shard)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name, instance)
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name, dc)
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_write_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name)
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, scheduling_group_name)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, scheduling_group_name)
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, scheduling_group_name)
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencya
|
||||
expr: sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster,scheduling_group_name)/sum(rate(scylla_storage_proxy_coordinator_read_latency_count{scheduling_group_name=~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, scheduling_group_name)
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: casrlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: casrlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: casrlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: casrlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: caswlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, shard, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
level: "1"
|
||||
- record: caswlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, instance, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "instance"
|
||||
level: "1"
|
||||
- record: caswlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, dc, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "dc"
|
||||
level: "1"
|
||||
- record: caswlatencyp95
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{scheduling_group_name!~"atexit|gossip|mem_compaction|memtable|streaming|background_reclaim|compaction|main|memtable_to_cache"}[60s])) by (cluster, le, scheduling_group_name))
|
||||
labels:
|
||||
by: "cluster"
|
||||
level: "1"
|
||||
- record: all_scheduling_group
|
||||
expr: sum by (cluster, scheduling_group_name) (scylla_storage_proxy_coordinator_write_latency_count + scylla_storage_proxy_coordinator_read_latency_count) > 0
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
groups:
|
||||
- name: scylla.rules
|
||||
rules:
|
||||
- record: wlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: wlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, le,ks,cf))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencyp99ks
|
||||
expr: histogram_quantile(0.99, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, le,ks,cf))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: wlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: wlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_write_latency_bucket{}[60s])) by (cluster, le,ks,cf))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, shard, le,ks,cf))
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, instance, le,ks,cf))
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, dc, le,ks,cf))
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencyp95ks
|
||||
expr: histogram_quantile(0.95, sum(rate(scylla_column_family_read_latency_bucket{}[60s])) by (cluster, le,ks,cf))
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: wlatencyaks
|
||||
expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc, instance, shard,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc, instance, shard,ks,cf)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: wlatencyaks
|
||||
expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc, instance,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc, instance,ks,cf)
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: wlatencyaks
|
||||
expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster, dc,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster, dc,ks,cf)
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: wlatencyaks
|
||||
expr: sum(rate(scylla_column_family_write_latency_sum{}[60s])) by (cluster,ks,cf)/sum(rate(scylla_column_family_write_latency_count{}[60s])) by (cluster,ks,cf)
|
||||
labels:
|
||||
by: "cluster"
|
||||
- record: rlatencyaks
|
||||
expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc, instance, shard,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc, instance, shard,ks,cf)
|
||||
labels:
|
||||
by: "instance,shard"
|
||||
- record: rlatencyaks
|
||||
expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc, instance,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc, instance,ks,cf)
|
||||
labels:
|
||||
by: "instance"
|
||||
- record: rlatencyaks
|
||||
expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster, dc,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster, dc,ks,cf)
|
||||
labels:
|
||||
by: "dc"
|
||||
- record: rlatencyaks
|
||||
expr: sum(rate(scylla_column_family_read_latency_sum{}[60s])) by (cluster,ks,cf)/sum(rate(scylla_column_family_read_latency_count{}[60s])) by (cluster,ks,cf)
|
||||
labels:
|
||||
by: "cluster"
|
||||
+1978
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,567 @@
|
||||
groups:
|
||||
- name: Target-Group-Alerts
|
||||
rules:
|
||||
- alert: UnhealthyHost
|
||||
expr: aws_applicationelb_un_healthy_host_count_maximum{job="aws_applicationelb", target_group !~ ".*data-p-metabase.*|.*sbpresto-metabase-tg.*|.*sbpresto-coordinator-read-new.*|.*presto-cordinator-int-tg.*|.*dataplatform-p-indo-messaging.*|.*tg-sbpresto-metabase-secondary.*|.*tg-sbpresto-metabase-main.*|.*data-p-sbpresto-metabase-co-tg.*|.*data-p-prism-ranger-tg.*|.*data-p-prism-livy-jupyterhub-int.*|.*data-p-prism-knox.*|.*data-p-rancher-oidcclient-tg-new.*|.*data-p-prism-jupyterhub-tg.*"} offset 10m >= 1
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "UnhealthyHost in TG {{ $labels.target_group }} is greater than 1"
|
||||
description: "UnhealthyHost in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_0.5_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*front-p-community.*|.*bac-p-storefront-app-search.*|.*bac-p-journey-tg.*|.*bac-p-storefront-app-secondary/.*|.*bac-p-storefront-int/.*|.*bac-p-storefront-int-primary/.*|.*bac-p-storefront-int-secondary/.*|.*bac-p-storefront-reroute/.*|.*bac-p-journey-consumer-tg.*|.*bac-p-notification-store-tg.*|.*bac-p-notification-consumer-tg.*|.*bac-p-order-state-management-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-edge-proxy|.*bac-p-onboarding.*|.*front-p-supplier-panel-new/.*|.*bac-p-advertisement.*|.*bac-p-login.*|.*bac-p-referral-earnings/.*|.*bac-p-pricing-admin.*|.*bac-p-reseller-tracking-tg.*|.*bac-p-opp-tg.*|.*bac-p-producer-tg.*|.*bac-p-promo-TG.*|.*bac-p-storefront-scaling.*|.*bac-p-supplier-store-tg.*|.*bac-p-supplyapi-int-tg.*|.*bac-p-taxonomy-int-tg.*|.*bac-p-travel.*|.*bac-p-manifest/.*|.*bac-p-manifest-serving.*|.*supply-api-v2-prod.*|.*bac-p-inventory.*|.*bac-p-AB-service.*|.*bac-p-CIS-consumer.*|.*bac-p-review-service.*|.*bac-p-pricing.*|.*bac-p-communicator-service-tg.*|.*front-p-reseller-api-tg.*|.*front-p-reseller-website-tg.*|.*front-p-webviews.*|.*platform-p-prometheus.*|.*platform-p-qwest-tg.*|.*data-p-metastoreapi-tg.*|.*bac-p-reseller-pricing.*|.*bac-p-rule-engine.*|.*bac-p-upload-panel-backend.*|.*bac-p-whatsapp/.*|.*bac-p-community/.*|.*bac-p-merchandising/.*" } offset 10m > 0.5
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 0.5 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_1_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-wallet/.*|.*platform-p-grafana-tg.*|.*front-p-community.*|.*front-p-meesho-com.*|.*front-p-pow-tg.*|.*bac-p-fastrack.*" } offset 10m >= 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 1 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_1_Sec_For_2min
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-storefront-app-events.*" } offset 10m >= 1
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 1 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_0.2_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-datascience-relay-tg.*|.*bac-p-datascience-relay-worker.*|.*bac-p-storefront-app-primary.*" } offset 10m >= 0.2
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 0.2 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_5_Sec_For_2min
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-file-upload-service-tg.*|.*bac-p-oauth-tg.*|.*bac-p-mentorship/.*|.*bac-p-wallet-worker-tg.*|.*bac-p-upload-backend-tg.*"} offset 10m >= 5
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 5 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_7_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-media/.*|.*bac-p-reviewadmin-service.*|.*front-p-supplier-panel-tg.*" } offset 10m >= 7
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 7 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_10sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-edge-proxy-admin.*" } offset 10m >= 10
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 10 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_60_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-invoice-tg.*|.*bac-p-community-bot/.*|.*front-p-supplier-panel-new-node.*" } offset 10m >= 60
|
||||
for: 120s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 60 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_3_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-mentorship-worker.*|.*bac-p-whatsapp-int.*" } offset 10m >= 3
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 3 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TargetResponseTime_Threshold_15_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*front-p-catalog-panel-new-tg.*|.*bac-p-referral-earnings-worker.*" } offset 10m >= 15
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 15 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_25_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-supplier-payment-tg.*" } offset 10m >= 25
|
||||
for: 125s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 25 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_30_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-CIS-ingestion.*|.*front-p-admin-panel-tg.*|.*bac-p-leaderboard-tg.*" } offset 10m >= 30
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 30 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TargetResponseTime_Threshold_35_Sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-payments.*" } offset 10m >= 35
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 35 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TargetResponseTime_Threshold_0.5_Sec_for_300sec
|
||||
expr: aws_applicationelb_target_response_time_average{job="aws_applicationelb", target_group =~ ".*bac-p-manifest-file-upload.*"} offset 10m >= 0.5
|
||||
for: 300s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TargetResponseTime in TG {{ $labels.target_group }} is greater than 0.5 sec"
|
||||
description: "TargetResponseTime in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
####################### Http5xx ######################
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_10+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group !~ ".*offer-platform.*|.*bac-p-qwest-vs-sort-filter-eks.*|.*bac-p-hero-product-web.*|.*bac-p-feed-aggregator-secondary.*|.*bac-p-AB-service.*|.*bac-p-feed-aggregator-search-tg.*|.*bac-p-merchandising.*|.*platform-p-qwest-sq.*|.*supplier-pltfrm-fulfillment-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-storefront.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-storefront-scaling.*|.*bac-p-taxonomy-int.*|.*storefront-app-serving-primary.*|.*storefront-internal-primary.*|.*storefront-internal-secondary.*|.*bac-p-sort-and-filter-sec-tg.*|.*bac-p-sort-and-filter-tg.*|.*bac-p-sort-and-filter-vs-tg.*|.*bac-p-AB-service.*|.*bac-p-CIS-ingestion.*|.*bac-p-ads-rti-enricher-search.*|.*bac-p-ads-rti-es-indexer-search.*|.*bac-p-advertisement.*|.*bac-p-advertisement-admin.*|.*bac-p-advertisement-text-search.*|.*bac-p-agent-supp-aggregator-tg.*|.*bac-p-bulletin-tg.*|.*bac-p-cart-tg.*|.*bac-p-cart-v2-tg.*|.*bac-p-cms-web-tg.*|.*bac-p-communicator-admin.*|.*bac-p-community.*|.*bac-p-community-bot.*|.*bac-p-community-consumer-tg.*|.*bac-p-config-service-tg.*|.*bac-p-datascience-relay-tg.*|.*bac-p-edge-proxy.*|.*bac-p-edge-proxy-farmiso-tg.*|.*bac-p-edge-proxy-int-second-tg.*|.*bac-p-edge-proxy-second-tg.*|.*bac-p-external-payment-gateway.*|.*bac-p-farmiso-tg.*|.*bac-p-fastrack.*|.*bac-p-feed-aggregator-search-tg.*|.*bac-p-feed-aggregator-secondary.*|.*bac-p-feed-aggregator-tg.*|.*bac-p-file-upload-tg.*|.*bac-p-gamification-tg.*|.*bac-p-gatewayservice.*|.*bac-p-growthmarketing.*|.*bac-p-ingestion-new.*|.*bac-p-ingestion-separate.*|.*bac-p-invoice-web.*|.*bac-p-journey-tg.*|.*bac-p-login.*|.*bac-p-logo-tg.*|.*bac-p-loyalty-earn-engine-tg.*|.*bac-p-marketplace-tg.*|.*bac-p-media-supply.*|.*bac-p-oauth-tg.*|.*bac-p-offer-service.*|.*bac-p-order-qwest.*|.*bac-p-order-qwest-index-tg.*|.*bac-p-order-state-management-tg.*|.*bac-p-osm-supply.*|.*bac-p-osm-supply-read.*|.*bac-p-pdf.*|.*bac-p-poa-supply-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-pow-tg.*|.*bac-p-pricing.*|.*bac-p-pricing-admin.*|.*bac-p-qwest-internal-tg.*|.*bac-p-qwest-vs-sort-filter-tg.*|.*bac-p-recommendation-service.*|.*bac-p-reseller-tracking-tg.*|.*bac-p-review-service.*|.*bac-p-search-preprocessor-tg.*|.*bac-p-shipment-service-admin-tg.*|.*bac-p-shipment-service-tg.*|.*bac-p-storefront-app-feed.*|.*bac-p-storefront-app-primary.*|.*bac-p-storefront-app-search.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-primary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-storefront-wishlist-tg.*|.*bac-p-supplier-orchestrator-tg.*|.*bac-p-supplier-payment-web-tg.*|.*bac-p-supplier-sf-tg.*|.*bac-p-supplier-store-admin-tg.*|.*bac-p-supplier-ums-tg.*|.*bac-p-supplyapi-int-tg.*|.*bac-p-support-service-tg.*|.*bac-p-taxonomy-admin.*|.*bac-p-taxonomy-int-tg.*|.*bac-p-trends-scraper-pp-tg.*|.*bac-p-ums.*|.*bac-p-ums-user-profile.*|.*bac-p-upload-backend-tg.*|.*bac-p-vision-tg.*|.*bac-p-widgetaggregator-tg.*|.*data-p-ingestion-web-tg.*|.*data-p-metabase-presto-tg.*|.*data-p-proxydataservice.*|.*front-p-community.*|.*front-p-community-tg.*|.*front-p-farmiso.*|.*front-p-farmiso-admin-panel.*|.*front-p-farmiso-champ-tg.*|.*front-p-farmiso-customer-tg.*|.*front-p-pow-new-tg.*|.*front-p-pow-tg.*|.*front-p-prism-node-ui-tg.*|.*front-p-reseller-logo.*|.*front-p-supplier-panel-new-node.*|.*front-p-supplier-panel-tg.*|.*front-p-supplier-panel-v2-node.*|.*front-p-webviews.*|.*front-p-webvitals.*|.*organic-feed-web-tg.*|.*platform-p-qwest-text-search-tg.*|.*platform-p-qwest-tg.*|.*qwest-sort-filter.*|.*secops-internal.*|.*supplier-platform-ads.*|.*supplier-pltfrm-cataloging-tg.*|.*supplier-pltfrm-container.*|.*supplier-pltfrm-payouts-tg.*|.*supplier-pltfrm-promotions-tg.*|.*supplier-pltfrm-services-tg.*|.*supply-api-v2-prod.*|.*supply-panel-new-node.*|.*supply-supplier-web.*|.*taxation-web.*|.*platform-p-qwest-vs.*"} offset 10m >= 10
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 10"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_1+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-shipment-service-admin-tg.*"} offset 10m >= 1
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 1"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_5+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-file-upload-tg.*|.*bac-p-growthmarketing.*|.*bac-p-order-state-management-tg.*|.*bac-p-reseller-tracking-tg.*|.*bac-p-shipment-service-tg.*|.*bac-p-upload-backend-tg.*"} offset 10m >= 5
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 5"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_15+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-media-supply.*|.*bac-p-order-service-tg.*|.*bac-p-external-payment-gateway.*|.*bac-p-invoice-web.*|.*bac-p-media-supply.*|.*bac-p-taxonomy-admin.*|.*secops-internal.*"} offset 10m >= 15
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 15"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_20+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-ads-rti-enricher-search.*|.*bac-p-hero-product-web.*|.*bac-p-ads-rti-es-indexer-search.*|.*bac-p-advertisement-admin.*|.*bac-p-community-bot.*|.*bac-p-recommendation-service.*|.*bac-p-supplier-store-admin-tg.*|.*bac-p-ums-user-profile.*|.*data-p-proxydataservice.*|.*front-p-supplier-panel-new-node.*|.*supplier-pltfrm-cataloging-tg.*|.*supplier-pltfrm-promotions-tg.*|.*supply-panel-new-node.*"} offset 10m >= 20
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 20"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_25+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-storefront-scaling.*|.*storefront-app-serving-primary.*|.*bac-p-cart-tg.*|.*bac-p-cart-v2-tg.*|.*bac-p-community.*|.*bac-p-community-consumer-tg.*|.*bac-p-gamificationtg.*|.*bac-p-journey-tg.*|.*bac-p-logo-tg.*|.*bac-p-oauth-tg.*|.*bac-p-review-servie.*|.*bac-p-supplierorchestrator-tg.*bac-p-supplier-us-tg.*|.*supplier-platform-ads.*|.*supplier-pltfrm-container.*|.*supplier-pltfrm-services-tg.*"} offset 10m >= 25
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 25"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_35+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-taxonomy-int.*|.*storefront-internal-secondary.*|.*bac-p-agent-supp-aggregator-tg.*|.*bac-p-bulletin-tg.*|.*bac-p-cms-web-tg.*|.*bac-p-config-service-tg.*|.*bac-p-datascience-relay-tg.*|.*bac-p-edge-proxy-farmiso-tg.*|.*bac-p-farmiso-tg.*|.*bac-p-login.*|.*bac-p-offer-service.*|.*bac-p-order-qwest.*|.*bac-p-order-qwest-index-tg.*|.*bac-p-osm-supply.*|.*bac-p-osm-supply-read.*|.*bac-p-pdf.*|.*bac-p-qwest-internal-tg.*|.*bac-p-search-preprocessor-tg.*|.*bac-p-storefront-wishlist-tg.*|.*bac-p-supplier-sf-tg.*|.*bac-p-supplyapi-int-tg.*|.*bac-p-taxonomy-int-tg.*|.*front-p-farmiso.*|.*front-p-farmiso-admin-panel.*|.*front-p-farmiso-champ-tg.*|.*front-p-farmiso-customer-tg.*|.*front-p-prism-node-ui-tg.*|.*front-p-supplier-panel-v2-node.*|.*front-p-webviews.*|.*supplier-pltfrm-payouts-tg.*|.*taxation-web.*"} offset 10m >= 35
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 35"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_50+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*platform-p-qwest-vs.*|.*bac-p-merchandising.*|.*offer-platform.*"} offset 10m >= 50
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 50"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_100+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-storefront.*|.*bac-p-qwest-vs-sort-filter-eks.*|.*storefront-internal-primary.*|.*bac-p-CIS-ingestion.*|.*bac-p-advertisement.*|.*bac-p-advertisement-text-search.*|.*bac-p-communicator-admin.*|.*bac-p-gatewayservice.*|.*bac-p-ingestion-new.*|.*bac-p-loyalty-earn-engine-tg.*|.*bac-p-merchandising.*|.*bac-p-pow-tg.*|.*bac-p-pricing-admin.*|.*bac-p-qwest-vs-sort-filter-tg.*|.*bac-p-storefront-app-primary.*|.*bac-p-vision-tg.*|.*data-p-metabase-presto-tg.*|.*front-p-community.*|.*front-p-community-tg.*|.*front-p-reseller-logo.*|.*front-p-webvitals.*|.*supplier-pltfrm-fulfillment-tg.*|.*organic-feed-web-tg.*|.*platform-p-qwest-vs.*"} offset 10m >= 100
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 100"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_200+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-sort-and-filter-sec-tg.*|.*bac-p-edge-proxy-int-second-tg.*|.*bac-p-edge-proxy-second-tg.*|.*bac-p-fastrack.*|.*bac-p-poa-supply-tg.*|.*bac-p-storefront-app-feed.*|.*bac-p-storefront-app-search.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-primary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-supplier-payment-web-tg.*|.*bac-p-support-service-tg.*|.*data-p-ingestion-web-tg.*|.*front-p-pow-new-tg.*|.*front-p-pow-tg.*|.*front-p-supplier-panel-tg.*|.*platform-p-qwest-text-search-tg.*|.*platform-p-qwest-tg.*"} offset 10m >= 200
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 200"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_300+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-AB-service.*|.*bac-p-ingestion-separate.*|.*bac-p-marketplace-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-trends-scraper-pp-tg.*|.*bac-p-widgetaggregator-tg.*|.*qwest-sort-filter.*|.*supply-api-v2-prod.*"} offset 10m >= 300
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 300"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_500+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-feed-aggregator-secondary.*|.*bac-p-feed-aggregator-tg.*|.*bac-p-feed-aggregator-search-tg.*"} offset 10m >= 500
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 500"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_1000+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-pricing.*|.*bac-p-post-order-aggregator-tg.*"} offset 10m >= 1000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 1000"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_2000+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-ums.*|.*platform-p-qwest-sq.*"} offset 10m >= 2000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 2000"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode5xx_Count_4000+
|
||||
expr: aws_applicationelb_httpcode_target_5_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-edge-proxy.*"} offset 10m >= 4000
|
||||
for: 10s
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode5xx in TG {{ $labels.target_group }} is greater than 4000"
|
||||
description: "TG_HttpCode5xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
####################### Http4xx ######################
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_Count_10+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group !~ ".*bac-p-qwest-product-ts.*|.*bac-p-order-service-tg.*|.*supplier-optin.*|.*bac-p-hero-product-web.*|.*offer-platform.*|.*bac-p-AB-service.*|.*bac-p-manifest.*|.*bac-p-pricing.*|.*bac-p-storefront-app-primary.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-storefront-open.*|.*bac-p-storefront-scaling.*|.*bac-p-supplier-store.*|.*bac-p-taxonomy-int.*|.*bac-p-ums.*|.*platform-p-qwest.*|.*storefront-app-serving-primary.*|.*storefront-internal-secondary.*|.*targetgroup-supply-api-v2-prod.*|.*supplier-pltfrm-promotions-tg.*|.*bac-p-AB-service.*|.*bac-p-CIS-ingestion.*|.*bac-p-OSM-read-tg.*|.*bac-p-address-search.*|.*bac-p-ads-rti-enricher-search.*|.*bac-p-ads-rti-es-indexer-search.*|.*bac-p-advertisement.*|.*bac-p-advertisement-admin.*|.*bac-p-advertisement-text-search.*|.*bac-p-agent-supp-aggregator-tg.*|.*bac-p-cart-tg.*|.*bac-p-cart-v2-tg.*|.*bac-p-cicd-tg.*|.*bac-p-communicator-new-tg.*|.*bac-p-communicator-service-tg.*|.*bac-p-community.*|.*bac-p-config-service-tg.*|.*bac-p-datascience-relay-tg.*|.*bac-p-db-access-new.*|.*bac-p-edge-proxy.*|.*bac-p-edge-proxy-farmiso-tg.*|.*bac-p-edge-proxy-int-second-tg.*|.*bac-p-edge-proxy-second-tg.*|.*bac-p-external-payment-gateway.*|.*bac-p-farmiso-pub.*|.*bac-p-farmiso-tg.*|.*bac-p-farmiso-worker.*|.*bac-p-fastrack.*|.*bac-p-feed-aggregator-search-tg.*|.*bac-p-feed-aggregator-secondary.*|.*bac-p-feed-aggregator-tg.*|.*bac-p-file-upload-tg.*|.*bac-p-gamification-tg.*|.*bac-p-gatewayservice.*|.*bac-p-graylog.*|.*bac-p-ingestion-new.*|.*bac-p-ingestion-separate.*|.*bac-p-inventory-admin-new.*|.*bac-p-jenkins-sonar-tg.*|.*bac-p-location-web.*|.*bac-p-login.*|.*bac-p-loyalty-earn-engine-tg.*|.*bac-p-manifest.*|.*bac-p-marketplace-tg.*|.*bac-p-mba-tg.*|.*bac-p-media-supply.*|.*bac-p-merchandising.*|.*bac-p-merchandising-admin-tg.*|.*bac-p-mini-url-tg.*|.*bac-p-oauth-tg.*|.*bac-p-offer-service.*|.*bac-p-order-service-tg.*|.*bac-p-order-service-v2-tg.*|.*bac-p-order-state-management-tg.*|.*bac-p-osm-supply.*|.*bac-p-osm-supply-read.*|.*bac-p-payment-aggregator-web-tg.*|.*bac-p-poa-supply-admin-tg.*|.*bac-p-poa-supply-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-pow-internal.*|.*bac-p-pow-tg.*|.*bac-p-pricing.*|.*bac-p-pricing-admin.*|.*bac-p-producer-fulfilment.*|.*bac-p-producer-new.*|.*bac-p-promo-TG.*|.*bac-p-recommendation-service.*|.*bac-p-referral-earnings.*|.*bac-p-reseller-tracking-tg.*|.*bac-p-review-service.*|.*bac-p-shipment-service-admin-tg.*|.*bac-p-shipment-service-tg.*|.*bac-p-sort-and-filter-sec-tg.*|.*bac-p-sort-and-filter-tg.*|.*bac-p-sort-and-filter-vs-tg.*|.*bac-p-storefront-app-feed.*|.*bac-p-storefront-app-primary.*|.*bac-p-storefront-app-search.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-int-primary.*|.*bac-p-storefront-int-secondary.*|.*bac-p-storefront-wishlist-tg.*|.*bac-p-supplier-orchestrator-tg.*|.*bac-p-supplier-payment-web-tg.*|.*bac-p-supplier-referrals-tg.*|.*bac-p-supplier-sf-tg.*|.*bac-p-supplier-store-admin-tg.*|.*bac-p-supplier-ums-tg.*|.*bac-p-supplyapi-int-tg.*|.*bac-p-taxonomy-admin.*|.*bac-p-taxonomy-int-tg.*|.*bac-p-ums.*|.*bac-p-ums-fallback-tg.*|.*bac-p-ums-user-profile.*|.*bac-p-upload-backend-tg.*|.*bac-p-widgetaggregator-tg.*|.*data-p-ingestion-web-tg.*|.*data-p-messagingapi-reactive-int.*|.*data-p-metabase-clone-tg.*|.*data-p-metabase-presto-tg.*|.*data-p-metabase-tg.*|.*data-p-metastoreapi-tg.*|.*data-p-prism-internal-new-tg.*|.*data-p-prism-tg.*|.*front-p-blog.*|.*front-p-community.*|.*front-p-community-tg.*|.*front-p-farmiso.*|.*front-p-farmiso-admin-panel.*|.*front-p-farmiso-champ-tg.*|.*front-p-farmiso-customer-tg.*|.*front-p-meesho-com.*|.*front-p-meesho-com-tg.*|.*front-p-pow-new-tg.*|.*front-p-pow-tg.*|.*front-p-prism-node-ui-tg.*|.*front-p-reseller-logo.*|.*front-p-reseller-website-tg.*|.*front-p-supplier-panel-new-node.*|.*front-p-supplier-panel-tg.*|.*front-p-supplier-panel-v2-node.*|.*front-p-webviews.*|.*platform-p-qwest-sq.*|.*platform-p-qwest-text-search-tg.*|.*platform-p-qwest-tg.*|.*supplier-platform-ads-tg.*|.*supplier-pltfrm-cataloging-tg.*|.*supplier-pltfrm-container.*|.*supplier-pltfrm-fulfillment-tg.*|.*supplier-pltfrm-growth-tg.*|.*supplier-pltfrm-payouts-tg.*|.*supplier-pltfrm-promotions-tg.*|.*supplier-pltfrm-services-tg.*|.*supply-api-v2-prod.*|.*supply-panel-new.*|.*supply-panel-new-node.*|.*supply-supplier-web.*|.*taxation-web.*"} offset 10m >= 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 10"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TG_HttpCode4xx_1+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-shipment-service-admin-tg.*"} offset 10m >= 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 1"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_5+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-offer-service.*"} offset 10m >= 5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 5"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_15+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-order-service-tg.*|.*bac-p-manifest.*|.*bac-p-supplier-store.*|.*bac-p-graylog.*|.*bac-p-jenkins-sonar-tg.*|.*bac-p-order-service-tg.*"} offset 10m >= 15
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 15"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_100+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-media-supply.*"} offset 10m >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 100"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_25+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-cart-tg.*|.*bac-p-cicd-tg.*|.*bac-p-external-payment-gateway.*|.*bac-p-file-upload-tg.*|.*bac-p-order-service-v2-tg.*|.*bac-p-promo-TG.*|.*data-p-metabase-clone-tg.*|.*data-p-metabase-tg.*"} offset 10m >= 25
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 25"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_50+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-pricing.*|.*bac-p-hero-product-web.*|.*bac-p-storefront-open.*|.*bac-p-taxonomy-int.*|.*platform-p-qwest.*|.*bac-p-advertisement|.*bac-p-advertisement-text-search.*|.*bac-p-cart-v2-tg.*|.*bac-p-datascience-relay-tg.*|.*bac-p-farmiso-worker.*|.*bac-p-fastrack.*|.*bac-p-feed-aggregator-search-tg.*|.*bac-p-location-web.*|.*bac-p-merchandising-admin-tg.*|.*bac-p-oauth-tg.*|.*bac-p-poa-supply-admin-tg.*|.*bac-p-pow-internal.*|.*bac-p-producer-fulfilment.*|.*bac-p-producer-new.*|.*bac-p-shipment-service-tg.*|.*bac-p-storefront-wishlist-tg.*|.*data-p-metastoreapi-tg.*|.*front-p-farmiso.*|.*front-p-farmiso-admin-panel.*|.*platform-p-qwest-sq.*|.*platform-p-qwest-text-search-tg.*|.*supplier-platform-ads-tg.*|.*supply-panel-new.*|.*supplier-pltfrm-promotions-tg.*"} offset 10m >= 50
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 50"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TG_HttpCode4xx_60+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-qwest-product-ts.*"} offset 10m >= 60
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 60"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TG_HttpCode4xx_100+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-storefront-int-secondary.*|.*platform-p-qwest-vs-eks.*|.*bac-p-merchandising.*|.*storefront-internal-secondary.*|.*bac-p-OSM-read-tg.*|.*bac-p-advertisement-admin.*|.*bac-p-community.*|.*bac-p-edge-proxy-farmiso-tg.*|.*bac-p-farmiso-pub.*|.*bac-p-farmiso-tg.*|.*bac-p-feed-aggregator-tg.*|.*bac-p-loyalty-earn-engine-tg.*|.*bac-p-mba-tg.*|.*bac-p-pricing.*|.*bac-p-recommendation-service.*|.*bac-p-referral-earnings.*|.*bac-p-reseller-tracking-tg.*|.*bac-p-sort-and-filter-sec-tg.*|.*bac-p-sort-and-filter-tg.*|.*bac-p-sort-and-filter-vs-tg.*|.*bac-p-storefront-int-primary.*|.*bac-p-supplier-referrals-tg.*|.*bac-p-taxonomy-int-tg.*|.*bac-p-upload-backend-tg.*|.*bac-p-widgetaggregator-tg.*|.*data-p-metabase-presto-tg.*|.*front-p-blog.*|.*front-p-community.*|.*front-p-farmiso-champ-tg.*|.*front-p-farmiso-customer-tg.*|.*front-p-meesho-com-tg.*|.*front-p-prism-node-ui-tg.*|.*front-p-reseller-logo.*|.*front-p-reseller-website-tg.*|.*front-p-webviews.*|.*platform-p-qwest-tg.*|.*supplier-pltfrm-payouts-tg.*|.*taxation-web.*"} offset 10m >= 100
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 100"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_200+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-CIS-ingestion.*|.*bac-p-ads-rti-enricher-search.*|.*bac-p-ads-rti-es-indexer-search.*|.*bac-p-agent-supp-aggregator-tg.*|.*bac-p-communicator-new-tg.*|.*bac-p-osm-supply-read.*|.*bac-p-pow-tg.*|.*bac-p-pricing-admin.*|.*bac-p-review-service.*|.*bac-p-supplier-orchestrator-tg.*|.*bac-p-supplier-sf-tg.*|.*front-p-supplier-panel-new-node.*|.*supplier-pltfrm-cataloging-tg.*|.*supplier-pltfrm-growth-tg.*"} offset 10m >= 200
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 200"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_300+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-communicator-service-tg.*|.*bac-p-gamification-tg.*|.*bac-p-inventory-admin-new.*|.*bac-p-manifest.*|.*bac-p-storefront-app-search.*|.*bac-p-storefront-int-secondary.*|.*bac-p-supplier-payment-web-tg.*|.*bac-p-supplier-store-admin-tg.*|.*bac-p-ums-user-profile.*|.*data-p-messagingapi-reactive-int.*|.*data-p-prism-tg.*|.*front-p-meesho-com.*|.*front-p-supplier-panel-tg.*|.*supplier-pltfrm-services-tg.*|.*supply-panel-new-node.*"} offset 10m >= 300
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 300"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_500+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-address-search.*|.*bac-p-db-access-new.*|.*bac-p-login.*|.*bac-p-mini-url-tg.*|.*bac-p-poa-supply-tg.*|.*bac-p-supplier-ums-tg.*|.*bac-p-taxonomy-admin.*|.*supply-supplier-web.*"} offset 10m >= 500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 500"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_1000+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-storefront-app-primary.*|.*supplier-pltfrm-fulfillment-tg.*|.*bac-p-storefront-app-secondary.*|.*bac-p-storefront-scaling.*|.*storefront-app-serving-primary.*|.*bac-p-osm-supply.*|.*bac-p-storefront-app-feed.*|.*front-p-pow-new-tg.*|.*front-p-pow-tg.*|.*bac-p-AB-service.*"} offset 10m >= 1000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 1000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_2500+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-edge-proxy-int-second-tg.*|.*bac-p-marketplace-tg.*|.*bac-p-order-state-management-tg.*|.*bac-p-payment-aggregator-web-tg.*|.*bac-p-post-order-aggregator-tg.*|.*bac-p-supplyapi-int-tg.*|.*data-p-prism-internal-new-tg.*|.*front-p-supplier-panel-v2-node.*|.*supplier-pltfrm-container.*|.*bac-p-ums-fallback-tg.*|.*front-p-community-tg.*|.*supply-api-v2-prod.*|.*bac-p-storefront-app-primary.*|.*bac-p-storefront-app-secondary.*"} offset 10m >= 2500
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 2500"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_7000+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-config-service-tg.*|.*bac-p-gatewayservice.*"} offset 10m >= 7000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 7000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_15k+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-ingestion-new.*|.*bac-p-ums.*|.*data-p-ingestion-web-tg.*"} offset 10m >= 15000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 15000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_18k+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-edge-proxy.*"} offset 10m >= 18000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 18000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
|
||||
- alert: TG_HttpCode4xx_150000+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-edge-proxy-second-tg.*"} offset 10m >= 150000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 150000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
- alert: TG_HttpCode4xx_200000+
|
||||
expr: aws_applicationelb_httpcode_target_4_xx_count_sum{job="aws_applicationelb", target_group =~ ".*bac-p-ingestion-separate.*"} offset 10m >= 200000
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
resource: AWS Target Group
|
||||
annotations:
|
||||
summary: "TG_HttpCode4xx in TG {{ $labels.target_group }} is greater than 200000"
|
||||
description: "TG_HttpCode4xx in TG {{ $labels.target_group }} is :{{ $value }}. Please check."
|
||||
|
||||
Reference in New Issue
Block a user