813 lines
37 KiB
YAML
813 lines
37 KiB
YAML
groups:
|
|
- name: Elastic-Search-Alerts
|
|
rules:
|
|
- alert: FreeStorageSpace-100G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name !~ 'bac-p-advertisement|bac-p-supplier-orders|bac-p-orders|bac-p-supplier-search|bac-p-advertisement-search|platform-p-qwest-analytics|bac-p-address-search|bac-p-sx-support|bac-p-review2'} offset 10m / 1024 < 100
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 100G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
- alert: FreeStorageSpace-3G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-sx-support'} offset 10m / 1024 < 3
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 3G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
- alert: FreeStorageSpace-25G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-orders'} offset 10m / 1024 < 25
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 25G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeStorageSpace-80G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'platform-p-qwest-analytics'} offset 10m / 1024 < 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 80G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeStorageSpace-90G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-supplier-search|bac-p-advertisement'} offset 10m / 1024 < 90
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 90G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeStorageSpace-85G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-review2'} offset 10m / 1024 < 85
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 85G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeStorageSpace-60G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-advertisement-search'} offset 10m / 1024 < 60
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 60G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeStorageSpace-50G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-address-search'} offset 10m / 1024 < 50
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 50G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeStorageSpace-40G
|
|
expr: aws_es_free_storage_space_average{job="aws_es", domain_name =~ 'bac-p-supplier-orders'} offset 10m / 1024 < 40
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is less than 40G"
|
|
description: "FreeStorageSpace in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: CPUUtilization_80+
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name !~ 'platform-p-garylog|bac-p-scraper|platform-p-qwest-analytics|platform-p-qwest'} offset 10m > 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
|
description: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
|
|
|
|
|
|
|
- alert: CPUUtilization_70+
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name =~ 'platform-p-qwest'} offset 10m > 70
|
|
for: 4m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 70%"
|
|
|
|
|
|
- alert: CPUUtilization_90+
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name =~ 'bac-p-scraper|platform-p-qwest-analytics'} offset 10m > 90
|
|
for: 4m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "CPUUtilization of Elastic Search {{ $labels.domain_name }} is greater than 90%"
|
|
|
|
|
|
- alert: JVMMemoryPressure
|
|
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name !~ 'platform-p-garylog|bac-p-orders|platform-p-ts-products|platform-p-qwest-analytics|platform-p-qwest-sf|platform-p-qwest-ts-catalogs|platform-p-ofs-catalogs|platform-p-ofs-products'} offset 10m >= 80
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
|
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
|
|
|
|
|
- alert: JVMMemoryPressure
|
|
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'platform-p-qwest-analytics'} offset 10m >= 80
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 80%"
|
|
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
|
|
|
|
|
|
|
- alert: JVMMemoryPressure
|
|
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'bac-p-orders'} offset 10m >= 95
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 95%"
|
|
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: JVMMemoryPressure
|
|
expr: aws_es_jvmmemory_pressure_maximum{job="aws_es", domain_name =~ 'platform-p-qwest-sf|platform-p-ofs-catalogs|platform-p-qwest-ts-catalogs|platform-p-ts-products|platform-p-ofs-products'} offset 10m >= 90
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is greater than 90%"
|
|
description: "JVMMemoryPressure in Elastic Search {{ $labels.domain_name }} is :{{ $value }}%. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: ClusterIndexWritesBlocked
|
|
expr: aws_es_cluster_index_writes_blocked_maximum{job="aws_es"} offset 10m >= 1
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "ClusterIndexWritesBlocked in Elastic Search {{ $labels.domain_name }} is blocked for more than 1 mins"
|
|
description: "ClusterIndexWritesBlocked in Elastic Search {{ $labels.domain_name }} is :{{ $value }}. Please check."
|
|
|
|
- alert: ClusterStatus.red
|
|
expr: aws_es_cluster_status_red_maximum{job="aws_es"} offset 10m >= 1
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "ClusterStatus.red for Elastic Search {{ $labels.domain_name }}."
|
|
description: "ClusterStatus.red for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: Nodes
|
|
expr: aws_es_nodes_minimum{job="aws_es", domain_name=~ "bac-p-review2|platform-p-qwest" } < 9 or aws_es_nodes_minimum{job="aws_es", domain_name="bac-p-scraper" } < 6 or aws_es_nodes_minimum{job="aws_es", domain_name="bac-p-supplier-search" } < 2
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Nodes for Elastic Search {{ $labels.domain_name }} is less than mentioned number in expression."
|
|
description: "Cluster ES node for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: KibanaHealthyNodes
|
|
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es"} offset 10m != 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
|
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: KibanaHealthyNodes
|
|
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name!~"platform-p-qwest-sort-filter|platform-p-qwest-analytics|platform-p-qwest-text-search" } offset 10m < 1
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
|
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: KibanaHealthyNodes
|
|
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name=~"platform-p-qwest-sort-filter|platform-p-qwest-text-search" } offset 10m < 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
|
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: KibanaHealthyNodes
|
|
expr: aws_es_kibana_healthy_nodes_minimum{job="aws_es", domain_name=~"platform-p-qwest-analytics" } offset 10m < 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "KibanaHealthyNodes {{ $labels.domain_name }} is less than mentioned number in expression."
|
|
description: "KibanaHealthyNodes for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: ES_4xx_20+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name!~"platform-p-graylog|campaign-performance|platform-p-ts-products|platform-p-qwest-sf|platform-p-garylog|platform-p-qwest-sort-filter|bac-p-advertisement|bac-p-supplier-search|platform-p-ofs|bac-p-advertisement-ts|platform-p-qwest|platform-p-qwest-text-search|bac-p-scraper|platform-p-qwest-analytics|platform-p-ofs-catalogs|platform-p-sf-catalogs"} offset 10m >= 20
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: ES_4xx_50+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-ofs|platform-p-qwest-sf"} offset 10m >= 50
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: ES_4xx_100+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"bac-p-scraper|campaign-performance|platform-p-qwest-analytics|platform-p-qwest-sort-filter|platform-p-qwest-text-search|platform-p-ofs-catalogs|bac-p-advertisement|bac-p-advertisement-ts"} offset 10m >= 100
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: ES_4xx_200+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-sf-catalogs"} offset 10m >= 200
|
|
for: 300s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
- alert: ES_4xx_170+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 170
|
|
for: 300s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
- alert: ES_4xx_1000+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-ts-products"} offset 10m >= 1000
|
|
for: 300s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
- alert: ES_4xx_50+
|
|
expr: aws_es_4xx_sum{job="aws_es", domain_name=~"platform-p-qwest"} offset 10m >= 50
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "4xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_5xx_5+
|
|
expr: aws_es_5xx_sum{job="aws_es", domain_name!~"platform-p-garylog"} offset 10m >= 5
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "5xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
description: "5xx for Elastic Search {{ $labels.domain_name }}. Please check."
|
|
|
|
|
|
|
|
|
|
|
|
# NODE WISE ES RULES
|
|
|
|
|
|
|
|
- alert: ES_Master_CPU_Utilization_60%
|
|
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name!~"bac-p-recommendation|bac-p-scraper|platform-p-garylog"} offset 10m >= 60
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60%. Please check."
|
|
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Master_CPU_Utilization_60%
|
|
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name=~"bac-p-recommendation|bac-p-recommendation-price"} offset 10m >= 60
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60%. Please check."
|
|
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Master_CPU_Utilization_30%
|
|
expr: aws_es_master_cpuutilization_average{job="aws_es",domain_name=~"bac-p-scraper"} offset 10m >= 30
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 30%. Please check."
|
|
description: "Master CPU Utilzation for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Node_CPU_Utilization_65%
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name!~"platform-p-qwest-analytics|platform-p-garylog|bac-p-scraper|platform-p-qwest-ts-catalogs"} offset 10m >= 65
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 65%. Please check."
|
|
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Node_CPU_Utilization_80%
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"bac-p-scraper"} offset 10m >= 80
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 80%. Please check."
|
|
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Node_CPU_Utilization_90%
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"platform-p-qwest-analytics"} offset 10m >= 90
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 90%. Please check."
|
|
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: ES_Node_CPU_Utilization_70%
|
|
expr: aws_es_cpuutilization_average{job="aws_es", domain_name=~"platform-p-qwest-ts-catalogs"} offset 10m >= 70
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 70%. Please check."
|
|
description: "Node CPU Utilzation for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Indexing_Rate_100k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name!~"bac-p-recommendation|platform-p-ofs-catalogs|bac-p-advertisement-search|campaign-performance|platform-p-ts-products|bac-p-supplier-orders|bac-p-merchandising|bac-p-advertisement|platform-p-garylog|platform-p-ofs|bac-p-supplier-search|platform-p-qwest-sort-filter|bac-p-review2|platform-p-qwest|platform-p-qwest-analytics|bac-p-recommendation-price|platform-p-qwest-text-search|platform-p-ofs-products|bac-p-orders"} offset 10m >= 100000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_100k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"campaign-performance"} offset 10m >= 100000
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_200k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-advertisement-search|bac-p-orders"} offset 10m >= 200000
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 200k. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_250k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"platform-p-ofs"} offset 10m >= 250000
|
|
for: 20s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 250k. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_150k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-merchandising|platform-p-ofs-catalogs|platform-p-qwest-text-search|platform-p-qwest-sort-filter|bac-p-advertisement|platform-p-qwest-analytics"} offset 10m >= 150000
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 150K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_170k
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"platform-p-ts-products"} offset 10m >= 170000
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 170K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Indexing_Rate_1.2M
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-recommendation"} offset 10m >= 1200000
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 1.2M. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: ES_Indexing_Rate_120K
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-review2|platform-p-qwest"} offset 10m >= 120000
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 120K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Indexing_Rate_700K
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 700000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 700K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Indexing_Rate_400K
|
|
expr: aws_es_indexing_rate_average{job="aws_es", domain_name=~"bac-p-recommendation-price"} offset 10m >= 400000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Indexing Rate for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 400K. Please check."
|
|
description: "Indexing Rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Latency_60ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name!~"bac-p-address-search|bac-p-recommendation|platform-p-ofs|platform-p-qwest-ts-catalogs|bac-p-recommendation-price|platform-p-garylog|bac-p-scraper|platform-p-qwest-sort-filter|bac-p-supplier-order-search|platform-p-qwest|bac-p-supplier-search|platform-p-qwest-analytics|platform-p-qwest-text-search|bac-p-review2|bac-p-orders|platform-p-qwest-sf|bac-p-merchandising|platform-p-ofs-products|platform-p-ts-products"} offset 10m >= 60
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 60ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Latency_50ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-supplier-order-search|bac-p-orders"} offset 10m >= 50
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 50ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Latency_500ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-sort-filter|platform-p-ts-products|bac-p-address-search"} offset 10m >= 500
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 500ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_80ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest"} offset 10m >= 80
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 80ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_1000ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-text-search"} offset 10m >= 1000
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 1000ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_75ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-ofs"} offset 10m >= 75
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 75ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
- alert: ES_Search_Latency_100ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-qwest-analytics|platform-p-qwest-ts-catalogs"} offset 10m >= 100
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_8000ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-merchandising"} offset 10m >= 8000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 8000ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Latency_300ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-review2|platform-p-qwest-sf"} offset 10m >= 300
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 300ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_200ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"platform-p-ofs-products"} offset 10m >= 200
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 200ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
- alert: ES_Search_Latency_250ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-supplier-search"} offset 10m >= 250
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 250ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Latency_450ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-scraper"} offset 10m >= 450
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 450ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Latency_100ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-recommendation-price"} offset 10m >= 100
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 100ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
- alert: ES_Search_Latency_150ms
|
|
expr: aws_es_search_latency_average{job="aws_es",domain_name=~"bac-p-recommendation"} offset 10m >= 150
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search Latency for Elastic Search {{ $labels.domain_name}}, {{ $labels.node_id}} is greater than 150ms. Please check."
|
|
description: "Search Latency for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}}is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Rate_20K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name!~"platform-p-ofs-catalogs|platform-p-qwest-ts-catalogs|bac-p-orders|bac-p-review2|bac-p-supplier-orders|platform-p-ofs|platform-p-garylog|bac-p-advertisement-ts|bac-p-supplier-order-search|bac-p-supplier-search|bac-p-advertisement|platform-p-qwest-text-search|platform-p-qwest-analytics|bac-p-community|bac-p-advertisement-search"} offset 10m >= 20000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 20K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
- alert: ES_Search_Rate_30K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-orders"} offset 10m >= 30000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 30K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Rate_40K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-advertisement-ts|bac-p-advertisement-search|platform-p-ofs|platform-p-ofs-catalogs"} offset 10m >= 40000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 40K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Rate_120K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-order-search"} offset 10m >= 120000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 120K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Rate_65K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"platform-p-qwest-analytics|platform-p-qwest-ts-catalogs"} offset 10m >= 65000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 65K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
- alert: ES_Search_Rate_60K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-orders"} offset 10m >= 60000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
- alert: ES_Search_Rate_60K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-community"} offset 10m >= 60000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 60K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Rate_55K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"platform-p-qwest-text-search"} offset 10m >= 55000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 55K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Rate_80K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-advertisement|bac-p-review2"} offset 10m >= 80000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 80K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: ES_Search_Rate_200K
|
|
expr: aws_es_search_rate_average{job="aws_es", domain_name=~"bac-p-supplier-search"} offset 10m >= 200000
|
|
for: 10s
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Elastic Search
|
|
annotations:
|
|
summary: "Search rate for Elastic Search {{ $labels.domain_name }}, {{ $labels.node_id}} is greater than 200K. Please check."
|
|
description: "Search rate for Elastic Search {{ $labels.domain_name}} , {{ $labels.node_id}} is {{ $value }}. Please check."
|