341 lines
18 KiB
YAML
341 lines
18 KiB
YAML
groups:
|
|
- name: Redis-Alerts
|
|
rules:
|
|
- alert: FreeableMemory-For-t3.medium
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-community.*|.*bac-p-farmiso.*|.*front-p-pow-int.*"} offset 10m / 1073741824 < 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 1GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-bulletin.*|bac-p-campaign-performance.*|bac-p-communicator-new.*|bac-p-datascience-relay.*|bac-p-feed-cluster.*|bac-p-manifest.*|bac-p-manifest-new.*|bac-p-ofs.*|bac-p-ofs-queue.*|bac-p-order-service.*|bac-p-vision-redis.*|bac-p-pricing-new.*|bac-p-supplier-ums.*|front-p-pow.*"} offset 10m / 1073741824 < 5
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 5GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-ctqct.*"} offset 10m / 1073741824 < 1.2
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 1.2GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.xLarge
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-advertisement-serving.*|bac-p-audience-new.*|bac-p-bulletin-new.*|bac-p-cdp-cluster.*|bac-p-inventory.*|bac-p-merchandising-cluster.*|bac-p-notification-store.*|bac-p-payments.*|bac-p-qwest-cluster-sf.*|bac-p-qwest-cluster-text-search.*|bac-p-qwest-cluster-vs.*|bac-p-qwest-sq.*|bac-p-search-preprocessor.*|bac-p-supply.*|bac-p-ums-clustered-users-profile.*|data-science-p-fy-exp.*"} offset 10m / 1073741824 < 8
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 8GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.2x.Large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-cis.*|bac-p-ct-qct-v2-newer.*|bac-p-datascience-relay-2.*|bac-p-organic-fy.*|bac-p-qwest-celery-text-search.*|bac-p-qwest-celery-text-search-2.*|bac-p-supplier-store-cluster.*|bac-p-taxonomy-monster.*|bac-p-transact.*|data-science-clp-exp.*"} offset 10m / 1073741824 < 10
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 10GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.4x.Large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-advertisement-views.*|bac-p-cart.*|bac-p-growth.*|bac-p-journey-new.*|bac-p-qwest-celery.*|bac-p-qwest-celery-vs-1.*|bac-p-qwest-cluster.*|bac-p-review-new.*"} offset 10m / 1073741824 < 14
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 14GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.8x.Large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-review.*"} offset 10m / 1073741824 < 18
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 18GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-r5-and-r6g.12x.Large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id=~ "bac-p-review.*"} offset 10m / 1073741824 < 25
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 25GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-m5.large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-mini-url.*|bac-p-qwest-celery-sf.*|data-p-airflow.*"} offset 10m / 1073741824 < 3
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 3GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: FreeableMemory-For-m5.large
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-opp.*"} offset 10m / 1073741824 < 2.5
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 2.5GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: FreeableMemory-For-m5.xlarge
|
|
expr: aws_elasticache_freeable_memory_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-deduplication-search.*|front-p-redis.*"} offset 10m / 1073741824 < 4
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is less than 4GB"
|
|
description: "FreeableMemory in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: EngineCPUUtilization
|
|
expr: aws_elasticache_engine_cpuutilization_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-journey-new-001"} offset 10m >= 70
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "EngineCPUUtilization in Redis {{ $labels.cache_cluster_id }} is greater than 70"
|
|
description: "EngineCPUUtilization in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: DatabaseMemoryUsagePercentage
|
|
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id !~ "bac-p-organic-fy.*|bac-p-manifest-002|bac-p-manifest-001|bac-p-datascience-relay-2.*|bac-p-ctqct.*|bac-p-review-new-001|bac-p-review-new-002"} offset 10m >= 70
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 70"
|
|
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: DatabaseMemoryUsagePercentage
|
|
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-datascience-relay-2.*|bac-p-ctqct.*"} offset 10m >= 85
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 85"
|
|
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: DatabaseMemoryUsagePercentage
|
|
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-review-new-001|bac-p-manifest-001|bac-p-manifest-002|bac-p-review-new-002"} offset 10m >= 80
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 80"
|
|
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: DatabaseMemoryUsagePercentage
|
|
expr: aws_elasticache_database_memory_usage_percentage_average{job="aws_elasticache",cache_cluster_id =~ "bac-p-organic-fy.*"} offset 10m >= 90
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is greater than 90"
|
|
description: "DatabaseMemoryUsagePercentage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: SwapUsage
|
|
expr: aws_elasticache_swap_usage_average{job="aws_elasticache"} offset 10m / 1000000 > 50
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "SwapUsage in Redis {{ $labels.cache_cluster_id }} is greater than 50MB"
|
|
description: "SwapUsage in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
- alert: ReplicationLag
|
|
expr: aws_elasticache_replication_lag_average{job="aws_elasticache"} offset 10m > 10
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "ReplicationLag in Redis {{ $labels.cache_cluster_id }} is greater than 10sec"
|
|
description: "ReplicationLag in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}. Please check."
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_70MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id !~ ".*bac-p-supplier-store-cluster.*|.*bac-p-manifest-serving.*|.*bac-p-farmiso.*|.*bac-p-organic-fy.*|.*bac-p-ums-users-profile-new.*|.*data-science-clp-exp.*|.*bac-p-merchandising.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*bac-p-feed.*|.*bac-p-notification-store.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*|.*bac-p-qwest-celery.*|.*bac-p-supplier-store-new.*|.*bac-p-ums-users-profile.*|bac-p-qwest-001|bac-p-qwest-002"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id !~ ".*bac-p-edge-proxy.*|.*bac-p-supplier-store-cluster.*|.*bac-p-manifest-serving.*|.*bac-p-farmiso.*|.*bac-p-organic-fy.*|.*bac-p-ums-users-profile-new.*|.*data-science-clp-exp.*|.*bac-p-merchandising.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*bac-p-feed.*|.*bac-p-notification-store.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*|.*bac-p-qwest-celery.*|.*bac-p-supplier-store-new.*|.*bac-p-ums-users-profile.*|bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000 ) > 70
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 70MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
- alert: NByteIn+NByteOut_110MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-manifest-serving.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-manifest-serving.*"} offset 10m / 60000000) > 110
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 110MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_120MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-notification-store.*|.*bac-p-farmiso.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*data-science-p-fy-exp.*|.*platform-p-qwest-2.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-edge-proxy.*|.*bac-p-farmiso.*|.*bac-p-notification-store.*|.*bac-p-inventory-new.*|.*bac-p-qwest-cluster.*|.*platform-p-qwest-2.*|.*data-science-p-fy-exp.*|.*bac-p-taxonomy-new.*|.*bac-p-pricing.*"} offset 10m / 60000000) > 120
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 120MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
- alert: NByteIn+NByteOut_250MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-organic-fy.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-organic-fy.*"} offset 10m / 60000000) > 250
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 250MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_90MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*data-science-clp-exp.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*data-science-clp-exp.*"} offset 10m / 60000000) > 90
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 90MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_156MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile.*"} offset 10m / 60000000) > 156
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 156MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
- alert: NByteIn+NByteOut_200MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-merchandising.*|.*bac-p-feed.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-merchandising.*|.*bac-p-feed.*"} offset 10m / 60000000) > 200
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 200MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
- alert: NByteIn+NByteOut_525MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-qwest-celery.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-qwest-celery.*"} offset 10m / 60000000) > 525
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 525MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_300MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile-new.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-ums-users-profile-new.*"} offset 10m / 60000000) > 300
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 300MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_215MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ "bac-p-qwest-001|bac-p-qwest-002|.*bac-p-review-0001.*"} offset 10m / 60000000) > 215
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 215MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|
|
|
|
|
|
|
|
- alert: NByteIn+NByteOut_250MB+
|
|
expr: (aws_elasticache_network_bytes_out_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-supplier-store-cluster.*"} offset 10m / 60000000 + aws_elasticache_network_bytes_in_average{job="aws_elasticache", cache_cluster_id =~ ".*bac-p-supplier-store-cluster.*"} offset 10m / 60000000) > 250
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
resource: AWS Redis
|
|
annotations:
|
|
summary: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is greater than 250MB"
|
|
description: "NetworkIn+NetWorkOut in Redis {{ $labels.cache_cluster_id }} is :{{ $value }}MB. Please check."
|