groups: - name: Kafka-Alerts rules: - alert: GlobalTopicCount_200+ expr: aws_kafka_global_topic_count_average{job="aws_kafka"} offset 5m > 200 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is greater than 200+" description: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: OfflinePartitionsCount_0+ expr: aws_kafka_offline_partitions_count_average{job="aws_kafka"} offset 5m > 0 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is greater than 0" description: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: KafkaDataLogsDiskUsed expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name!~"bac-p-communicator.*|bac-p-datahub.*|bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 80 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 80" description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: KafkaDataLogsDiskUsed expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-communicator.*"} offset 5m > 60 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60" description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: KafkaDataLogsDiskUsed expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 60 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60" description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: KafkaAppLogsDiskUsed expr: aws_kafka_kafka_app_logs_disk_used_average{job="aws_kafka"} offset 10m > 50 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 50" description: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: KafkaDataLogsDiskUsed expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-datahub.*"} offset 5m > 90 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 90" description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: RootDiskUsed expr: aws_kafka_root_disk_used_average{job="aws_kafka"} offset 10m > 60 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60" description: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: NetworkRxDropped expr: aws_kafka_network_rx_dropped_average{job="aws_kafka"} offset 10m > 0 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is greater than 0" description: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: NetworkTxDropped expr: aws_kafka_network_tx_dropped_average{job="aws_kafka"} offset 10m > 0 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is greater than 0" description: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: CpuUser expr: aws_kafka_cpu_user_average{job="aws_kafka"} offset 10m > 85 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "CpuUser in Kafka {{ $labels.cluster_name }} is greater than 85" description: "CpuUser in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: ZooKeeperSessionState expr: aws_kafka_zoo_keeper_session_state_sample_count{job="aws_kafka"} offset 10m != 1 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is not equal to 1" description: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: ActiveControllerCount expr: aws_kafka_active_controller_count_sample_count{cluster_name =~ "data-platform-p-kafka|data-platform-p-prism-kafka|bac-p-communicator-kafka", job="aws_kafka"} offset 10m < 3 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 3" description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: Kafka_BytesInPerSec expr: aws_kafka_bytes_in_per_sec_average{job="aws_kafka"} offset 10m == 0 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is 0" description: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: Kafka_BytesOutPerSec expr: aws_kafka_bytes_out_per_sec_average{job="aws_kafka"} offset 10m == 0 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is not equal to 3" description: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: Kafka_ReplicationBytes_plus_bytes_in_and_out_200MB+ expr: (aws_kafka_bytes_in_per_sec_average offset 10m + aws_kafka_bytes_out_per_sec_average offset 10m + aws_kafka_replication_bytes_in_per_sec_average offset 10m + aws_kafka_replication_bytes_out_per_sec_average offset 10m ) > 200000000 for: 60s labels: severity: critical resource: AWS Kafka annotations: summary: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is not equal to 200MB" description: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is {{ $value }}" - alert: ActiveControllerCount_IngestionKafka expr: aws_kafka_active_controller_count_sample_count{cluster_name="bac-p-ingestion-kafka",job="aws_kafka"} offset 10m != 4 for: 120s labels: severity: critical resource: AWS Kafka annotations: summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 4" description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"