devops-infra-helm-charts-gcp/helm-templates/victoria-metrics-alerts-config/configmap/generic/kafka-configmap.yaml

groups:
- name: Kafka-Alerts
  rules:
  - alert: GlobalTopicCount_200+
    expr: aws_kafka_global_topic_count_average{job="aws_kafka"} offset 5m  > 200
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is greater than 200+"
      description: "GlobalTopicCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: OfflinePartitionsCount_0+
    expr: aws_kafka_offline_partitions_count_average{job="aws_kafka"} offset 5m > 0
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is greater than 0"
      description: "OfflinePartitionsCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: KafkaDataLogsDiskUsed
    expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name!~"bac-p-communicator.*|bac-p-datahub.*|bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 80
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 80"
      description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"
      
      
  - alert: KafkaDataLogsDiskUsed
    expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-communicator.*"} offset 5m > 60
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
      description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: KafkaDataLogsDiskUsed
    expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-discovery-kafka.*|bac-p-discovery-search-kafka.*"} offset 5m > 60
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
      description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"

  - alert: KafkaAppLogsDiskUsed
    expr: aws_kafka_kafka_app_logs_disk_used_average{job="aws_kafka"} offset 10m  > 50
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 50"
      description: "KafkaAppLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: KafkaDataLogsDiskUsed
    expr: aws_kafka_kafka_data_logs_disk_used_average{job="aws_kafka", cluster_name=~"bac-p-datahub.*"} offset 5m > 90
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 90"
      description: "KafkaDataLogsDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"      
  

  
  - alert: RootDiskUsed
    expr: aws_kafka_root_disk_used_average{job="aws_kafka"} offset 10m > 60
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is greater than 60"
      description: "RootDiskUsed in Kafka {{ $labels.cluster_name }} is {{ $value }}"


   
  - alert: NetworkRxDropped
    expr: aws_kafka_network_rx_dropped_average{job="aws_kafka"} offset 10m > 0
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is greater than 0"
      description: "NetworkRxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}"
   

   
  - alert: NetworkTxDropped
    expr: aws_kafka_network_tx_dropped_average{job="aws_kafka"} offset 10m > 0
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is greater than 0"
      description: "NetworkTxDropped in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: CpuUser
    expr: aws_kafka_cpu_user_average{job="aws_kafka"} offset 10m > 85
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "CpuUser in Kafka {{ $labels.cluster_name }} is greater than 85"
      description: "CpuUser in Kafka {{ $labels.cluster_name }} is {{ $value }}"
  

  
  - alert: ZooKeeperSessionState
    expr: aws_kafka_zoo_keeper_session_state_sample_count{job="aws_kafka"} offset 10m != 1
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is not equal to 1"
      description: "ZooKeeperSessionState in Kafka {{ $labels.cluster_name }} is {{ $value }}"

  
  - alert: ActiveControllerCount
    expr: aws_kafka_active_controller_count_sample_count{cluster_name =~ "data-platform-p-kafka|data-platform-p-prism-kafka|bac-p-communicator-kafka", job="aws_kafka"} offset 10m < 3
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 3"
      description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: Kafka_BytesInPerSec
    expr: aws_kafka_bytes_in_per_sec_average{job="aws_kafka"} offset 10m == 0
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is 0"
      description: "Kafka_BytesInPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}"

  - alert: Kafka_BytesOutPerSec
    expr: aws_kafka_bytes_out_per_sec_average{job="aws_kafka"} offset 10m == 0
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is not equal to 3"
      description: "Kafka_BytesOutPerSec in Kafka {{ $labels.cluster_name }} is {{ $value }}"


  - alert: Kafka_ReplicationBytes_plus_bytes_in_and_out_200MB+
    expr: (aws_kafka_bytes_in_per_sec_average offset 10m + aws_kafka_bytes_out_per_sec_average offset 10m + aws_kafka_replication_bytes_in_per_sec_average offset 10m + aws_kafka_replication_bytes_out_per_sec_average offset 10m ) > 200000000
    for: 60s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is not equal to 200MB"
      description: "Kafka_ReplicationBytes_plus_bytes_in_and_out in Kafka {{ $labels.cluster_name }} is {{ $value }}"

  - alert: ActiveControllerCount_IngestionKafka
    expr: aws_kafka_active_controller_count_sample_count{cluster_name="bac-p-ingestion-kafka",job="aws_kafka"} offset 10m != 4
    for: 120s
    labels:
      severity: critical
      resource: AWS Kafka
    annotations:
      summary: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is not equal to 4"
      description: "ActiveControllerCount in Kafka {{ $labels.cluster_name }} is {{ $value }}"