groups:
- name: EC2-grafana-Alerts
  rules:
  - alert: grafana-CPU-Crossed-80%
    expr: 100 - (avg(irate(node_cpu_seconds_total{job=~'node_exporter', mode="idle", type=~'.*grafana.*'}[1m])) by (name, private_ip) * 100) > 80
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: grafana
      env: prd
      priority: p0
    annotations:
      summary: grafana Instance out of CPU (instance {{ $labels.private_ip }})
      description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of CPU\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: grafana-Memory-Crossed-90%
    expr: 100 * (1 - ((avg_over_time(node_memory_MemFree_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Cached_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]) + avg_over_time(node_memory_Buffers_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m])) / avg_over_time(node_memory_MemTotal_bytes{job=~'node_exporter', type=~'.*grafana.*'}[1m]))) > 90
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: grafana
      env: prd
      priority: p0
    annotations:
      summary: grafana Instance out of Memory (instance {{ $labels.private_ip }})
      description: "{{ $labels.name }}/{{ $labels.private_ip }} is out of Memory\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: grafana-DiskSpace-Crossed-80%
    expr: 100 - ((node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter",  type=~'.*grafana.*'} * 100) / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs", job=~"node_exporter",  type=~'.*grafana.*'}) > 80
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: grafana
      env: prd
      priority: p0
    annotations:
      summary: grafana Instance running out of Disk Space (instance {{ $labels.private_ip }})
      description: "{{ $labels.name }}/{{ $labels.private_ip }} is running out of Disk Space\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: grafana-Instance-Down
    expr: up{job="node_exporter", type=~'.*grafana.*'} == 0
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: grafana
      env: prd
      priority: p0
    annotations:
      summary: grafana Instance is DOWN (instance {{ $labels.private_ip }})
      description: "{{ $labels.name }}/{{ $labels.private_ip }} is DOWN\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

  - alert: grafana-Down
    expr: probe_http_status_code{job="blackbox", instance="https://grafana.meesho.com"} != 200
    for: 1m
    labels:
      severity: critical
      bu: infra
      team: sre
      service: grafana
      env: prd
      priority: p0
    annotations:
      summary: grafana is DOWN (URL - {{ $labels.instance }})
      description: "{{ $labels.instance }} is DOWN\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"