## Chart information nameOverride: "" fullnameOverride: "" namespaceOverride: "elastalert-prd" commonLabels: {} commonAnnotations: {} appKubernetesIoComponent: elastalert2 # Folder where Helm can find local rules prior to deployment to the k8s cluster. By default, # 'rules' folder must be located in the root of the chart directory. Note that this setting # will override the rules and secretRulesName values. Again, these rules are only read # during the time of the chart deployment (installation) into the cluster. # rootRulesFolder: "rules" # enabledRules: ["deadman_slack", "deadman_pagerduty"] # number of replicas to run replicaCount: 1 # update strategy to use (default : RollingUpdate) but can be Recreate updateStrategy: type: RollingUpdate rollingUpdate: {} # number of helm release revisions to retain revisionHistoryLimit: 5 # number of seconds for which a newly created Pod should be ready without any of its containers crashing, for it to be considered available. minReadySeconds: 5 # Default internal between alert checks against the elasticsearch datasource, in minutes runIntervalMins: 1 # Location of directory where rules reside rulesFolder: "/opt/elastalert/rules" # Enable/disabe subdirectory scanning for rules scanSubdirectories: true # Default rule buffer duration, in minutes bufferTimeMins: 15 # Amount of time to retry and deliver failed alerts (1440 minutes per day) alertRetryLimitMins: 2880 # Default time before realerting, in minutes realertIntervalMins: "" # For ES 5: The name of the index which stores elastalert 2 statuses, typically elastalert_status # For ES 6: The prefix of the names of indices which store elastalert 2 statuses, typically elastalert # writebackIndex: elastalert image: # docker image repository: jertel/elastalert2 # docker image tag tag: 2.29.0 pullPolicy: IfNotPresent pullSecret: "" resources: requests: memory: 8Gi cpu: "1" limits: memory: 10Gi cpu: "3" # Annotations to be added to deployment deploymentAnnotations: {} # Annotations to be added to pods podAnnotations: {} elasticsearch: # ECK-managed ES service: -es-http..svc.cluster.local host: eck-observability-prd-es-http.eck-observability-prd.svc.cluster.local # elasticsearch port port: 9200 # whether or not to connect to es_host using TLS # TLS is disabled on the ES HTTP layer (selfSignedCertificate.disabled: true in elasticsearch.yaml) useSsl: "False" # Username if authenticating to ES with basic auth username: "elastic" # Password if authenticating to ES with basic auth # Get from: kubectl get secret eck-observability-stg-es-elastic-user -n eck-observability-stg -o jsonpath='{.data.elastic}' | base64 -d password: "qT448rgRqJkIesBerfrcAXH2" # Specifies an existing secret to be used for the ES username/password credentialsSecret: "" # The key in elasticsearch.credentialsSecret that stores the ES password credentialsSecretUsernameKey: "" # The key in elasticsearch.credentialsSecret that stores the ES username credentialsSecretPasswordKey: "" # whether or not to verify TLS certificates # False because TLS is disabled on this cluster verifyCerts: "False" # Enable certificate based authentication # path to a PEM certificate to use as the client certificate # clientCert: "/certs/client.pem" # path to a private key file to use as the client key # clientKey: "/certs/client-key.pem" # path to a CA cert bundle to use to verify SSL connections # caCerts: "/certs/ca.pem" # # certs volumes, required to mount ssl certificates when elasticsearch has tls enabled # certsVolumes: # - name: es-certs # secret: # defaultMode: 420 # secretName: es-certs # # mount certs volumes, required to mount ssl certificates when elasticsearch has tls enabled # certsVolumeMounts: # - name: es-certs # mountPath: /certs # readOnly: true # Optional env variables for the pod optEnv: [] ## Specify optional additional containers to run alongside the Elastalert2 container. extraContainers: [] ## Specify optional additional initContainers to run prior to the Elastalert2 container. extraInitContainers: [] extraConfigOptions: {} # # Options to propagate to all rules, e.g. a common slack_webhook_url or kibana_url # # Please note at the time of implementing this value, it will not work for required_locals # # Which MUST be set at the rule level, these are: ['alert', 'type', 'name', 'index'] # kibana_url: https://kibana.yourdomain.com # slack_webhook_url: dummy # To load ElastAlert 2 config via secret, uncomment the line below # secretConfigName: elastalert-config-secret # Example of a secret config #apiVersion: v1 #kind: Secret #metadata: # name: elastalert-config-secret #type: Opaque #stringData: # elastalert_config: |- # rules_folder: /opt/elastalert/rules # scan_subdirectories: false # run_every: # minutes: 1 # buffer_time: # minutes: 15 # es_host: elasticsearch # es_port: 9200 # writeback_index: elastalert # use_ssl: False # verify_certs: True # alert_time_limit: # minutes: 2880 # slack_webhook_url: https://hooks.slack.com/services/xxxx # slack_channel_override: '#alerts' # To load ElastAlert's rules via secret, uncomment the line below #secretRulesName: elastalert-rules-secret # Additionally, you must specificy which rules to load from the secret #secretRulesList: [ "rule_1", "rule_2" ] # Example of secret rules #apiVersion: v1 #kind: Secret #metadata: # name: elastalert-rules-secret # namespace: elastic-system #type: Opaque #stringData: # rule_1: |- # name: Rule 1 # type: frequency # index: index1-* # num_events: 3 # timeframe: # minutes: 1 # alert: # - "slack" # rule_2: |- # name: Rule 2 # type: frequency # index: index2-* # num_events: 5 # timeframe: # minutes: 10 # alert: # - "slack" # Command and args override for container e.g. (https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/) # command: ["YOUR_CUSTOM_COMMAND"] # args: ["YOUR", "CUSTOM", "ARGS"] # specifies the rules volume to be used rulesVolumeName: "rules" # additional rule configurations e.g. (http://elastalert2.readthedocs.io/en/latest/) rules: multi_index_doc_count_threshold: |- --- name: "High Doc Count Alert - GCP and GKE Indices" type: frequency # 1. Target Index Selection: Combines all indices with these prefixes index: "gcp-prd-*,gcp-kubeevents-*,gke-mcs-*" # 2. Threshold: Total count across all matched indices num_events: 950000000 timeframe: hours: 1 # 3. Optimization: Essential for Basic Tier and high doc counts use_count_query: true doc_type: "_doc" # 4. Filter: Count everything in those indices filter: - query: match_all: {} # 5. PagerDuty Action (Configured per your requirements) alert: - "pagerduty" # Routing and Severity pagerduty_service_key: "63386d7276d24c08d0b795122bfea0b1" pagerduty_severity: "critical" # Client and Description pagerduty_client_name: "high doc alert" pagerduty_description: "High Document Count Alert: Total count across gcp-prd, gcp-kubeevents, and gke-mcs has exceeded 950,000,000." # Deduplication Key (to avoid multiple pages for the same spike) pagerduty_incident_key: "high-doc-count-gcp" # deadman_slack: |- # --- # name: Deadman Switch Slack # type: frequency # index: containers-* # num_events: 3 # timeframe: # minutes: 3 # filter: # - term: # message: "deadmanslack" # alert: # - "slack" # slack: # slack_webhook_url: dummy # deadman_pagerduty: |- # --- # name: Deadman Switch PagerDuty # type: frequency # index: containers-* # num_events: 3 # timeframe: # minutes: 3 # filter: # - term: # message: "deadmanpd" # alert: # - "pagerduty" # pagerduty: # pagerduty_service_key: dummy # pagerduty_client_name: ElastAlert Deadman Switch # Probes configuration livenessProbe: enabled: false readinessProbe: enabled: false serviceAccount: # Specifies whether a service account should be created create: true # Annotations to add to the service account annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: # Enable pod security policy # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ # DEPRECATED in Kubernetes 1.21 (https://kubernetes.io/blog/2021/04/06/podsecuritypolicy-deprecation-past-present-and-future/) podSecurityPolicy: create: false securityContext: runAsNonRoot: true runAsUser: 1000 podSecurityContext: fsGroup: 1000 runAsUser: 1000 runAsGroup: 1000 # Support using node selectors and tolerations nodeSelector: cloud.google.com/gke-nodepool: np-admin-default-v131-prd-ase1 team: shared # Specify node affinity or anti-affinity specifications affinity: {} # Autoscaling configuration autoscaling: enabled: true minReplicas: 2 maxReplicas: 5 targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 # Optional automatic SMTP mail server credential management. # smtp_auth: # username: "" # password: "" extraVolumes: [] # - name: smtp-auth # secret: # secretName: elastalert-smtp-auth # items: # - key: smtp_auth.yaml # path: smtp_auth.yaml # mode: 0400 extraVolumeMounts: [] # - name: smtp-auth # mountPath: /opt/elastalert/config-smtp/smtp_auth.yaml # subPath: smtp_auth.yaml # readOnly: true ## @section Metrics parameters ## Prometheus metrics ## metrics: ## @param metrics.enabled Enable the export of Prometheus metrics ## enabled: false prometheusPort: 8080 prometheusPortName: http-alt # Prometheus Exporter defined by port: prometheusScrapeAnnotations: prometheus.io/scrape: "true" prometheus.io/path: "/" service: type: ClusterIP # clusterIP: "" # externalTrafficPolicy: Cluster # loadBalancerIP: "" # loadBalancerSourceRanges: {} # nodePorts: "" ## Prometheus Operator ServiceMonitor configuration ## serviceMonitor: ## @param metrics.serviceMonitor.enabled Specify if a ServiceMonitor will be deployed for Prometheus Operator ## enabled: false ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running ## namespace: "" ## @param metrics.serviceMonitor.labels Extra labels for the ServiceMonitor ## Normally used for prometheus operator to detect the servicemonitor if deployed to different namespace ## labels: ## release: prometheus-operator labels: {} ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in Prometheus ## jobLabel: "" ## @param metrics.serviceMonitor.interval How frequently to scrape metrics ## e.g: ## interval: 10s ## interval: "" ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended ## e.g: ## scrapeTimeout: 10s ## scrapeTimeout: "" ## @param metrics.serviceMonitor.metricRelabelings [array] Specify additional relabeling of metrics ## metricRelabelings: ## # Drop GO metrics ## - sourceLabels: [__name__] ## regex: go_.* ## action: drop ## # Drop python_gc metrics ## - sourceLabels: [__name__] ## regex: python_gc.* ## action: drop ## # Normalise POD names ## - sourceLabels: [pod] ## regex: (.+elastalert2)\-([\w\d]+)\-([\w\d]+) ## replacement: $1 ## targetLabel: pod metricRelabelings: [] ## @param metrics.serviceMonitor.relabelings [array] Specify general relabeling ## relabelings: [] ## @param metrics.serviceMonitor.selector Prometheus instance selector labels ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration ## selector: {} ## PrometheusRule CRD configuration ## prometheusRule: ## @param metrics.prometheusRule.enabled If `true`, creates a Prometheus Operator PrometheusRule (also requires `metrics.enabled` to be `true`) ## enabled: false ## @param metrics.prometheusRule.namespace Namespace in which the PrometheusRule CRD is created ## namespace: "" ## @param metrics.prometheusRule.additionalLabels Additional labels for the prometheusRule ## to be detected by prometheus-operator ## additionalLabels: ## release: prometheus-operator additionalLabels: {} ## @param metrics.prometheusRule.rules Prometheus Rules for ElastAlert 2. ## These are just examples rules, please adapt them to your needs. ## rules: |- ## groups: ## - name: elastalert ## rules: ## - alert: elastalert Pod down ## annotations: ## description: Prometheus is unable to scrape metrics service. Check pod logs for details ## summary: elastalert POD is down ## expr: up{service="{{ template "common.names.servicename" . }}",container="elastalert"} == 0 ## for: 5m ## labels: ## severity: critical ## production: 'True' ## - alert: elastalert file descriptors use ## annotations: ## description: Elastalert pod nearly exhausting file descriptors ## summary: too many file descriptors used ## expr: |- ## process_open_fds{service="{{ template "common.names.servicename" . }}",container="elastalert"} ## / ## process_max_fds{service="{{ template "common.names.servicename" . }}",container="elastalert"} ## > 0.9 ## for: 3m ## labels: ## severity: critical ## production: 'True' ## - alert: elastalert scrapes failing ## annotations: ## description: Elastalert is not scraping for a rule {{ "{{" }} $labels.rule_name {{ "}}" }} ## summary: scrapes for rule stalled {{ "{{" }} $labels.rule_name {{ "}}" }} ## expr: |- ## rate(elastalert_scrapes_total{service="{{ template "common.names.servicename" . }}",container="elastalert"}[1m]) == 0 ## for: 5m ## labels: ## severity: critical ## production: 'True' rules: []