nameOverride: ""
fullnameOverride: ""

# DaemonSet, Deployment or StatefulSet
kind: "DaemonSet"
# azureblob, cloudwatch, elasticsearch7, elasticsearch8, gcs, graylog , kafka, kafka2, kinesis, opensearch
variant: gcs
# # Only applicable for Deployment or StatefulSet
# replicaCount: 1

image:
  repository: "asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/sre/fluentd-v2"
  pullPolicy: "Always"
  tag: "edge-debian"

## Optional array of imagePullSecrets containing private registry credentials
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets: []

serviceAccount:
  create: true
  annotations: {
    iam.gke.io/gcp-service-account: sa-dsci-dssre-fluentd-prd@meesho-datascience-prd-0622.iam.gserviceaccount.com
  }
  name: null

rbac:
  create: true

# from Kubernetes 1.25, PSP is deprecated
# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
# We automatically disable PSP if Kubernetes version is 1.25 or higher
podSecurityPolicy:
  enabled: true
  annotations: {}

## Security Context policies for controller pods
## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
## notes on enabling and using sysctls
##
podSecurityContext: {}
  # seLinuxOptions:
  #   type: "spc_t"

securityContext: {}
  # capabilities:
  #   drop:
  #   - ALL
  # readOnlyRootFilesystem: true
  # runAsNonRoot: true
  # runAsUser: 1000

# Configure the livecycle
# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
lifecycle: {}
  # preStop:
  #   exec:
  #     command: ["/bin/sh", "-c", "sleep 20"]

# Configure the livenessProbe
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
#livenessProbe:
#  httpGet:
#    path: /metrics
#    port: metrics
  # initialDelaySeconds: 0
  # periodSeconds: 10
  # timeoutSeconds: 1
  # successThreshold: 1
  # failureThreshold: 3

# Configure the readinessProbe
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
#readinessProbe:
#  httpGet:
#    path: /metrics
#    port: metrics
  # initialDelaySeconds: 0
  # periodSeconds: 10
  # timeoutSeconds: 1
  # successThreshold: 1
  # failureThreshold: 3

resources:
  requests:
    cpu: 100m
    memory: 50Mi
  limits:
    memory: 2000Mi
    cpu: 2000m

## only available if kind is Deployment
autoscaling:
  enabled: false
  minReplicas: 1
  maxReplicas: 100
  targetCPUUtilizationPercentage: 80
  # targetMemoryUtilizationPercentage: 80
  ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
  customRules: []
    # - type: Pods
    #   pods:
    #     metric:
    #       name: packets-per-second
    #     target:
    #       type: AverageValue
    #       averageValue: 1k
  ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
  # behavior:
  #   scaleDown:
  #     policies:
  #       - type: Pods
  #         value: 4
  #         periodSeconds: 60
  #       - type: Percent
  #         value: 10
  #         periodSeconds: 60


priorityClassName: "system-node-critical"

nodeSelector: {}

## Node tolerations for server scheduling to nodes with taints
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
##
tolerations:
 - operator: Exists

## Affinity and anti-affinity
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
##
affinity: {}

## Annotations to be added to fluentd DaemonSet/Deployment
##
annotations: {}

## Labels to be added to fluentd DaemonSet/Deployment
##
labels:
  bu: dsgpu
  team: sre
  type: fluentd
  service: fluentd-dsgpu-a-prd
  priority: p0
  env: prd

## Annotations to be added to fluentd pods
##
podAnnotations: {}

## Labels to be added to fluentd pods
##
podLabels:
  bu: dsgpu
  team: sre
  type: fluentd
  service: fluentd-dsgpu-a-prd
  priority: p0
  env: prd


## How long (in seconds) a pods needs to be stable before progressing the deployment
##
minReadySeconds:

## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
##
terminationGracePeriodSeconds:

## Deployment strategy / DaemonSet updateStrategy
##
updateStrategy:
  type: RollingUpdate
  rollingUpdate:
    maxUnavailable: 25%
    maxSurge: 0

## Additional environment variables to set for fluentd pods
## Additional environment variables to set for fluentd pods
env:
- name: APP_NAME
  value: namespace_name
- name: SUB_SYSTEM
  value: container_name
# - name: FLUENTD_CONF
#   value: "../../etc/fluent/fluent.conf"
- name: APP_NAME_SYSTEMD
  value: systemd
- name: SUB_SYSTEM_SYSTEMD
  value: kubelet.service
- name: ENDPOINT
  value: ingress.coralogixsg.com
- name: LOG_LEVEL
  value: error
- name: TZ
  value: "Asia/Kolkata"
- name: K8S_NODE_NAME
  valueFrom:
    fieldRef:
      fieldPath: spec.nodeName


externalSecret:
  secretStoreRef:
    name: vault-backend
    path: meesho/prd/cntr/devop/coralogix-keys

# externalSecret:
#   enabled: true
#   key: dev/devops/coralogix
#   secretStoreRef:
#     name: vault-backend

envFrom:
# - secretRef:
#     name: integrations-privatekey
- secretRef:
    name: es-password

initContainers: []

## Name of the configMap containing a custom fluentd.conf configuration file to use instead of the default.
# mainConfigMapNameOverride: ""

## Name of the configMap containing files to be placed under /etc/fluent/config.d/
## NOTE: This will replace ALL default files in the aforementioned path!
# extraFilesConfigMapNameOverride: ""

mountVarLogDirectory: true
mountDockerContainersDirectory: true

volumes: []
# - name: varlog
#   hostPath:
#     path: /var/log
# - name: varlibdockercontainers
#   hostPath:
#     path: /var/lib/docker/containers
# - name: etcfluentd-main
#   configMap:
#     name: fluentd-main
#     defaultMode: 0777
# - name: etcfluentd-config
#   configMap:
#     name: fluentd-config
#     defaultMode: 0777

volumeMounts: []
# - name: varlog
#   mountPath: /var/log
# - name: varlibdockercontainers
#   mountPath: /var/lib/docker/containers
#   readOnly: true
# - name: etcfluentd-main
#   mountPath: /etc/fluent
# - name: etcfluentd-config
#   mountPath: /etc/fluent/config.d/

## Only available if kind is StatefulSet
## Fluentd persistence
##
persistence:
  enabled: false
  storageClass: ""
  accessMode: ReadWriteOnce
  size: 10Gi

## Fluentd service
##
service:
  enabled: true
  type: "ClusterIP"
  annotations: {}
  # loadBalancerIP:
  # externalTrafficPolicy: Local
  ports: []
  # - name: "forwarder"
  #   protocol: TCP
  #   containerPort: 24224

## Prometheus Monitoring
##
metrics:
  serviceMonitor:
    enabled: false
    additionalLabels:
      release: prometheus-operator
    namespace: ""
    namespaceSelector: {}
    ## metric relabel configs to apply to samples before ingestion.
    ##
    metricRelabelings: []
    # - sourceLabels: [__name__]
    #   separator: ;
    #   regex: ^fluentd_output_status_buffer_(oldest|newest)_.+
    #   replacement: $1
    #   action: drop
    ## relabel configs to apply to samples after ingestion.
    ##
    relabelings: []
    # - sourceLabels: [__meta_kubernetes_pod_node_name]
    #   separator: ;
    #   regex: ^(.*)$
    #   targetLabel: nodename
    #   replacement: $1
    #   action: replace
    ## Additional serviceMonitor config
    ##
    # jobLabel: fluentd
    # scrapeInterval: 30s
    # scrapeTimeout: 5s
    # honorLabels: true

  prometheusRule:
    enabled: false
    additionalLabels: {}
    namespace: ""
    rules: []
    # - alert: FluentdDown
    #   expr: up{job="fluentd"} == 0
    #   for: 5m
    #   labels:
    #     context: fluentd
    #     severity: warning
    #   annotations:
    #     summary: "Fluentd Down"
    #     description: "{{ $labels.pod }} on {{ $labels.nodename }} is down"
    # - alert: FluentdScrapeMissing
    #   expr: absent(up{job="fluentd"} == 1)
    #   for: 15m
    #   labels:
    #     context: fluentd
    #     severity: warning
    #   annotations:
    #     summary: "Fluentd Scrape Missing"
    #     description: "Fluentd instance has disappeared from Prometheus target discovery"

## Grafana Monitoring Dashboard
##
dashboards:
  enabled: "true"
  namespace: ""
  labels:
    grafana_dashboard: '"1"'

## Fluentd list of plugins to install
##
plugins: []
# - fluent-plugin-out-http

## Add fluentd config files from K8s configMaps
##
configMapConfigs:
  - fluentd-prometheus-conf
#  - fluentd-systemd-conf

## Fluentd configurations:
##
fileConfigs:
  01_sources.conf: |-
    <source>
      @type systemd
      path /var/log/journal
      tag sys-log
      read_from_head true
    </source>
    <source>
      @id fluentd-containers.log
      @type tail
      encoding utf-8
      path /var/log/containers/*.log
      pos_file /var/log/containers.log.pos
      exclude_path ["/var/log/containers/*telegraf*.log","/var/log/containers/*opentelemetry*.log","/var/log/containers/*fluentbit*.log","/var/log/containers/*gke-metrics*.log","/var/log/containers/*event-exporter-gke*.log","/var/log/containers/*metadata-server*.log","/var/log/containers/*metrics-server*.log","/var/log/containers/*filestore-node*.log"]
      path_key filename
      tag raw.containers.*
      read_from_head true
      <parse>
        @type multi_format
        <pattern>
          format json
          time_key time
          time_format %Y-%m-%dT%H:%M:%S.%NZ
          keep_time_key true
        </pattern>
        <pattern>
          format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
          time_format %Y-%m-%dT%H:%M:%S.%N%:z
          keep_time_key true
        </pattern>
        <pattern>
          format /^(?<log>fsp\s+.+)$/
          ignorecase false
          multiline false
        </pattern>
      </parse>
    </source>
    <match raw.containers.**>
      @id raw.containers
      @type detect_exceptions
      remove_tag_prefix raw
      message log
      stream stream
      multiline_flush_interval 5
      max_bytes 500000
      max_lines 1000
    </match>

    <filter containers.**>
      @type kubernetes_metadata
    </filter>

    <filter containers.**>
      @type record_transformer
      enable_ruby true
      <record>
        container_id ${record.dig("docker", "container_id")}
      </record>
    </filter>

    <match containers.**>
      @type rewrite_tag_filter
      <rule>
        key $.kubernetes.namespace_name
        pattern ^(.+)$
        tag $1.${tag}
      </rule>
    </match>

  02_filters.conf: |-

  03_dispatch.conf: |-
    <match {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,sys-log,**victoriametrics**}>
      @type "relabel"
      @label @NOCONCATDISPATCH
    </match>
    <match {**prd**,**int**}>
      @type "relabel"
      @label @CONCATDISPATCH
    </match>

    <label @CONCATDISPATCH>
      <filter {**prd**,**int**}>
          @type concat
          key log
          stream_identity_key container_id
          multiline_start_regexp /^.+\d{2}\:\d{2}\:\d{2}\.\d{3}/
          separator "\n"
          flush_interval 30
          timeout_label @DISPATCH
      </filter>
      <match **>
        @type "relabel"
        @label @DISPATCH
      </match>
    </label> 

    <label @DISPATCH>
      <match {**prd**}>
        @type copy
        <store>
          @type "gcs"
          bucket "gcs-infr-dvps-meesho-logs-prd"
          path "dsgpu/${tag[0]}/dt=%d.%m.%Y/hr=%H/${tag[5]}"
          time_slice_format %Y%m%d%H
          object_key_format "%{path}/%{time_slice}_%{index}.%{file_extension}"
          store_as "gzip"
          <format>
            @type "json"
            localtime true
          </format>
          <buffer tag,time>
            timekey 30m
            @type "file"
            path "/tmp/td-agent/buffer/"
            flush_thread_count 8
            timekey_wait 10m
            chunk_limit_size 50m
            flush_at_shutdown true
          </buffer>
        </store>
        <store>
          @type "relabel"
          @label @PRD
        </store>
      </match>
      <match {**int**}>
        @type copy
        <store>
          @type "relabel"
          @label @PREPROD
        </store>
      </match>
    </label>

    <label @PRD>
      <filter {**prd**}>
        @type grep
        <regexp>
          key log
          pattern /ERROR|WARN|error|warn/
        </regexp>
      </filter>
      <filter  {**prd**} >
        @type parser
        key_name log
        reserve_time true
        reserve_data true
        <parse>
            @type multi_format
            <pattern>
            format regexp
            expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
            </pattern>
            <pattern>
                format /^(?<log>.*)$/
            </pattern>
        </parse>
      </filter>
      <filter  {**prd**}>
        @type record_transformer
        enable_ruby true
        auto_typecast true
        renew_record true
        <record>
          log_level ${record.dig("log_level")}
          logger ${record.dig("logger")}
          method ${record.dig("method")}
          line_number ${record.dig("line_number")}
          pid ${record.dig("pid")}
          thread ${record.dig("thread")}
          x3_trace_id ${record.dig("x3_trace_id")}
          x3_span_id ${record.dig("x3_span_id")}
          trace_id ${record.dig("trace_id")}
          span_id ${record.dig("span_id")}
          user_id ${record.dig("user_id")}
          extras ${record.dig("extras")}
          message ${record.dig("log")}
          throwable ${record.dig("throwable")}
          application_name ${record.dig("kubernetes", "namespace_name")}
          service_name ${record.dig("kubernetes", "container_name")}
          host ${record.dig("kubernetes", "host")}
          pod_name ${record.dig("kubernetes", "pod_name")}
          pod_ip ${record.dig("kubernetes", "pod_ip")}
        </record>
      </filter>
      <filter  {**prd**}>
        @type prometheus
        <metric>
          name fluentd_log_count_total
          type counter
          desc Total number of log entries processed by Fluentd (prd namespace breakdown)
          <labels>
            application_name $.application_name
            log_level $.log_level
            pod_name $.pod_name
          </labels>
        </metric>
      </filter>
      <match {**prd**}>
        @type elasticsearch
        host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
        port 9200
        scheme http
        compression_level best_speed
        ssl_verify false
        emit_error_for_missing_id true
        id_key request_id 
        user elastic
        password "#{ENV['ECK_ES_PASSWORD']}"
        index_name gcp-prd
        reload_on_failure true
        reconnect_on_error true
        include_timestamp true
        request_timeout 60s
      </match>
    </label>

    <label @NOCONCATDISPATCH>
    <filter sys-log >
        @type record_transformer
        enable_ruby true
        auto_typecast true
        renew_record true
        renew_time_key ${record.dig("SYSLOG_TIMESTAMP")}
        <record>
          cluster_name "dsgpu-prd"
          application_name "systemd"
          node_name ${record.dig("_HOSTNAME")}
          text ${record.to_json}
        </record>
      </filter>
      <filter {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,**victoriametrics**}>
        @type record_transformer
        enable_ruby true
        auto_typecast true
        renew_record true
        <record>
          application_name ${record.dig("kubernetes", "namespace_name")}
          service_name ${record.dig("kubernetes", "container_name")}
          host ${record.dig("kubernetes", "host")}
          pod_name ${record.dig("kubernetes", "pod_name")}
          pod_ip ${record.dig("kubernetes", "pod_ip")}
          text ${record.dig("log")}
        </record>
      </filter>
      <match {**gke-mcs**}>
        @type elasticsearch
        host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
        port 9200
        scheme http
        compression_level best_speed
        ssl_verify false
        emit_error_for_missing_id true
        id_key request_id 
        user elastic
        index_name gke-mcs
        password "#{ENV['ECK_ES_PASSWORD']}"
        reload_on_failure true
        reconnect_on_error true
        include_timestamp true
        request_timeout 60s
      </match>
      <match {**kube-events**,**kube-system**,**external-secrets**,**keda**,sys-log,**victoriametrics**}>
        @type elasticsearch
        host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
        port 9200
        scheme http
        compression_level best_speed
        ssl_verify false
        emit_error_for_missing_id true
        id_key request_id 
        user elastic
        index_name gcp-kubeevents
        password "#{ENV['ECK_ES_PASSWORD']}"
        reload_on_failure true
        reconnect_on_error true
        include_timestamp true
        request_timeout 60s
      </match>
      <match **>
        @type elasticsearch
        host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
        port 9200
        scheme http
        compression_level best_speed
        ssl_verify false
        emit_error_for_missing_id true
        id_key request_id 
        user elastic
        index_name gcp-prd
        password "#{ENV['ECK_ES_PASSWORD']}"
        reload_on_failure true
        reconnect_on_error true
        include_timestamp true
        request_timeout 60s
      </match>
    </label>

    <label @PREPROD>
      <filter {**int**}>
        @type grep
        <regexp>
          key log
          pattern /INFO|ERROR|WARN|info|error|warn/
        </regexp>
      </filter>
      <filter  {**int**} >
        @type parser
        key_name log
        reserve_time true
        reserve_data true
        <parse>
            @type multi_format
            <pattern>
            format regexp
            expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
            </pattern>
            <pattern>
                format /^(?<log>.*)$/
            </pattern>
        </parse>
      </filter>
      <filter  {**int**}>
        @type record_transformer
        enable_ruby true
        auto_typecast true
        renew_record true
        <record>
          log_level ${record.dig("log_level")}
          logger ${record.dig("logger")}
          method ${record.dig("method")}
          line_number ${record.dig("line_number")}
          pid ${record.dig("pid")}
          thread ${record.dig("thread")}
          x3_trace_id ${record.dig("x3_trace_id")}
          x3_span_id ${record.dig("x3_span_id")}
          trace_id ${record.dig("trace_id")}
          span_id ${record.dig("span_id")}
          user_id ${record.dig("user_id")}
          extras ${record.dig("extras")}
          message ${record.dig("log")}
          throwable ${record.dig("throwable")}
          application_name ${record.dig("kubernetes", "namespace_name")}
          service_name ${record.dig("kubernetes", "container_name")}
          host ${record.dig("kubernetes", "host")}
          pod_name ${record.dig("kubernetes", "pod_name")}
          pod_ip ${record.dig("kubernetes", "pod_ip")}
        </record>
      </filter>
      <match {**int**}>
        @type elasticsearch
        host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
        port 9200
        scheme http
        compression_level best_speed
        ssl_verify false
        emit_error_for_missing_id true
        id_key request_id 
        user elastic
        index_name gcp-int
        password "#{ENV['ECK_ES_PASSWORD']}"
        reload_on_failure true
        reconnect_on_error true
        include_timestamp true
        request_timeout 60s
      </match>
    </label>
  04_outputs.conf: |-