731 lines
20 KiB
YAML
731 lines
20 KiB
YAML
nameOverride: ""
|
|
fullnameOverride: ""
|
|
|
|
# DaemonSet, Deployment or StatefulSet
|
|
kind: "DaemonSet"
|
|
# azureblob, cloudwatch, elasticsearch7, elasticsearch8, gcs, graylog , kafka, kafka2, kinesis, opensearch
|
|
variant: gcs
|
|
# # Only applicable for Deployment or StatefulSet
|
|
# replicaCount: 1
|
|
|
|
image:
|
|
repository: "asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/sre/fluentd-v2"
|
|
pullPolicy: "Always"
|
|
tag: "edge-debian"
|
|
|
|
## Optional array of imagePullSecrets containing private registry credentials
|
|
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
|
imagePullSecrets: []
|
|
|
|
serviceAccount:
|
|
create: true
|
|
annotations: {
|
|
iam.gke.io/gcp-service-account: sa-deng-desre-fluentd-prd@meesho-dataengg-prd-0622.iam.gserviceaccount.com
|
|
}
|
|
name: null
|
|
|
|
rbac:
|
|
create: true
|
|
|
|
# from Kubernetes 1.25, PSP is deprecated
|
|
# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
|
|
# We automatically disable PSP if Kubernetes version is 1.25 or higher
|
|
podSecurityPolicy:
|
|
enabled: true
|
|
annotations: {}
|
|
|
|
## Security Context policies for controller pods
|
|
## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
|
|
## notes on enabling and using sysctls
|
|
##
|
|
podSecurityContext: {}
|
|
# seLinuxOptions:
|
|
# type: "spc_t"
|
|
|
|
securityContext: {}
|
|
# capabilities:
|
|
# drop:
|
|
# - ALL
|
|
# readOnlyRootFilesystem: true
|
|
# runAsNonRoot: true
|
|
# runAsUser: 1000
|
|
|
|
# Configure the livecycle
|
|
# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
|
lifecycle: {}
|
|
# preStop:
|
|
# exec:
|
|
# command: ["/bin/sh", "-c", "sleep 20"]
|
|
|
|
# Configure the livenessProbe
|
|
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
|
#livenessProbe:
|
|
# httpGet:
|
|
# path: /metrics
|
|
# port: metrics
|
|
# initialDelaySeconds: 0
|
|
# periodSeconds: 10
|
|
# timeoutSeconds: 1
|
|
# successThreshold: 1
|
|
# failureThreshold: 3
|
|
|
|
# Configure the readinessProbe
|
|
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
|
#readinessProbe:
|
|
# httpGet:
|
|
# path: /metrics
|
|
# port: metrics
|
|
# initialDelaySeconds: 0
|
|
# periodSeconds: 10
|
|
# timeoutSeconds: 1
|
|
# successThreshold: 1
|
|
# failureThreshold: 3
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 50Mi
|
|
limits:
|
|
memory: 2000Mi
|
|
cpu: 2000m
|
|
|
|
## only available if kind is Deployment
|
|
autoscaling:
|
|
enabled: false
|
|
minReplicas: 1
|
|
maxReplicas: 100
|
|
targetCPUUtilizationPercentage: 80
|
|
# targetMemoryUtilizationPercentage: 80
|
|
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
|
|
customRules: []
|
|
# - type: Pods
|
|
# pods:
|
|
# metric:
|
|
# name: packets-per-second
|
|
# target:
|
|
# type: AverageValue
|
|
# averageValue: 1k
|
|
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
|
|
# behavior:
|
|
# scaleDown:
|
|
# policies:
|
|
# - type: Pods
|
|
# value: 4
|
|
# periodSeconds: 60
|
|
# - type: Percent
|
|
# value: 10
|
|
# periodSeconds: 60
|
|
|
|
|
|
priorityClassName: "system-node-critical"
|
|
|
|
nodeSelector: {}
|
|
|
|
## Node tolerations for server scheduling to nodes with taints
|
|
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
|
|
##
|
|
tolerations:
|
|
- operator: Exists
|
|
|
|
## Affinity and anti-affinity
|
|
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
|
|
##
|
|
affinity: {}
|
|
|
|
## Annotations to be added to fluentd DaemonSet/Deployment
|
|
##
|
|
annotations: {}
|
|
|
|
## Labels to be added to fluentd DaemonSet/Deployment
|
|
##
|
|
labels:
|
|
bu: dataengg
|
|
team: sre
|
|
type: fluentd
|
|
service: fluentd-dataengg-prd
|
|
priority: p0
|
|
env: prd
|
|
|
|
## Annotations to be added to fluentd pods
|
|
##
|
|
podAnnotations: {}
|
|
|
|
## Labels to be added to fluentd pods
|
|
##
|
|
podLabels:
|
|
bu: dataengg
|
|
team: sre
|
|
type: fluentd
|
|
service: fluentd-dataengg-prd
|
|
priority: p0
|
|
env: prd
|
|
|
|
|
|
## How long (in seconds) a pods needs to be stable before progressing the deployment
|
|
##
|
|
minReadySeconds:
|
|
|
|
## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
|
|
##
|
|
terminationGracePeriodSeconds:
|
|
|
|
## Deployment strategy / DaemonSet updateStrategy
|
|
##
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxUnavailable: 25%
|
|
maxSurge: 0
|
|
|
|
## Additional environment variables to set for fluentd pods
|
|
## Additional environment variables to set for fluentd pods
|
|
env:
|
|
- name: APP_NAME
|
|
value: namespace_name
|
|
- name: SUB_SYSTEM
|
|
value: container_name
|
|
# - name: FLUENTD_CONF
|
|
# value: "../../etc/fluent/fluent.conf"
|
|
- name: APP_NAME_SYSTEMD
|
|
value: systemd
|
|
- name: SUB_SYSTEM_SYSTEMD
|
|
value: kubelet.service
|
|
- name: ENDPOINT
|
|
value: ingress.coralogixsg.com
|
|
- name: LOG_LEVEL
|
|
value: error
|
|
- name: TZ
|
|
value: "Asia/Kolkata"
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
|
|
externalSecret:
|
|
secretStoreRef:
|
|
name: vault-backend
|
|
path: meesho/prd/cntr/devop/coralogix-keys
|
|
|
|
# externalSecret:
|
|
# enabled: true
|
|
# key: dev/devops/coralogix
|
|
# secretStoreRef:
|
|
# name: vault-backend
|
|
|
|
envFrom:
|
|
- secretRef:
|
|
name: integrations-privatekey
|
|
- secretRef:
|
|
name: es-password
|
|
|
|
initContainers: []
|
|
|
|
## Name of the configMap containing a custom fluentd.conf configuration file to use instead of the default.
|
|
# mainConfigMapNameOverride: ""
|
|
|
|
## Name of the configMap containing files to be placed under /etc/fluent/config.d/
|
|
## NOTE: This will replace ALL default files in the aforementioned path!
|
|
# extraFilesConfigMapNameOverride: ""
|
|
|
|
mountVarLogDirectory: true
|
|
mountDockerContainersDirectory: true
|
|
|
|
volumes: []
|
|
# - name: varlog
|
|
# hostPath:
|
|
# path: /var/log
|
|
# - name: varlibdockercontainers
|
|
# hostPath:
|
|
# path: /var/lib/docker/containers
|
|
# - name: etcfluentd-main
|
|
# configMap:
|
|
# name: fluentd-main
|
|
# defaultMode: 0777
|
|
# - name: etcfluentd-config
|
|
# configMap:
|
|
# name: fluentd-config
|
|
# defaultMode: 0777
|
|
|
|
volumeMounts: []
|
|
# - name: varlog
|
|
# mountPath: /var/log
|
|
# - name: varlibdockercontainers
|
|
# mountPath: /var/lib/docker/containers
|
|
# readOnly: true
|
|
# - name: etcfluentd-main
|
|
# mountPath: /etc/fluent
|
|
# - name: etcfluentd-config
|
|
# mountPath: /etc/fluent/config.d/
|
|
|
|
## Only available if kind is StatefulSet
|
|
## Fluentd persistence
|
|
##
|
|
persistence:
|
|
enabled: false
|
|
storageClass: ""
|
|
accessMode: ReadWriteOnce
|
|
size: 10Gi
|
|
|
|
## Fluentd service
|
|
##
|
|
service:
|
|
enabled: true
|
|
type: "ClusterIP"
|
|
annotations: {}
|
|
# loadBalancerIP:
|
|
# externalTrafficPolicy: Local
|
|
ports: []
|
|
# - name: "forwarder"
|
|
# protocol: TCP
|
|
# containerPort: 24224
|
|
|
|
## Prometheus Monitoring
|
|
##
|
|
metrics:
|
|
serviceMonitor:
|
|
enabled: false
|
|
additionalLabels:
|
|
release: prometheus-operator
|
|
namespace: ""
|
|
namespaceSelector: {}
|
|
## metric relabel configs to apply to samples before ingestion.
|
|
##
|
|
metricRelabelings: []
|
|
# - sourceLabels: [__name__]
|
|
# separator: ;
|
|
# regex: ^fluentd_output_status_buffer_(oldest|newest)_.+
|
|
# replacement: $1
|
|
# action: drop
|
|
## relabel configs to apply to samples after ingestion.
|
|
##
|
|
relabelings: []
|
|
# - sourceLabels: [__meta_kubernetes_pod_node_name]
|
|
# separator: ;
|
|
# regex: ^(.*)$
|
|
# targetLabel: nodename
|
|
# replacement: $1
|
|
# action: replace
|
|
## Additional serviceMonitor config
|
|
##
|
|
# jobLabel: fluentd
|
|
# scrapeInterval: 30s
|
|
# scrapeTimeout: 5s
|
|
# honorLabels: true
|
|
|
|
prometheusRule:
|
|
enabled: false
|
|
additionalLabels: {}
|
|
namespace: ""
|
|
rules: []
|
|
# - alert: FluentdDown
|
|
# expr: up{job="fluentd"} == 0
|
|
# for: 5m
|
|
# labels:
|
|
# context: fluentd
|
|
# severity: warning
|
|
# annotations:
|
|
# summary: "Fluentd Down"
|
|
# description: "{{ $labels.pod }} on {{ $labels.nodename }} is down"
|
|
# - alert: FluentdScrapeMissing
|
|
# expr: absent(up{job="fluentd"} == 1)
|
|
# for: 15m
|
|
# labels:
|
|
# context: fluentd
|
|
# severity: warning
|
|
# annotations:
|
|
# summary: "Fluentd Scrape Missing"
|
|
# description: "Fluentd instance has disappeared from Prometheus target discovery"
|
|
|
|
## Grafana Monitoring Dashboard
|
|
##
|
|
dashboards:
|
|
enabled: "true"
|
|
namespace: ""
|
|
labels:
|
|
grafana_dashboard: '"1"'
|
|
|
|
## Fluentd list of plugins to install
|
|
##
|
|
plugins: []
|
|
# - fluent-plugin-out-http
|
|
|
|
## Add fluentd config files from K8s configMaps
|
|
##
|
|
configMapConfigs:
|
|
- fluentd-prometheus-conf
|
|
# - fluentd-systemd-conf
|
|
|
|
## Fluentd configurations:
|
|
##
|
|
fileConfigs:
|
|
01_sources.conf: |-
|
|
<source>
|
|
@type systemd
|
|
path /var/log/journal
|
|
tag sys-log
|
|
read_from_head true
|
|
</source>
|
|
<source>
|
|
@id fluentd-containers.log
|
|
@type tail
|
|
encoding utf-8
|
|
from_encoding "utf-8"
|
|
path /var/log/containers/*.log
|
|
pos_file /var/log/containers.log.pos
|
|
exclude_path ["/var/log/containers/*telegraf*.log","/var/log/containers/*opentelemetry*.log","/var/log/containers/*fluentbit*.log","/var/log/containers/*gke-metrics*.log","/var/log/containers/*event-exporter-gke*.log","/var/log/containers/*metadata-server*.log","/var/log/containers/*metrics-server*.log","/var/log/containers/*filestore-node*.log"]
|
|
path_key filename
|
|
tag raw.containers.*
|
|
read_from_head true
|
|
<parse>
|
|
@type multi_format
|
|
<pattern>
|
|
format json
|
|
time_key time
|
|
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
keep_time_key true
|
|
</pattern>
|
|
<pattern>
|
|
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
|
|
time_format %Y-%m-%dT%H:%M:%S.%N%:z
|
|
keep_time_key true
|
|
</pattern>
|
|
<pattern>
|
|
format /^(?<log>fsp\s+.+)$/
|
|
ignorecase false
|
|
multiline false
|
|
</pattern>
|
|
</parse>
|
|
</source>
|
|
<match raw.containers.**>
|
|
@id raw.containers
|
|
@type detect_exceptions
|
|
remove_tag_prefix raw
|
|
message log
|
|
stream stream
|
|
multiline_flush_interval 5
|
|
max_bytes 500000
|
|
max_lines 1000
|
|
</match>
|
|
|
|
<filter containers.**>
|
|
@type kubernetes_metadata
|
|
</filter>
|
|
|
|
<filter containers.**>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
<record>
|
|
container_id ${record.dig("docker", "container_id")}
|
|
</record>
|
|
</filter>
|
|
|
|
<match containers.**>
|
|
@type rewrite_tag_filter
|
|
<rule>
|
|
key $.kubernetes.namespace_name
|
|
pattern ^(.+)$
|
|
tag $1.${tag}
|
|
</rule>
|
|
</match>
|
|
|
|
02_filters.conf: |-
|
|
|
|
03_dispatch.conf: |-
|
|
<match {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,**os-trino-shared-bk1**,**os-prismsdk-backend-bk3**,sys-log,**victoriametrics**}>
|
|
@type "relabel"
|
|
@label @NOCONCATDISPATCH
|
|
</match>
|
|
<match {**prd**,**int**}>
|
|
@type "relabel"
|
|
@label @CONCATDISPATCH
|
|
</match>
|
|
|
|
<label @CONCATDISPATCH>
|
|
<filter {**prd**,**int**}>
|
|
@type concat
|
|
key log
|
|
stream_identity_key container_id
|
|
multiline_start_regexp /^.+\d{2}\:\d{2}\:\d{2}\.\d{3}/
|
|
separator "\n"
|
|
flush_interval 30
|
|
timeout_label @DISPATCH
|
|
</filter>
|
|
<match **>
|
|
@type "relabel"
|
|
@label @DISPATCH
|
|
</match>
|
|
</label>
|
|
|
|
<label @DISPATCH>
|
|
<match {**prd**,**int-spark-operator**}>
|
|
@type copy
|
|
<store>
|
|
@type "gcs"
|
|
bucket "gcs-infr-dvps-meesho-logs-prd"
|
|
path "dataengg/${tag[0]}/dt=%d.%m.%Y/hr=%H/${tag[5]}"
|
|
time_slice_format %Y%m%d%H
|
|
object_key_format "%{path}/%{time_slice}_%{index}.%{file_extension}"
|
|
store_as "gzip"
|
|
<format>
|
|
@type "json"
|
|
localtime true
|
|
</format>
|
|
<buffer tag,time>
|
|
timekey 30m
|
|
@type "file"
|
|
path "/tmp/td-agent/buffer/"
|
|
flush_thread_count 8
|
|
timekey_wait 10m
|
|
chunk_limit_size 50m
|
|
flush_at_shutdown true
|
|
</buffer>
|
|
</store>
|
|
<store>
|
|
@type "relabel"
|
|
@label @PRD
|
|
</store>
|
|
</match>
|
|
<match {**int**}>
|
|
@type copy
|
|
<store>
|
|
@type "relabel"
|
|
@label @PREPROD
|
|
</store>
|
|
</match>
|
|
</label>
|
|
|
|
<label @PRD>
|
|
<filter {**prd**}>
|
|
@type grep
|
|
<regexp>
|
|
key log
|
|
pattern /ERROR|WARN|error|warn/
|
|
</regexp>
|
|
</filter>
|
|
<filter {**prd**} >
|
|
@type parser
|
|
key_name log
|
|
reserve_time true
|
|
reserve_data true
|
|
<parse>
|
|
@type multi_format
|
|
<pattern>
|
|
format regexp
|
|
expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
|
|
</pattern>
|
|
<pattern>
|
|
format /^(?<log>.*)$/
|
|
</pattern>
|
|
</parse>
|
|
</filter>
|
|
<filter {**prd**}>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
auto_typecast true
|
|
renew_record true
|
|
<record>
|
|
log_level ${record.dig("log_level")}
|
|
logger ${record.dig("logger")}
|
|
method ${record.dig("method")}
|
|
line_number ${record.dig("line_number")}
|
|
pid ${record.dig("pid")}
|
|
thread ${record.dig("thread")}
|
|
x3_trace_id ${record.dig("x3_trace_id")}
|
|
x3_span_id ${record.dig("x3_span_id")}
|
|
trace_id ${record.dig("trace_id")}
|
|
span_id ${record.dig("span_id")}
|
|
user_id ${record.dig("user_id")}
|
|
extras ${record.dig("extras")}
|
|
message ${record.dig("log")}
|
|
throwable ${record.dig("throwable")}
|
|
application_name ${record.dig("kubernetes", "namespace_name")}
|
|
service_name ${record.dig("kubernetes", "container_name")}
|
|
host ${record.dig("kubernetes", "host")}
|
|
pod_name ${record.dig("kubernetes", "pod_name")}
|
|
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
|
</record>
|
|
</filter>
|
|
<filter {**prd**}>
|
|
@type prometheus
|
|
<metric>
|
|
name fluentd_log_count_total
|
|
type counter
|
|
desc Total number of log entries processed by Fluentd (prd namespace breakdown)
|
|
<labels>
|
|
application_name $.application_name
|
|
log_level $.log_level
|
|
pod_name $.pod_name
|
|
</labels>
|
|
</metric>
|
|
</filter>
|
|
<match {**prd**}>
|
|
@type elasticsearch
|
|
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
|
port 9200
|
|
scheme http
|
|
compression_level best_speed
|
|
ssl_verify false
|
|
emit_error_for_missing_id true
|
|
id_key request_id
|
|
user elastic
|
|
password "#{ENV['ECK_ES_PASSWORD']}"
|
|
index_name gcp-prd
|
|
reload_on_failure true
|
|
reconnect_on_error true
|
|
include_timestamp true
|
|
request_timeout 60s
|
|
</match>
|
|
</label>
|
|
|
|
<label @NOCONCATDISPATCH>
|
|
<filter sys-log >
|
|
@type record_transformer
|
|
enable_ruby true
|
|
auto_typecast true
|
|
renew_record true
|
|
renew_time_key ${record.dig("SYSLOG_TIMESTAMP")}
|
|
<record>
|
|
cluster_name "dataengg-prd"
|
|
application_name "systemd"
|
|
node_name ${record.dig("_HOSTNAME")}
|
|
text ${record.to_json}
|
|
</record>
|
|
</filter>
|
|
<filter {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,**os-trino-shared-bk1**,**os-prismsdk-backend-bk3**,**victoriametrics**}>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
auto_typecast true
|
|
renew_record true
|
|
<record>
|
|
application_name ${record.dig("kubernetes", "namespace_name")}
|
|
service_name ${record.dig("kubernetes", "container_name")}
|
|
host ${record.dig("kubernetes", "host")}
|
|
pod_name ${record.dig("kubernetes", "pod_name")}
|
|
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
|
text ${record.dig("log")}
|
|
</record>
|
|
</filter>
|
|
<match {**gke-mcs**}>
|
|
@type elasticsearch
|
|
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
|
port 9200
|
|
scheme http
|
|
compression_level best_speed
|
|
ssl_verify false
|
|
emit_error_for_missing_id true
|
|
id_key request_id
|
|
user elastic
|
|
index_name gke-mcs
|
|
password "#{ENV['ECK_ES_PASSWORD']}"
|
|
reload_on_failure true
|
|
reconnect_on_error true
|
|
include_timestamp true
|
|
request_timeout 60s
|
|
</match>
|
|
<match {**kube-events**,**kube-system**,**external-secrets**,**keda**,sys-log,**victoriametrics**}>
|
|
@type elasticsearch
|
|
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
|
port 9200
|
|
scheme http
|
|
compression_level best_speed
|
|
ssl_verify false
|
|
emit_error_for_missing_id true
|
|
id_key request_id
|
|
user elastic
|
|
index_name gcp-kubeevents
|
|
password "#{ENV['ECK_ES_PASSWORD']}"
|
|
reload_on_failure true
|
|
reconnect_on_error true
|
|
include_timestamp true
|
|
request_timeout 60s
|
|
</match>
|
|
<match **>
|
|
@type elasticsearch
|
|
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
|
port 9200
|
|
scheme http
|
|
compression_level best_speed
|
|
ssl_verify false
|
|
emit_error_for_missing_id true
|
|
id_key request_id
|
|
user elastic
|
|
index_name gcp-prd
|
|
password "#{ENV['ECK_ES_PASSWORD']}"
|
|
reload_on_failure true
|
|
reconnect_on_error true
|
|
include_timestamp true
|
|
request_timeout 60s
|
|
</match>
|
|
</label>
|
|
<label @PREPROD>
|
|
<filter {**int**}>
|
|
@type grep
|
|
<regexp>
|
|
key log
|
|
pattern /INFO|ERROR|WARN|info|error|warn/
|
|
</regexp>
|
|
</filter>
|
|
<filter {**int**} >
|
|
@type parser
|
|
key_name log
|
|
reserve_time true
|
|
reserve_data true
|
|
<parse>
|
|
@type multi_format
|
|
<pattern>
|
|
format regexp
|
|
expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
|
|
</pattern>
|
|
<pattern>
|
|
format /^(?<log>.*)$/
|
|
</pattern>
|
|
</parse>
|
|
</filter>
|
|
<filter {**int**}>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
auto_typecast true
|
|
renew_record true
|
|
<record>
|
|
log_level ${record.dig("log_level")}
|
|
logger ${record.dig("logger")}
|
|
method ${record.dig("method")}
|
|
line_number ${record.dig("line_number")}
|
|
pid ${record.dig("pid")}
|
|
thread ${record.dig("thread")}
|
|
x3_trace_id ${record.dig("x3_trace_id")}
|
|
x3_span_id ${record.dig("x3_span_id")}
|
|
trace_id ${record.dig("trace_id")}
|
|
span_id ${record.dig("span_id")}
|
|
user_id ${record.dig("user_id")}
|
|
extras ${record.dig("extras")}
|
|
message ${record.dig("log")}
|
|
throwable ${record.dig("throwable")}
|
|
application_name ${record.dig("kubernetes", "namespace_name")}
|
|
service_name ${record.dig("kubernetes", "container_name")}
|
|
host ${record.dig("kubernetes", "host")}
|
|
pod_name ${record.dig("kubernetes", "pod_name")}
|
|
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
|
</record>
|
|
</filter>
|
|
<match {**int**}>
|
|
@type elasticsearch
|
|
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
|
port 9200
|
|
scheme http
|
|
compression_level best_speed
|
|
ssl_verify false
|
|
emit_error_for_missing_id true
|
|
id_key request_id
|
|
user elastic
|
|
index_name gcp-int
|
|
password "#{ENV['ECK_ES_PASSWORD']}"
|
|
reload_on_failure true
|
|
reconnect_on_error true
|
|
include_timestamp true
|
|
request_timeout 60s
|
|
</match>
|
|
</label>
|
|
|
|
04_outputs.conf: |-
|