added repo
This commit is contained in:
Your Name
@@ -0,0 +1,731 @@
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
# DaemonSet, Deployment or StatefulSet
|
||||
kind: "DaemonSet"
|
||||
# azureblob, cloudwatch, elasticsearch7, elasticsearch8, gcs, graylog , kafka, kafka2, kinesis, opensearch
|
||||
variant: gcs
|
||||
# # Only applicable for Deployment or StatefulSet
|
||||
# replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: "asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/sre/fluentd-v2"
|
||||
pullPolicy: "Always"
|
||||
tag: "edge-debian"
|
||||
|
||||
## Optional array of imagePullSecrets containing private registry credentials
|
||||
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||
imagePullSecrets: []
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {
|
||||
iam.gke.io/gcp-service-account: sa-dsci-dssre-fluentd-prd@meesho-datascience-prd-0622.iam.gserviceaccount.com
|
||||
}
|
||||
name: null
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
# from Kubernetes 1.25, PSP is deprecated
|
||||
# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
|
||||
# We automatically disable PSP if Kubernetes version is 1.25 or higher
|
||||
podSecurityPolicy:
|
||||
enabled: true
|
||||
annotations: {}
|
||||
|
||||
## Security Context policies for controller pods
|
||||
## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
|
||||
## notes on enabling and using sysctls
|
||||
##
|
||||
podSecurityContext: {}
|
||||
# seLinuxOptions:
|
||||
# type: "spc_t"
|
||||
|
||||
securityContext: {}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
# Configure the livecycle
|
||||
# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
lifecycle: {}
|
||||
# preStop:
|
||||
# exec:
|
||||
# command: ["/bin/sh", "-c", "sleep 20"]
|
||||
|
||||
# Configure the livenessProbe
|
||||
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
||||
#livenessProbe:
|
||||
# httpGet:
|
||||
# path: /metrics
|
||||
# port: metrics
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
# successThreshold: 1
|
||||
# failureThreshold: 3
|
||||
|
||||
# Configure the readinessProbe
|
||||
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
||||
#readinessProbe:
|
||||
# httpGet:
|
||||
# path: /metrics
|
||||
# port: metrics
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
# successThreshold: 1
|
||||
# failureThreshold: 3
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 50Mi
|
||||
limits:
|
||||
memory: 2000Mi
|
||||
cpu: 2000m
|
||||
|
||||
## only available if kind is Deployment
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
|
||||
customRules: []
|
||||
# - type: Pods
|
||||
# pods:
|
||||
# metric:
|
||||
# name: packets-per-second
|
||||
# target:
|
||||
# type: AverageValue
|
||||
# averageValue: 1k
|
||||
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
|
||||
# behavior:
|
||||
# scaleDown:
|
||||
# policies:
|
||||
# - type: Pods
|
||||
# value: 4
|
||||
# periodSeconds: 60
|
||||
# - type: Percent
|
||||
# value: 10
|
||||
# periodSeconds: 60
|
||||
|
||||
|
||||
priorityClassName: "system-node-critical"
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
## Node tolerations for server scheduling to nodes with taints
|
||||
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
|
||||
##
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
|
||||
## Affinity and anti-affinity
|
||||
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
|
||||
##
|
||||
affinity: {}
|
||||
|
||||
## Annotations to be added to fluentd DaemonSet/Deployment
|
||||
##
|
||||
annotations: {}
|
||||
|
||||
## Labels to be added to fluentd DaemonSet/Deployment
|
||||
##
|
||||
labels:
|
||||
bu: datascience
|
||||
team: sre
|
||||
type: fluentd
|
||||
service: fluentd-datascience-prd
|
||||
priority: p0
|
||||
env: prd
|
||||
|
||||
## Annotations to be added to fluentd pods
|
||||
##
|
||||
podAnnotations: {}
|
||||
|
||||
## Labels to be added to fluentd pods
|
||||
##
|
||||
podLabels:
|
||||
bu: datascience
|
||||
team: sre
|
||||
type: fluentd
|
||||
service: fluentd-datascience-prd
|
||||
priority: p0
|
||||
env: prd
|
||||
|
||||
|
||||
## How long (in seconds) a pods needs to be stable before progressing the deployment
|
||||
##
|
||||
minReadySeconds:
|
||||
|
||||
## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
|
||||
##
|
||||
terminationGracePeriodSeconds:
|
||||
|
||||
## Deployment strategy / DaemonSet updateStrategy
|
||||
##
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 25%
|
||||
maxSurge: 0
|
||||
|
||||
## Additional environment variables to set for fluentd pods
|
||||
## Additional environment variables to set for fluentd pods
|
||||
env:
|
||||
- name: APP_NAME
|
||||
value: namespace_name
|
||||
- name: SUB_SYSTEM
|
||||
value: container_name
|
||||
# - name: FLUENTD_CONF
|
||||
# value: "../../etc/fluent/fluent.conf"
|
||||
- name: APP_NAME_SYSTEMD
|
||||
value: systemd
|
||||
- name: SUB_SYSTEM_SYSTEMD
|
||||
value: kubelet.service
|
||||
- name: ENDPOINT
|
||||
value: ingress.coralogixsg.com
|
||||
- name: LOG_LEVEL
|
||||
value: error
|
||||
- name: TZ
|
||||
value: "Asia/Kolkata"
|
||||
- name: K8S_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
|
||||
|
||||
externalSecret:
|
||||
secretStoreRef:
|
||||
name: vault-backend
|
||||
path: meesho/prd/cntr/devop/coralogix-keys
|
||||
|
||||
# externalSecret:
|
||||
# enabled: true
|
||||
# key: dev/devops/coralogix
|
||||
# secretStoreRef:
|
||||
# name: vault-backend
|
||||
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: integrations-privatekey
|
||||
- secretRef:
|
||||
name: es-password
|
||||
|
||||
initContainers: []
|
||||
|
||||
## Name of the configMap containing a custom fluentd.conf configuration file to use instead of the default.
|
||||
# mainConfigMapNameOverride: ""
|
||||
|
||||
## Name of the configMap containing files to be placed under /etc/fluent/config.d/
|
||||
## NOTE: This will replace ALL default files in the aforementioned path!
|
||||
# extraFilesConfigMapNameOverride: ""
|
||||
|
||||
mountVarLogDirectory: true
|
||||
mountDockerContainersDirectory: true
|
||||
|
||||
volumes: []
|
||||
# - name: varlog
|
||||
# hostPath:
|
||||
# path: /var/log
|
||||
# - name: varlibdockercontainers
|
||||
# hostPath:
|
||||
# path: /var/lib/docker/containers
|
||||
# - name: etcfluentd-main
|
||||
# configMap:
|
||||
# name: fluentd-main
|
||||
# defaultMode: 0777
|
||||
# - name: etcfluentd-config
|
||||
# configMap:
|
||||
# name: fluentd-config
|
||||
# defaultMode: 0777
|
||||
|
||||
volumeMounts: []
|
||||
# - name: varlog
|
||||
# mountPath: /var/log
|
||||
# - name: varlibdockercontainers
|
||||
# mountPath: /var/lib/docker/containers
|
||||
# readOnly: true
|
||||
# - name: etcfluentd-main
|
||||
# mountPath: /etc/fluent
|
||||
# - name: etcfluentd-config
|
||||
# mountPath: /etc/fluent/config.d/
|
||||
|
||||
## Only available if kind is StatefulSet
|
||||
## Fluentd persistence
|
||||
##
|
||||
persistence:
|
||||
enabled: false
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
|
||||
## Fluentd service
|
||||
##
|
||||
service:
|
||||
enabled: true
|
||||
type: "ClusterIP"
|
||||
annotations: {}
|
||||
# loadBalancerIP:
|
||||
# externalTrafficPolicy: Local
|
||||
ports: []
|
||||
# - name: "forwarder"
|
||||
# protocol: TCP
|
||||
# containerPort: 24224
|
||||
|
||||
## Prometheus Monitoring
|
||||
##
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
additionalLabels:
|
||||
release: prometheus-operator
|
||||
namespace: ""
|
||||
namespaceSelector: {}
|
||||
## metric relabel configs to apply to samples before ingestion.
|
||||
##
|
||||
metricRelabelings: []
|
||||
# - sourceLabels: [__name__]
|
||||
# separator: ;
|
||||
# regex: ^fluentd_output_status_buffer_(oldest|newest)_.+
|
||||
# replacement: $1
|
||||
# action: drop
|
||||
## relabel configs to apply to samples after ingestion.
|
||||
##
|
||||
relabelings: []
|
||||
# - sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
# separator: ;
|
||||
# regex: ^(.*)$
|
||||
# targetLabel: nodename
|
||||
# replacement: $1
|
||||
# action: replace
|
||||
## Additional serviceMonitor config
|
||||
##
|
||||
# jobLabel: fluentd
|
||||
# scrapeInterval: 30s
|
||||
# scrapeTimeout: 5s
|
||||
# honorLabels: true
|
||||
|
||||
prometheusRule:
|
||||
enabled: false
|
||||
additionalLabels: {}
|
||||
namespace: ""
|
||||
rules: []
|
||||
# - alert: FluentdDown
|
||||
# expr: up{job="fluentd"} == 0
|
||||
# for: 5m
|
||||
# labels:
|
||||
# context: fluentd
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: "Fluentd Down"
|
||||
# description: "{{ $labels.pod }} on {{ $labels.nodename }} is down"
|
||||
# - alert: FluentdScrapeMissing
|
||||
# expr: absent(up{job="fluentd"} == 1)
|
||||
# for: 15m
|
||||
# labels:
|
||||
# context: fluentd
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: "Fluentd Scrape Missing"
|
||||
# description: "Fluentd instance has disappeared from Prometheus target discovery"
|
||||
|
||||
## Grafana Monitoring Dashboard
|
||||
##
|
||||
dashboards:
|
||||
enabled: "true"
|
||||
namespace: ""
|
||||
labels:
|
||||
grafana_dashboard: '"1"'
|
||||
|
||||
## Fluentd list of plugins to install
|
||||
##
|
||||
plugins: []
|
||||
# - fluent-plugin-out-http
|
||||
|
||||
## Add fluentd config files from K8s configMaps
|
||||
##
|
||||
configMapConfigs:
|
||||
- fluentd-prometheus-conf
|
||||
# - fluentd-systemd-conf
|
||||
|
||||
## Fluentd configurations:
|
||||
##
|
||||
fileConfigs:
|
||||
01_sources.conf: |-
|
||||
<source>
|
||||
@type systemd
|
||||
path /var/log/journal
|
||||
tag sys-log
|
||||
read_from_head true
|
||||
</source>
|
||||
<source>
|
||||
@id fluentd-containers.log
|
||||
@type tail
|
||||
encoding utf-8
|
||||
from_encoding "utf-8"
|
||||
path /var/log/containers/*.log
|
||||
pos_file /var/log/containers.log.pos
|
||||
exclude_path ["/var/log/containers/*telegraf*.log","/var/log/containers/*opentelemetry*.log","/var/log/containers/*fluentbit*.log","/var/log/containers/*gke-metrics*.log","/var/log/containers/*event-exporter-gke*.log","/var/log/containers/*metadata-server*.log","/var/log/containers/*metrics-server*.log","/var/log/containers/*filestore-node*.log"]
|
||||
path_key filename
|
||||
tag raw.containers.*
|
||||
read_from_head true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format json
|
||||
time_key time
|
||||
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
||||
keep_time_key true
|
||||
</pattern>
|
||||
<pattern>
|
||||
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
|
||||
time_format %Y-%m-%dT%H:%M:%S.%N%:z
|
||||
keep_time_key true
|
||||
</pattern>
|
||||
<pattern>
|
||||
format /^(?<log>fsp\s+.+)$/
|
||||
ignorecase false
|
||||
multiline false
|
||||
</pattern>
|
||||
</parse>
|
||||
</source>
|
||||
<match raw.containers.**>
|
||||
@id raw.containers
|
||||
@type detect_exceptions
|
||||
remove_tag_prefix raw
|
||||
message log
|
||||
stream stream
|
||||
multiline_flush_interval 5
|
||||
max_bytes 500000
|
||||
max_lines 1000
|
||||
</match>
|
||||
|
||||
<filter containers.**>
|
||||
@type kubernetes_metadata
|
||||
</filter>
|
||||
|
||||
<filter containers.**>
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
<record>
|
||||
container_id ${record.dig("docker", "container_id")}
|
||||
</record>
|
||||
</filter>
|
||||
|
||||
<match containers.**>
|
||||
@type rewrite_tag_filter
|
||||
<rule>
|
||||
key $.kubernetes.namespace_name
|
||||
pattern ^(.+)$
|
||||
tag $1.${tag}
|
||||
</rule>
|
||||
</match>
|
||||
|
||||
02_filters.conf: |-
|
||||
|
||||
03_dispatch.conf: |-
|
||||
<match {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,sys-log,**victoriametrics**}>
|
||||
@type "relabel"
|
||||
@label @NOCONCATDISPATCH
|
||||
</match>
|
||||
<match {**prd**,**int**}>
|
||||
@type "relabel"
|
||||
@label @CONCATDISPATCH
|
||||
</match>
|
||||
|
||||
<label @CONCATDISPATCH>
|
||||
<filter {**prd**,**int**}>
|
||||
@type concat
|
||||
key log
|
||||
stream_identity_key container_id
|
||||
multiline_start_regexp /^.+\d{2}\:\d{2}\:\d{2}\.\d{3}/
|
||||
separator "\n"
|
||||
flush_interval 30
|
||||
timeout_label @DISPATCH
|
||||
</filter>
|
||||
<match **>
|
||||
@type "relabel"
|
||||
@label @DISPATCH
|
||||
</match>
|
||||
</label>
|
||||
|
||||
<label @DISPATCH>
|
||||
<match {**prd**}>
|
||||
@type copy
|
||||
<store>
|
||||
@type "gcs"
|
||||
bucket "gcs-infr-dvps-meesho-logs-prd"
|
||||
path "datascience/${tag[0]}/dt=%d.%m.%Y/hr=%H/${tag[5]}"
|
||||
time_slice_format %Y%m%d%H
|
||||
object_key_format "%{path}/%{time_slice}_%{index}.%{file_extension}"
|
||||
store_as "gzip"
|
||||
<format>
|
||||
@type "json"
|
||||
localtime true
|
||||
</format>
|
||||
<buffer tag,time>
|
||||
timekey 30m
|
||||
@type "file"
|
||||
path "/tmp/td-agent/buffer/"
|
||||
flush_thread_count 8
|
||||
timekey_wait 10m
|
||||
chunk_limit_size 50m
|
||||
flush_at_shutdown true
|
||||
</buffer>
|
||||
</store>
|
||||
<store>
|
||||
@type "relabel"
|
||||
@label @PRD
|
||||
</store>
|
||||
</match>
|
||||
<match {**int**}>
|
||||
@type copy
|
||||
<store>
|
||||
@type "relabel"
|
||||
@label @PREPROD
|
||||
</store>
|
||||
</match>
|
||||
</label>
|
||||
|
||||
<label @PRD>
|
||||
<filter {**prd**}>
|
||||
@type grep
|
||||
<regexp>
|
||||
key log
|
||||
pattern /ERROR|WARN|error|warn/
|
||||
</regexp>
|
||||
</filter>
|
||||
<filter {**prd**} >
|
||||
@type parser
|
||||
key_name log
|
||||
reserve_time true
|
||||
reserve_data true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format regexp
|
||||
expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
|
||||
</pattern>
|
||||
<pattern>
|
||||
format /^(?<log>.*)$/
|
||||
</pattern>
|
||||
</parse>
|
||||
</filter>
|
||||
<filter {**prd**}>
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
auto_typecast true
|
||||
renew_record true
|
||||
<record>
|
||||
log_level ${record.dig("log_level")}
|
||||
logger ${record.dig("logger")}
|
||||
method ${record.dig("method")}
|
||||
line_number ${record.dig("line_number")}
|
||||
pid ${record.dig("pid")}
|
||||
thread ${record.dig("thread")}
|
||||
x3_trace_id ${record.dig("x3_trace_id")}
|
||||
x3_span_id ${record.dig("x3_span_id")}
|
||||
trace_id ${record.dig("trace_id")}
|
||||
span_id ${record.dig("span_id")}
|
||||
user_id ${record.dig("user_id")}
|
||||
extras ${record.dig("extras")}
|
||||
message ${record.dig("log")}
|
||||
throwable ${record.dig("throwable")}
|
||||
application_name ${record.dig("kubernetes", "namespace_name")}
|
||||
service_name ${record.dig("kubernetes", "container_name")}
|
||||
host ${record.dig("kubernetes", "host")}
|
||||
pod_name ${record.dig("kubernetes", "pod_name")}
|
||||
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
||||
</record>
|
||||
</filter>
|
||||
<filter {**prd**}>
|
||||
@type prometheus
|
||||
<metric>
|
||||
name fluentd_log_count_total
|
||||
type counter
|
||||
desc Total number of log entries processed by Fluentd (prd namespace breakdown)
|
||||
<labels>
|
||||
application_name $.application_name
|
||||
log_level $.log_level
|
||||
pod_name $.pod_name
|
||||
</labels>
|
||||
</metric>
|
||||
</filter>
|
||||
<match {**prd**}>
|
||||
@type elasticsearch
|
||||
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
||||
port 9200
|
||||
scheme http
|
||||
compression_level best_speed
|
||||
ssl_verify false
|
||||
emit_error_for_missing_id true
|
||||
id_key request_id
|
||||
user elastic
|
||||
password "#{ENV['ECK_ES_PASSWORD']}"
|
||||
index_name gcp-prd
|
||||
reload_on_failure true
|
||||
reconnect_on_error true
|
||||
include_timestamp true
|
||||
request_timeout 60s
|
||||
</match>
|
||||
</label>
|
||||
|
||||
<label @NOCONCATDISPATCH>
|
||||
<filter sys-log >
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
auto_typecast true
|
||||
renew_record true
|
||||
renew_time_key ${record.dig("SYSLOG_TIMESTAMP")}
|
||||
<record>
|
||||
cluster_name "datascience-prd"
|
||||
application_name "systemd"
|
||||
node_name ${record.dig("_HOSTNAME")}
|
||||
text ${record.to_json}
|
||||
</record>
|
||||
</filter>
|
||||
<filter {kube**,**coredns**,**external-secrets**,**keda**,**gke-mcs**,**victoriametrics**}>
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
auto_typecast true
|
||||
renew_record true
|
||||
<record>
|
||||
application_name ${record.dig("kubernetes", "namespace_name")}
|
||||
service_name ${record.dig("kubernetes", "container_name")}
|
||||
host ${record.dig("kubernetes", "host")}
|
||||
pod_name ${record.dig("kubernetes", "pod_name")}
|
||||
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
||||
text ${record.dig("log")}
|
||||
</record>
|
||||
</filter>
|
||||
<match {**gke-mcs**}>
|
||||
@type elasticsearch
|
||||
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
||||
port 9200
|
||||
scheme http
|
||||
compression_level best_speed
|
||||
ssl_verify false
|
||||
emit_error_for_missing_id true
|
||||
id_key request_id
|
||||
user elastic
|
||||
index_name gke-mcs
|
||||
password "#{ENV['ECK_ES_PASSWORD']}"
|
||||
reload_on_failure true
|
||||
reconnect_on_error true
|
||||
include_timestamp true
|
||||
request_timeout 60s
|
||||
</match>
|
||||
<match {**kube-events**,**kube-system**,**external-secrets**,**keda**,sys-log,**victoriametrics**}>
|
||||
@type elasticsearch
|
||||
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
||||
port 9200
|
||||
scheme http
|
||||
compression_level best_speed
|
||||
ssl_verify false
|
||||
emit_error_for_missing_id true
|
||||
id_key request_id
|
||||
user elastic
|
||||
index_name gcp-kubeevents
|
||||
password "#{ENV['ECK_ES_PASSWORD']}"
|
||||
reload_on_failure true
|
||||
reconnect_on_error true
|
||||
include_timestamp true
|
||||
request_timeout 60s
|
||||
</match>
|
||||
<match **>
|
||||
@type elasticsearch
|
||||
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
||||
port 9200
|
||||
scheme http
|
||||
compression_level best_speed
|
||||
ssl_verify false
|
||||
emit_error_for_missing_id true
|
||||
id_key request_id
|
||||
user elastic
|
||||
index_name gcp-prd
|
||||
password "#{ENV['ECK_ES_PASSWORD']}"
|
||||
reload_on_failure true
|
||||
reconnect_on_error true
|
||||
include_timestamp true
|
||||
request_timeout 60s
|
||||
</match>
|
||||
</label>
|
||||
|
||||
<label @PREPROD>
|
||||
<filter {**int**}>
|
||||
@type grep
|
||||
<regexp>
|
||||
key log
|
||||
pattern /INFO|ERROR|WARN|info|error|warn/
|
||||
</regexp>
|
||||
</filter>
|
||||
<filter {**int**} >
|
||||
@type parser
|
||||
key_name log
|
||||
reserve_time true
|
||||
reserve_data true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format regexp
|
||||
expression /(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - \[(?<log_level>[A-Z\s]{1,6})\] - \[(?<logger>[^\]]+):(?<method>[^\]:]+)?:(?<line_number>\d+)?\] - \[(?<pid>\d+)?, (?<thread>[^\]]+)?\] - \((?<x3_trace_id>[^,]*)?,(?<x3_span_id>[^)]*)?\)\|\((?<trace_id>[^,]*)?,(?<span_id>[^)]*)?\) - \[(?<user_id>[^\]]*)?\] - \((?<extras>[^)]*)?\) - (?<log>[^\n]*)\n?(?<throwable>[^|]*)$/
|
||||
</pattern>
|
||||
<pattern>
|
||||
format /^(?<log>.*)$/
|
||||
</pattern>
|
||||
</parse>
|
||||
</filter>
|
||||
<filter {**int**}>
|
||||
@type record_transformer
|
||||
enable_ruby true
|
||||
auto_typecast true
|
||||
renew_record true
|
||||
<record>
|
||||
log_level ${record.dig("log_level")}
|
||||
logger ${record.dig("logger")}
|
||||
method ${record.dig("method")}
|
||||
line_number ${record.dig("line_number")}
|
||||
pid ${record.dig("pid")}
|
||||
thread ${record.dig("thread")}
|
||||
x3_trace_id ${record.dig("x3_trace_id")}
|
||||
x3_span_id ${record.dig("x3_span_id")}
|
||||
trace_id ${record.dig("trace_id")}
|
||||
span_id ${record.dig("span_id")}
|
||||
user_id ${record.dig("user_id")}
|
||||
extras ${record.dig("extras")}
|
||||
message ${record.dig("log")}
|
||||
throwable ${record.dig("throwable")}
|
||||
application_name ${record.dig("kubernetes", "namespace_name")}
|
||||
service_name ${record.dig("kubernetes", "container_name")}
|
||||
host ${record.dig("kubernetes", "host")}
|
||||
pod_name ${record.dig("kubernetes", "pod_name")}
|
||||
pod_ip ${record.dig("kubernetes", "pod_ip")}
|
||||
</record>
|
||||
</filter>
|
||||
<match {**int**}>
|
||||
@type elasticsearch
|
||||
host eck-observability-prd-es-hot-warm.eck-observability-prd.svc.clusterset.local
|
||||
port 9200
|
||||
scheme http
|
||||
compression_level best_speed
|
||||
ssl_verify false
|
||||
emit_error_for_missing_id true
|
||||
id_key request_id
|
||||
user elastic
|
||||
index_name gcp-int
|
||||
password "#{ENV['ECK_ES_PASSWORD']}"
|
||||
reload_on_failure true
|
||||
reconnect_on_error true
|
||||
include_timestamp true
|
||||
request_timeout 60s
|
||||
</match>
|
||||
</label>
|
||||
04_outputs.conf: |-
|
||||
Reference in New Issue
Block a user