added repo

2026-08-26 03:39:42 +05:30
parent 45c25a95af
commit b8575bb8b9
6889 changed files with 1217125 additions and 0 deletions
@@ -0,0 +1,181 @@
+# Elasticsearch for observability: logs, traces (via APM), hot/warm tiers.
+# Stack monitoring: metrics + logs ship to eck-observability-monitoring-prd (ECK-managed ref).
+# Topology: 3× (master + data_hot + ingest), 2× (data_warm + ingest), 6g heap, 500Gi disk per node.
+# Tolerations: align with your GKE node pool taint (key/value below).
+# ILM: configure index templates to route hot → warm (data_hot / data_warm) in Kibana / API.
+# SSO: Google login is handled at nginx + oauth2-proxy (ingress), not Elasticsearch OIDC (Platinum).
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: eck-observability-prd
+  namespace: eck-observability-prd
+spec:
+  version: 9.3.1
+  monitoring:
+    metrics:
+      elasticsearchRefs:
+        - name: eck-observability-monitoring-prd
+          namespace: eck-observability-prd
+    logs:
+      elasticsearchRefs:
+        - name: eck-observability-monitoring-prd
+          namespace: eck-observability-prd
+  nodeSets:
+    # 3 nodes: master-eligible + data_hot + ingest (schedule on tainted pool)
+    - name: hot
+      count: 9
+      config:
+        node.roles: ["data_hot", "ingest", "data_content", "transform"]
+      volumeClaimTemplates:
+      - metadata:
+          name: elasticsearch-data
+        spec:
+          storageClassName: gke-hyperdisk-balanced-35k
+          accessModes:
+          - ReadWriteOnce
+          resources:
+            requests:
+              storage: 4Ti
+      podTemplate:
+        metadata:
+          labels:
+            elasticsearch.k8s.elastic.co/service-account: eck-observability-es-sa-prd
+            elasticsearch.k8s.elastic.co/nodeset: hot
+            elasticsearch.k8s.elastic.co/tier: hot-warm-data
+            elasticsearch.k8s.elastic.co/tier22: hot-warm-data-22
+          annotations:
+            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+        spec:
+          serviceAccountName: eck-observability-es-sa-prd
+          # Tolerations for dedicated ES node pool (edit key/value to match your taint)
+          tolerations:
+          - effect: NoSchedule
+            key: elastic-observability-hot-nodes
+            operator: Equal
+            value: "true"
+          nodeSelector:
+            elastic-observability-hot-nodes: "true"
+          initContainers:
+          - name: sysctl
+            securityContext:
+              privileged: true
+              runAsUser: 0
+            command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
+          containers:
+            - name: elasticsearch
+              env:
+              - name: ES_JAVA_OPTS
+                value: "-Xms31g -Xmx31g"
+              resources:
+                requests:
+                  memory: 110Gi
+                  cpu: "26"
+                limits:
+                  memory: 110Gi
+                  cpu: "28"
+
+    # 2 nodes: data_warm + ingest (schedule on tainted pool)
+    - name: warm
+      count: 6
+      config:
+        node.roles: ["data_warm", "ingest", "data_content"]
+      volumeClaimTemplates:
+      - metadata:
+          name: elasticsearch-data
+        spec:
+          storageClassName: gke-hyperdisk-ssd
+          accessModes:
+          - ReadWriteOnce
+          resources:
+            requests:
+              storage: 6.5Ti
+      podTemplate:
+        metadata:
+          labels:
+            elasticsearch.k8s.elastic.co/service-account: eck-observability-es-sa-prd
+            elasticsearch.k8s.elastic.co/nodeset: warm
+            elasticsearch.k8s.elastic.co/tier: hot-warm-data
+          annotations:
+            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+        spec:
+          serviceAccountName: eck-observability-es-sa-prd
+          tolerations:
+          - effect: NoSchedule
+            key: elastic-observability-warm-nodes
+            operator: Equal
+            value: "true"
+          nodeSelector:
+            elastic-observability-warm-nodes: "true"
+          initContainers:
+          - name: sysctl
+            securityContext:
+              privileged: true
+              runAsUser: 0
+            command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
+          containers:
+            - name: elasticsearch
+              env:
+              - name: ES_JAVA_OPTS
+                value: "-Xms31g -Xmx31g"
+              resources:
+                requests:
+                  memory: 107Gi
+                  cpu: "24"
+                limits:
+                  memory: 107Gi
+                  cpu: "26"
+
+    # 3 nodes: master-only (dedicated cluster state management)
+    - name: master
+      count: 3
+      config:
+        node.roles: ["master"]
+      volumeClaimTemplates:
+      - metadata:
+          name: elasticsearch-data
+        spec:
+          storageClassName: gke-hyperdisk-ssd
+          accessModes:
+          - ReadWriteOnce
+          resources:
+            requests:
+              storage: 200Gi
+      podTemplate:
+        metadata:
+          labels:
+            elasticsearch.k8s.elastic.co/service-account: eck-observability-es-sa-prd
+            elasticsearch.k8s.elastic.co/nodeset: master
+          annotations:
+            cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+        spec:
+          serviceAccountName: eck-observability-es-sa-prd
+          tolerations:
+          - effect: NoSchedule
+            key: elastic-observability-master-nodes
+            operator: Equal
+            value: "true"
+          nodeSelector:
+            elastic-observability-master-nodes: "true"
+          initContainers:
+          - name: sysctl
+            securityContext:
+              privileged: true
+              runAsUser: 0
+            command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
+          containers:
+            - name: elasticsearch
+              env:
+              - name: ES_JAVA_OPTS
+                value: "-Xms5g -Xmx5g"
+              resources:
+                requests:
+                  memory: 10Gi
+                  cpu: "5"
+                limits:
+                  memory: 10Gi
+                  cpu: "6"
+
+  http:
+    tls:
+      selfSignedCertificate:
+        disabled: true