diff --git a/helm-overrides/gke-toolshed-prd-usc1/victoria-metrics-single/custom-values.yaml b/helm-overrides/gke-toolshed-prd-usc1/victoria-metrics-single/custom-values.yaml index f897738..2875db8 100644 --- a/helm-overrides/gke-toolshed-prd-usc1/victoria-metrics-single/custom-values.yaml +++ b/helm-overrides/gke-toolshed-prd-usc1/victoria-metrics-single/custom-values.yaml @@ -12,12 +12,30 @@ victoria-metrics-single: # the homelab's local-path, this class can be expanded later. size: 3Gi + # Raised from 128Mi/512Mi, which this no longer fits inside. + # + # The pod ran for weeks at the old limit and then OOM-killed in a loop + # the first time it was restarted — exit 137 roughly fifty seconds after + # a clean start, every time. Nothing had changed about its configuration; + # the dataset had simply grown into 3.14 billion rows, and the memory a + # restart needs to resume ingestion no longer fit. A long-lived pod can + # sit well past the limit it would need to start again, so the failure + # only appears the next time something restarts it. + # + # Memory here tracks active time series rather than disk, which is why + # shortening retentionPeriod above would not have helped: the scrape + # targets are the same either way, and several of them carry 40+ labels + # (see the maxLabelsPerTimeseries warnings in its log). + # + # Affordable: memory requests across the three nodes sit at 62%, 18% and + # 47%, so there is room. CPU is the constrained resource on this cluster, + # and this costs none. resources: requests: cpu: 50m - memory: 128Mi + memory: 256Mi limits: - memory: 512Mi + memory: 1Gi # vmui, VictoriaMetrics' built-in query UI, on the same pod and port — # ad-hoc PromQL only, no saved dashboards; Grafana is the real UI. Free