victoria-metrics-agent: # Ships to victoria-metrics-single. The write path is /api/v1/write, # the same endpoint any Prometheus remote_write client uses — this is # the one config line that actually wires the two components together; # the chart names don't imply it on their own. # # Cluster-internal Service DNS: this call happens from a pod, so # CoreDNS resolves it. Same reasoning as everywhere else in this # project that a Service name (not an Ingress hostname) is correct # here — see claude.md's registry-hostname convention for the one case # where cluster DNS specifically does NOT work (a node-level pull), # which does not apply to this pod-to-pod write. remoteWrite: - url: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428/api/v1/write # config.scrape_configs is left at the chart's own default — it already # includes kubernetes-nodes-cadvisor (kubelet's cAdvisor endpoint, # comment literally says "COPY from Prometheus helm chart") and # kubernetes-service-endpoints (the prometheus.io/scrape annotation # convention node-exporter's Service carries). Nothing here needs a # scrape target the chart doesn't already define out of the box — # except Contour's own Envoy, added below via extraScrapeConfigs (the # chart concatenates this onto config.scrape_configs rather than # replacing it, so the defaults above are unaffected). # # Confirmed live, not assumed: Cilium's own embedded Envoy (a separate # thing — its L7 policy proxy, kube-system namespace) was already being # scraped via the annotation-based kubernetes-pods job, which is what # first showed envoy_* metrics existed at all in this cluster. Contour's # ingress Envoy (projectcontour namespace — the actual data plane for # everything routed through this homelab's Ingress, hostPort 80/443) # carries no such annotation, confirmed by its total absence from # `envoy_http_downstream_rq_total{namespace="projectcontour"}` before # this job existed — so it needs its own explicit target. # # Filtered by container port number (8002, the official Contour # chart's fixed metrics port for Envoy — projectcontour/values or # equivalent) rather than by pod label: this cluster's Contour install # auto-detected object names after install rather than assuming the # chart's defaults (see claude.md's Contour install history), so a # label guess is less trustworthy here than the one thing that has to # be true for Envoy's own metrics port to exist at all. # # /stats/prometheus is Envoy's own built-in admin endpoint format, not # a Contour-specific path — this is how any Envoy exposes Prometheus # metrics once a metrics listener is configured, independent of chart. # # node-exporter needed the same treatment, for a different reason: # confirmed live that kubernetes-service-endpoints (role: endpointslice, # keyed on the *Service's* prometheus.io/scrape annotation — where the # node-exporter chart actually puts it, not on the pod) finds nothing # at all in this cluster — not "down", entirely absent from `up`, no # `kubernetes-service-endpoints`/`kubernetes-services` job present # whatsoever. Rather than chase why that discovery path is empty here, # targeting node-exporter's pod directly by its declared container port # (9100, the chart's fixed default) sidesteps it the same way the # Envoy job above does, and is no less correct for not depending on # whichever annotation-propagation mechanism isn't working. extraScrapeConfigs: - job_name: contour-envoy kubernetes_sd_configs: - role: pod namespaces: names: ["projectcontour"] relabel_configs: - action: keep source_labels: [__meta_kubernetes_pod_container_port_number] regex: "8002" - target_label: __metrics_path__ replacement: /stats/prometheus - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_node_name] target_label: node - job_name: node-exporter kubernetes_sd_configs: - role: pod namespaces: names: ["monitoring"] relabel_configs: - action: keep source_labels: [__meta_kubernetes_pod_container_port_number] regex: "9100" - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - source_labels: [__meta_kubernetes_pod_name] target_label: pod - source_labels: [__meta_kubernetes_namespace] target_label: namespace - source_labels: [__meta_kubernetes_pod_node_name] target_label: node resources: requests: cpu: 25m memory: 64Mi limits: memory: 192Mi