diff --git a/helm-overrides/k8s-admin-prd-ase1/grafana/custom-values.yaml b/helm-overrides/k8s-admin-prd-ase1/grafana/custom-values.yaml index 61b471d..c11797b 100644 --- a/helm-overrides/k8s-admin-prd-ase1/grafana/custom-values.yaml +++ b/helm-overrides/k8s-admin-prd-ase1/grafana/custom-values.yaml @@ -31,17 +31,241 @@ grafana: # than set by hand: it survives a pod restart and a fresh install gets # it automatically. VictoriaMetrics speaks Prometheus's own query API, # so `type: prometheus` here is correct even though the URL is VM's — - # see devops-infra-helm-charts' victoria-metrics-single chart for why. + # see this repo's victoria-metrics-single chart for why. datasources: datasources.yaml: apiVersion: 1 datasources: - name: VictoriaMetrics + # Fixed uid, not left to auto-generate — the provisioned + # dashboard below references this datasource by uid, and an + # auto-generated one would only exist after Grafana's first + # boot, too late for a dashboard provisioned in the same boot. + uid: victoriametrics type: prometheus access: proxy url: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 isDefault: true + # Dashboard provisioning. Provisioned rather than built by hand in the + # UI for the same reason the datasource above is: it survives a pod + # restart (this deployment has no persistent Grafana database beyond + # the 1Gi PVC, and even with one, a fresh install should not start + # with an empty dashboard list) and a `git diff` shows what changed. + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + + # "Homelab Overview" — total Envoy/Contour RPS, per-namespace CPU and + # memory (the $namespace template variable filters every relevant + # panel), cluster-wide utilization against actual node capacity, and a + # total-resources row (cores/memory/pods/disk). Envoy and node-exporter + # metrics both required their own vmagent scrape job — see + # helm-overrides/.../vmagent/custom-values.yaml for why neither was + # reachable through the chart's own defaults in this cluster. + # + # Every panel except "Disk free" was run against the live deployment + # (vmui, over Tailscale) before being written in here — RPS, per- + # namespace CPU/memory, machine_cpu_cores/machine_memory_bytes all + # returned real data. "Disk free" depends on the node-exporter scrape + # job added alongside this same change, which had not been live yet to + # verify against — worth checking once this actually deploys, same as + # everything else in this repo that gets a `helm template` check but + # cannot get a live one before the first sync. + dashboards: + default: + homelab: + json: | + { + "title": "Homelab Overview", + "uid": "homelab-overview", + "schemaVersion": 39, + "editable": true, + "timezone": "browser", + "time": { "from": "now-1h", "to": "now" }, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "query": "label_values(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}, namespace)", + "refresh": 2, + "multi": true, + "includeAll": true, + "current": { "selected": true, "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { "type": "row", "title": "Ingress (Envoy / Contour)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100 }, + + { + "type": "stat", "title": "Total RPS", "id": 1, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "sum(rate(envoy_http_downstream_rq_total{namespace=\"projectcontour\"}[5m]))", "legendFormat": "rps" }], + "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" } + }, + { + "type": "stat", "title": "Active downstream connections", "id": 2, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "sum(envoy_http_downstream_cx_active{namespace=\"projectcontour\"})", "legendFormat": "connections" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "type": "timeseries", "title": "Requests by response class", "id": 3, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ + "expr": "sum by (envoy_response_code_class) (rate(envoy_http_downstream_rq_xx{namespace=\"projectcontour\"}[5m]))", + "legendFormat": "{{envoy_response_code_class}}xx" + }], + "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } } + }, + + { "type": "row", "title": "Service level (by namespace)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 101 }, + + { + "type": "timeseries", "title": "CPU usage by namespace", "id": 10, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ + "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", + "legendFormat": "{{namespace}}" + }], + "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] }, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } } + }, + { + "type": "timeseries", "title": "Memory usage by namespace", "id": 11, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ + "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", + "legendFormat": "{{namespace}}" + }], + "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] }, + "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } } + }, + { + "type": "table", "title": "Current usage per namespace", "id": 12, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [ + { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", "format": "table", "instant": true, "refId": "A" }, + { "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", "format": "table", "instant": true, "refId": "B" }, + { "expr": "count by (namespace) (count by (namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}))", "format": "table", "instant": true, "refId": "C" } + ], + "transformations": [ + { "id": "merge", "options": {} }, + { "id": "organize", "options": { + "excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "Time 3": true }, + "renameByName": { "Value #A": "CPU (cores)", "Value #B": "Memory", "Value #C": "Pods" } + } } + ], + "fieldConfig": { "defaults": {}, "overrides": [ + { "matcher": { "id": "byName", "options": "Memory" }, "properties": [{ "id": "unit", "value": "bytes" }] }, + { "matcher": { "id": "byName", "options": "CPU (cores)" }, "properties": [{ "id": "unit", "value": "short" }, { "id": "decimals", "value": 3 }] } + ] } + }, + + { "type": "row", "title": "Cluster utilization", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 102 }, + + { + "type": "gauge", "title": "CPU utilization", "id": 20, + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 25 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)" }], + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } + ] } }, "overrides": [] } + }, + { + "type": "gauge", "title": "Memory utilization", "id": 21, + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 25 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)" }], + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, + "thresholds": { "mode": "absolute", "steps": [ + { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } + ] } }, "overrides": [] } + }, + { + "type": "timeseries", "title": "Cluster CPU utilization over time", "id": 22, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [ + { "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)", "legendFormat": "CPU %" }, + { "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)", "legendFormat": "Memory %" } + ], + "fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] } + }, + + { "type": "row", "title": "Total resources", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 103 }, + + { + "type": "stat", "title": "Node CPU capacity", "id": 30, + "gridPos": { "h": 5, "w": 4, "x": 0, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "max(machine_cpu_cores)" }], + "fieldConfig": { "defaults": { "unit": "short", "displayName": "cores" }, "overrides": [] } + }, + { + "type": "stat", "title": "Node memory capacity", "id": 31, + "gridPos": { "h": 5, "w": 4, "x": 4, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "max(machine_memory_bytes)" }], + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } + }, + { + "type": "stat", "title": "CPU used (cluster)", "id": 32, + "gridPos": { "h": 5, "w": 4, "x": 8, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m]))" }], + "fieldConfig": { "defaults": { "unit": "short", "displayName": "cores", "decimals": 2 }, "overrides": [] } + }, + { + "type": "stat", "title": "Memory used (cluster)", "id": 33, + "gridPos": { "h": 5, "w": 4, "x": 12, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"})" }], + "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } + }, + { + "type": "stat", "title": "Running pods", "id": 34, + "gridPos": { "h": 5, "w": 4, "x": 16, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "count(count by (namespace, pod) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))" }], + "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } + }, + { + "type": "stat", "title": "Disk free (root)", "id": 35, + "gridPos": { "h": 5, "w": 4, "x": 20, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "victoriametrics" }, + "targets": [{ "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}" }], + "fieldConfig": { "defaults": { "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [ + { "color": "red", "value": null }, { "color": "yellow", "value": 5000000000 }, { "color": "green", "value": 15000000000 } + ] } }, "overrides": [] } + } + ] + } + ingress: enabled: true ingressClassName: contour diff --git a/helm-overrides/k8s-admin-prd-ase1/vmagent/custom-values.yaml b/helm-overrides/k8s-admin-prd-ase1/vmagent/custom-values.yaml index 7c89d79..f9488be 100644 --- a/helm-overrides/k8s-admin-prd-ase1/vmagent/custom-values.yaml +++ b/helm-overrides/k8s-admin-prd-ase1/vmagent/custom-values.yaml @@ -18,7 +18,81 @@ victoria-metrics-agent: # comment literally says "COPY from Prometheus helm chart") and # kubernetes-service-endpoints (the prometheus.io/scrape annotation # convention node-exporter's Service carries). Nothing here needs a - # scrape target the chart doesn't already define out of the box. + # scrape target the chart doesn't already define out of the box — + # except Contour's own Envoy, added below via extraScrapeConfigs (the + # chart concatenates this onto config.scrape_configs rather than + # replacing it, so the defaults above are unaffected). + # + # Confirmed live, not assumed: Cilium's own embedded Envoy (a separate + # thing — its L7 policy proxy, kube-system namespace) was already being + # scraped via the annotation-based kubernetes-pods job, which is what + # first showed envoy_* metrics existed at all in this cluster. Contour's + # ingress Envoy (projectcontour namespace — the actual data plane for + # everything routed through this homelab's Ingress, hostPort 80/443) + # carries no such annotation, confirmed by its total absence from + # `envoy_http_downstream_rq_total{namespace="projectcontour"}` before + # this job existed — so it needs its own explicit target. + # + # Filtered by container port number (8002, the official Contour + # chart's fixed metrics port for Envoy — projectcontour/values or + # equivalent) rather than by pod label: this cluster's Contour install + # auto-detected object names after install rather than assuming the + # chart's defaults (see claude.md's Contour install history), so a + # label guess is less trustworthy here than the one thing that has to + # be true for Envoy's own metrics port to exist at all. + # + # /stats/prometheus is Envoy's own built-in admin endpoint format, not + # a Contour-specific path — this is how any Envoy exposes Prometheus + # metrics once a metrics listener is configured, independent of chart. + # + # node-exporter needed the same treatment, for a different reason: + # confirmed live that kubernetes-service-endpoints (role: endpointslice, + # keyed on the *Service's* prometheus.io/scrape annotation — where the + # node-exporter chart actually puts it, not on the pod) finds nothing + # at all in this cluster — not "down", entirely absent from `up`, no + # `kubernetes-service-endpoints`/`kubernetes-services` job present + # whatsoever. Rather than chase why that discovery path is empty here, + # targeting node-exporter's pod directly by its declared container port + # (9100, the chart's fixed default) sidesteps it the same way the + # Envoy job above does, and is no less correct for not depending on + # whichever annotation-propagation mechanism isn't working. + extraScrapeConfigs: + - job_name: contour-envoy + kubernetes_sd_configs: + - role: pod + namespaces: + names: ["projectcontour"] + relabel_configs: + - action: keep + source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "8002" + - target_label: __metrics_path__ + replacement: /stats/prometheus + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + - job_name: node-exporter + kubernetes_sd_configs: + - role: pod + namespaces: + names: ["monitoring"] + relabel_configs: + - action: keep + source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9100" + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node resources: requests: