grafana: # From Vault via ExternalSecret (devops-infra-argo-config/secretstores/ # grafana-admin-credentials.yaml), same pattern as gitea/harbor/jenkins # admin credentials elsewhere in this project — never a plaintext # adminPassword in this file. admin: existingSecret: grafana-admin-credentials userKey: username passwordKey: password persistence: # 1Gi, not the chart's 10Gi default: this holds dashboards, folders and # Grafana's own sqlite state, not metric data — VictoriaMetrics keeps # that. Unlike the homelab's local-path, standard-rwo can be expanded # later if that ever proves tight. enabled: true storageClassName: standard-rwo size: 1Gi resources: requests: cpu: 50m memory: 128Mi limits: memory: 384Mi # Provisioned at boot, not clicked through in the UI — the same reason # every other credential/config in this project is committed rather # than set by hand: it survives a pod restart and a fresh install gets # it automatically. VictoriaMetrics speaks Prometheus's own query API, # so `type: prometheus` here is correct even though the URL is VM's — # see this repo's victoria-metrics-single chart for why. datasources: datasources.yaml: apiVersion: 1 datasources: - name: VictoriaMetrics # Fixed uid, not left to auto-generate — the provisioned # dashboard below references this datasource by uid, and an # auto-generated one would only exist after Grafana's first # boot, too late for a dashboard provisioned in the same boot. uid: victoriametrics type: prometheus access: proxy url: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428 isDefault: true # Dashboard provisioning. Provisioned rather than built by hand in the # UI for the same reason the datasource above is: it survives a pod # restart (this deployment has no persistent Grafana database beyond # the 1Gi PVC, and even with one, a fresh install should not start # with an empty dashboard list) and a `git diff` shows what changed. dashboardProviders: dashboardproviders.yaml: apiVersion: 1 providers: - name: default orgId: 1 folder: "" type: file disableDeletion: false editable: true options: path: /var/lib/grafana/dashboards/default # "Homelab Overview" — total Envoy/Contour RPS, per-namespace CPU and # memory (the $namespace template variable filters every relevant # panel), cluster-wide utilization against actual node capacity, and a # total-resources row (cores/memory/pods/disk). Envoy and node-exporter # metrics both required their own vmagent scrape job — see # helm-overrides/.../vmagent/custom-values.yaml for why neither was # reachable through the chart's own defaults in this cluster. # # Every panel except "Disk free" was run against the live deployment # (vmui, over Tailscale) before being written in here — RPS, per- # namespace CPU/memory, machine_cpu_cores/machine_memory_bytes all # returned real data. "Disk free" depends on the node-exporter scrape # job added alongside this same change, which had not been live yet to # verify against — worth checking once this actually deploys, same as # everything else in this repo that gets a `helm template` check but # cannot get a live one before the first sync. dashboards: default: homelab: json: | { "title": "Homelab Overview", "uid": "homelab-overview", "schemaVersion": 39, "editable": true, "timezone": "browser", "time": { "from": "now-1h", "to": "now" }, "refresh": "30s", "templating": { "list": [ { "name": "namespace", "type": "query", "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "query": "label_values(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}, namespace)", "refresh": 2, "multi": true, "includeAll": true, "current": { "selected": true, "text": "All", "value": "$__all" } } ] }, "panels": [ { "type": "row", "title": "Ingress (Envoy / Contour)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100 }, { "type": "stat", "title": "Total RPS", "id": 1, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum(rate(envoy_http_downstream_rq_total{namespace=\"projectcontour\"}[5m]))", "legendFormat": "rps" }], "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" } }, { "type": "stat", "title": "Active downstream connections", "id": 2, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum(envoy_http_downstream_cx_active{namespace=\"projectcontour\"})", "legendFormat": "connections" }], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } }, { "type": "timeseries", "title": "Requests by response class", "id": 3, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum by (envoy_response_code_class) (rate(envoy_http_downstream_rq_xx{namespace=\"projectcontour\"}[5m]))", "legendFormat": "{{envoy_response_code_class}}xx" }], "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" } } }, { "type": "row", "title": "Service level (by namespace)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 101 }, { "type": "timeseries", "title": "CPU usage by namespace", "id": 10, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", "legendFormat": "{{namespace}}" }], "fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } } }, { "type": "timeseries", "title": "Memory usage by namespace", "id": 11, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", "legendFormat": "{{namespace}}" }], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } } }, { "type": "table", "title": "Current usage per namespace", "id": 12, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [ { "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", "format": "table", "instant": true, "refId": "A" }, { "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", "format": "table", "instant": true, "refId": "B" }, { "expr": "count by (namespace) (count by (namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}))", "format": "table", "instant": true, "refId": "C" } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "Time 3": true }, "renameByName": { "Value #A": "CPU (cores)", "Value #B": "Memory", "Value #C": "Pods" } } } ], "fieldConfig": { "defaults": {}, "overrides": [ { "matcher": { "id": "byName", "options": "Memory" }, "properties": [{ "id": "unit", "value": "bytes" }] }, { "matcher": { "id": "byName", "options": "CPU (cores)" }, "properties": [{ "id": "unit", "value": "short" }, { "id": "decimals", "value": 3 }] } ] } }, { "type": "row", "title": "Cluster utilization", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 102 }, { "type": "gauge", "title": "CPU utilization", "id": 20, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 25 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)" }], "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } ] } }, "overrides": [] } }, { "type": "gauge", "title": "Memory utilization", "id": 21, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 25 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)" }], "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 } ] } }, "overrides": [] } }, { "type": "timeseries", "title": "Cluster CPU utilization over time", "id": 22, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [ { "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)", "legendFormat": "CPU %" }, { "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)", "legendFormat": "Memory %" } ], "fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] } }, { "type": "row", "title": "Total resources", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 103 }, { "type": "stat", "title": "Node CPU capacity", "id": 30, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "max(machine_cpu_cores)" }], "fieldConfig": { "defaults": { "unit": "short", "displayName": "cores" }, "overrides": [] } }, { "type": "stat", "title": "Node memory capacity", "id": 31, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "max(machine_memory_bytes)" }], "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } }, { "type": "stat", "title": "CPU used (cluster)", "id": 32, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m]))" }], "fieldConfig": { "defaults": { "unit": "short", "displayName": "cores", "decimals": 2 }, "overrides": [] } }, { "type": "stat", "title": "Memory used (cluster)", "id": 33, "gridPos": { "h": 5, "w": 4, "x": 12, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"})" }], "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } }, { "type": "stat", "title": "Running pods", "id": 34, "gridPos": { "h": 5, "w": 4, "x": 16, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "count(count by (namespace, pod) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))" }], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } }, { "type": "stat", "title": "Disk free (root)", "id": 35, "gridPos": { "h": 5, "w": 4, "x": 20, "y": 33 }, "datasource": { "type": "prometheus", "uid": "victoriametrics" }, "targets": [{ "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}" }], "fieldConfig": { "defaults": { "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 5000000000 }, { "color": "green", "value": 15000000000 } ] } }, "overrides": [] } } ] } ingress: enabled: true ingressClassName: contour annotations: # Only the real domain appears in tls below — Let's Encrypt cannot # issue for nip.io, and one certificate spanning both would fail # outright rather than covering the half it can serve. cert-manager.io/cluster-issuer: letsencrypt-prod path: / hosts: - grafana.35.238.248.203.nip.io - grafana.infra.deployshed.com tls: - secretName: grafana-tls hosts: - grafana.infra.deployshed.com