Mukul SharmaandClaude Opus 5
f83cedca65
Ports the rest of the homelab's stack: external-secrets, harbor, jenkins, postgresql, redis, victoria-metrics-single, vmagent, node-exporter and grafana. All nine verified with helm template. Most are the homelab's values with storage moved from local-path to standard-rwo and hostnames rebuilt on the reserved ingress IP. vmagent and node-exporter are unchanged outright — everything in them is addressed by cluster DNS, namespace or container port, none of which differs here. Harbor is the substantial one. The homelab serves it over plain HTTP and makes containerd accept that by hand-editing hosts.toml on the node; GKE nodes are managed and replaced, so that edit cannot survive. Instead the node pool was told at creation to trust a private CA for exactly this hostname, and cert-manager now signs Harbor's certificate from that same CA via an ingress-shim annotation. certSource is "secret" rather than the chart's "auto", which would self-sign a certificate nothing trusts. externalURL moves to https to match, since Harbor hands that URL to docker clients and a mismatch surfaces as registry errors. Jenkins drops secondaryingress, which exists in the homelab only to serve its Tailscale hostname. Its plugin pins are carried over deliberately: each fixes a failure whose symptom points somewhere else, above all the kubernetes/kubernetes-client-api pairing, without which agents never come online and builds hang at "Still waiting to schedule task". Postgres and Redis keep the homelab's deliberately small memory settings. Those were chosen for an 8GB node under pressure, and while this cluster has room, a bigger cache buys nothing for a handful of small tools. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LEsTefWWifp4ikvhHF5s6N
273 lines
15 KiB
YAML
273 lines
15 KiB
YAML
grafana:
|
|
# From Vault via ExternalSecret (devops-infra-argo-config/secretstores/
|
|
# grafana-admin-credentials.yaml), same pattern as gitea/harbor/jenkins
|
|
# admin credentials elsewhere in this project — never a plaintext
|
|
# adminPassword in this file.
|
|
admin:
|
|
existingSecret: grafana-admin-credentials
|
|
userKey: username
|
|
passwordKey: password
|
|
|
|
persistence:
|
|
# 1Gi, not the chart's 10Gi default: this holds dashboards, folders and
|
|
# Grafana's own sqlite state, not metric data — VictoriaMetrics keeps
|
|
# that. Unlike the homelab's local-path, standard-rwo can be expanded
|
|
# later if that ever proves tight.
|
|
enabled: true
|
|
storageClassName: standard-rwo
|
|
size: 1Gi
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
memory: 384Mi
|
|
|
|
# Provisioned at boot, not clicked through in the UI — the same reason
|
|
# every other credential/config in this project is committed rather
|
|
# than set by hand: it survives a pod restart and a fresh install gets
|
|
# it automatically. VictoriaMetrics speaks Prometheus's own query API,
|
|
# so `type: prometheus` here is correct even though the URL is VM's —
|
|
# see this repo's victoria-metrics-single chart for why.
|
|
datasources:
|
|
datasources.yaml:
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: VictoriaMetrics
|
|
# Fixed uid, not left to auto-generate — the provisioned
|
|
# dashboard below references this datasource by uid, and an
|
|
# auto-generated one would only exist after Grafana's first
|
|
# boot, too late for a dashboard provisioned in the same boot.
|
|
uid: victoriametrics
|
|
type: prometheus
|
|
access: proxy
|
|
url: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
|
|
isDefault: true
|
|
|
|
# Dashboard provisioning. Provisioned rather than built by hand in the
|
|
# UI for the same reason the datasource above is: it survives a pod
|
|
# restart (this deployment has no persistent Grafana database beyond
|
|
# the 1Gi PVC, and even with one, a fresh install should not start
|
|
# with an empty dashboard list) and a `git diff` shows what changed.
|
|
dashboardProviders:
|
|
dashboardproviders.yaml:
|
|
apiVersion: 1
|
|
providers:
|
|
- name: default
|
|
orgId: 1
|
|
folder: ""
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards/default
|
|
|
|
# "Homelab Overview" — total Envoy/Contour RPS, per-namespace CPU and
|
|
# memory (the $namespace template variable filters every relevant
|
|
# panel), cluster-wide utilization against actual node capacity, and a
|
|
# total-resources row (cores/memory/pods/disk). Envoy and node-exporter
|
|
# metrics both required their own vmagent scrape job — see
|
|
# helm-overrides/.../vmagent/custom-values.yaml for why neither was
|
|
# reachable through the chart's own defaults in this cluster.
|
|
#
|
|
# Every panel except "Disk free" was run against the live deployment
|
|
# (vmui, over Tailscale) before being written in here — RPS, per-
|
|
# namespace CPU/memory, machine_cpu_cores/machine_memory_bytes all
|
|
# returned real data. "Disk free" depends on the node-exporter scrape
|
|
# job added alongside this same change, which had not been live yet to
|
|
# verify against — worth checking once this actually deploys, same as
|
|
# everything else in this repo that gets a `helm template` check but
|
|
# cannot get a live one before the first sync.
|
|
dashboards:
|
|
default:
|
|
homelab:
|
|
json: |
|
|
{
|
|
"title": "Homelab Overview",
|
|
"uid": "homelab-overview",
|
|
"schemaVersion": 39,
|
|
"editable": true,
|
|
"timezone": "browser",
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"refresh": "30s",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"query": "label_values(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}, namespace)",
|
|
"refresh": 2,
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"current": { "selected": true, "text": "All", "value": "$__all" }
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{ "type": "row", "title": "Ingress (Envoy / Contour)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100 },
|
|
|
|
{
|
|
"type": "stat", "title": "Total RPS", "id": 1,
|
|
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(rate(envoy_http_downstream_rq_total{namespace=\"projectcontour\"}[5m]))", "legendFormat": "rps" }],
|
|
"fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Active downstream connections", "id": 2,
|
|
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(envoy_http_downstream_cx_active{namespace=\"projectcontour\"})", "legendFormat": "connections" }],
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Requests by response class", "id": 3,
|
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (envoy_response_code_class) (rate(envoy_http_downstream_rq_xx{namespace=\"projectcontour\"}[5m]))",
|
|
"legendFormat": "{{envoy_response_code_class}}xx"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "list", "placement": "bottom" } }
|
|
},
|
|
|
|
{ "type": "row", "title": "Service level (by namespace)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 101 },
|
|
|
|
{
|
|
"type": "timeseries", "title": "CPU usage by namespace", "id": 10,
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))",
|
|
"legendFormat": "{{namespace}}"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Memory usage by namespace", "id": 11,
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})",
|
|
"legendFormat": "{{namespace}}"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } }
|
|
},
|
|
{
|
|
"type": "table", "title": "Current usage per namespace", "id": 12,
|
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [
|
|
{ "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", "format": "table", "instant": true, "refId": "A" },
|
|
{ "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", "format": "table", "instant": true, "refId": "B" },
|
|
{ "expr": "count by (namespace) (count by (namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}))", "format": "table", "instant": true, "refId": "C" }
|
|
],
|
|
"transformations": [
|
|
{ "id": "merge", "options": {} },
|
|
{ "id": "organize", "options": {
|
|
"excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "Time 3": true },
|
|
"renameByName": { "Value #A": "CPU (cores)", "Value #B": "Memory", "Value #C": "Pods" }
|
|
} }
|
|
],
|
|
"fieldConfig": { "defaults": {}, "overrides": [
|
|
{ "matcher": { "id": "byName", "options": "Memory" }, "properties": [{ "id": "unit", "value": "bytes" }] },
|
|
{ "matcher": { "id": "byName", "options": "CPU (cores)" }, "properties": [{ "id": "unit", "value": "short" }, { "id": "decimals", "value": 3 }] }
|
|
] }
|
|
},
|
|
|
|
{ "type": "row", "title": "Cluster utilization", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 102 },
|
|
|
|
{
|
|
"type": "gauge", "title": "CPU utilization", "id": 20,
|
|
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)" }],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100,
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }
|
|
] } }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "gauge", "title": "Memory utilization", "id": 21,
|
|
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)" }],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100,
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }
|
|
] } }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Cluster CPU utilization over time", "id": 22,
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [
|
|
{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)", "legendFormat": "CPU %" },
|
|
{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)", "legendFormat": "Memory %" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] }
|
|
},
|
|
|
|
{ "type": "row", "title": "Total resources", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 103 },
|
|
|
|
{
|
|
"type": "stat", "title": "Node CPU capacity", "id": 30,
|
|
"gridPos": { "h": 5, "w": 4, "x": 0, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "max(machine_cpu_cores)" }],
|
|
"fieldConfig": { "defaults": { "unit": "short", "displayName": "cores" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Node memory capacity", "id": 31,
|
|
"gridPos": { "h": 5, "w": 4, "x": 4, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "max(machine_memory_bytes)" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "CPU used (cluster)", "id": 32,
|
|
"gridPos": { "h": 5, "w": 4, "x": 8, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m]))" }],
|
|
"fieldConfig": { "defaults": { "unit": "short", "displayName": "cores", "decimals": 2 }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Memory used (cluster)", "id": 33,
|
|
"gridPos": { "h": 5, "w": 4, "x": 12, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"})" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Running pods", "id": 34,
|
|
"gridPos": { "h": 5, "w": 4, "x": 16, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "count(count by (namespace, pod) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))" }],
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Disk free (root)", "id": 35,
|
|
"gridPos": { "h": 5, "w": 4, "x": 20, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes",
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "red", "value": null }, { "color": "yellow", "value": 5000000000 }, { "color": "green", "value": 15000000000 }
|
|
] } }, "overrides": [] }
|
|
}
|
|
]
|
|
}
|
|
|
|
ingress:
|
|
enabled: true
|
|
ingressClassName: contour
|
|
path: /
|
|
hosts:
|
|
- grafana.35.238.248.203.nip.io
|