Mukul SharmaandClaude Opus 5
f703f0b55a
Each of these now answers on <name>.infra.deployshed.com alongside the nip.io name it already had. Both are kept on purpose: nothing that currently references the old name breaks, and the new one can be proved before anything depends on it. Removing nip.io is a separate step, and a larger one, because Harbor's name is embedded in every running app's image reference. TLS covers the real domain only. Let's Encrypt cannot issue for nip.io — it is not on the public suffix list and every *.nip.io certificate shares one rate limit — so a tls block naming both would request one certificate spanning them and receive nothing for either. Each tls block therefore lists exactly the one new hostname, which is why they are written out rather than derived from the host list beside them. The charts disagree about how to express a second host, so each is done the way its own chart supports: gitea, grafana, vault, victoria-metrics-single take host lists, so the new name joins the existing one on a single Ingress. jenkins' primary ingress accepts exactly one hostName, so the new name goes on secondaryingress — a whole second Ingress object at the same backend. paths must be set explicitly there; left at the chart's default of [] it renders zero routes and the hostname answers nothing. argo-cd takes extraHosts natively, but its ingress.tls is a boolean bound to one fixed secret covering every host at once. Turning it on would request a certificate including nip.io and fail, and there is no extraTls to scope it. So ArgoCD gains the hostname now and its certificate when nip.io goes. Harbor is untouched here. It has no multi-host mechanism at all, so its second hostname needs a standalone Ingress, and its externalURL is what docker clients are handed — both deserve their own change rather than riding along with a hostname tidy-up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LEsTefWWifp4ikvhHF5s6N
283 lines
15 KiB
YAML
283 lines
15 KiB
YAML
grafana:
|
|
# From Vault via ExternalSecret (devops-infra-argo-config/secretstores/
|
|
# grafana-admin-credentials.yaml), same pattern as gitea/harbor/jenkins
|
|
# admin credentials elsewhere in this project — never a plaintext
|
|
# adminPassword in this file.
|
|
admin:
|
|
existingSecret: grafana-admin-credentials
|
|
userKey: username
|
|
passwordKey: password
|
|
|
|
persistence:
|
|
# 1Gi, not the chart's 10Gi default: this holds dashboards, folders and
|
|
# Grafana's own sqlite state, not metric data — VictoriaMetrics keeps
|
|
# that. Unlike the homelab's local-path, standard-rwo can be expanded
|
|
# later if that ever proves tight.
|
|
enabled: true
|
|
storageClassName: standard-rwo
|
|
size: 1Gi
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
memory: 384Mi
|
|
|
|
# Provisioned at boot, not clicked through in the UI — the same reason
|
|
# every other credential/config in this project is committed rather
|
|
# than set by hand: it survives a pod restart and a fresh install gets
|
|
# it automatically. VictoriaMetrics speaks Prometheus's own query API,
|
|
# so `type: prometheus` here is correct even though the URL is VM's —
|
|
# see this repo's victoria-metrics-single chart for why.
|
|
datasources:
|
|
datasources.yaml:
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: VictoriaMetrics
|
|
# Fixed uid, not left to auto-generate — the provisioned
|
|
# dashboard below references this datasource by uid, and an
|
|
# auto-generated one would only exist after Grafana's first
|
|
# boot, too late for a dashboard provisioned in the same boot.
|
|
uid: victoriametrics
|
|
type: prometheus
|
|
access: proxy
|
|
url: http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428
|
|
isDefault: true
|
|
|
|
# Dashboard provisioning. Provisioned rather than built by hand in the
|
|
# UI for the same reason the datasource above is: it survives a pod
|
|
# restart (this deployment has no persistent Grafana database beyond
|
|
# the 1Gi PVC, and even with one, a fresh install should not start
|
|
# with an empty dashboard list) and a `git diff` shows what changed.
|
|
dashboardProviders:
|
|
dashboardproviders.yaml:
|
|
apiVersion: 1
|
|
providers:
|
|
- name: default
|
|
orgId: 1
|
|
folder: ""
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards/default
|
|
|
|
# "Homelab Overview" — total Envoy/Contour RPS, per-namespace CPU and
|
|
# memory (the $namespace template variable filters every relevant
|
|
# panel), cluster-wide utilization against actual node capacity, and a
|
|
# total-resources row (cores/memory/pods/disk). Envoy and node-exporter
|
|
# metrics both required their own vmagent scrape job — see
|
|
# helm-overrides/.../vmagent/custom-values.yaml for why neither was
|
|
# reachable through the chart's own defaults in this cluster.
|
|
#
|
|
# Every panel except "Disk free" was run against the live deployment
|
|
# (vmui, over Tailscale) before being written in here — RPS, per-
|
|
# namespace CPU/memory, machine_cpu_cores/machine_memory_bytes all
|
|
# returned real data. "Disk free" depends on the node-exporter scrape
|
|
# job added alongside this same change, which had not been live yet to
|
|
# verify against — worth checking once this actually deploys, same as
|
|
# everything else in this repo that gets a `helm template` check but
|
|
# cannot get a live one before the first sync.
|
|
dashboards:
|
|
default:
|
|
homelab:
|
|
json: |
|
|
{
|
|
"title": "Homelab Overview",
|
|
"uid": "homelab-overview",
|
|
"schemaVersion": 39,
|
|
"editable": true,
|
|
"timezone": "browser",
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"refresh": "30s",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"query": "label_values(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}, namespace)",
|
|
"refresh": 2,
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"current": { "selected": true, "text": "All", "value": "$__all" }
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{ "type": "row", "title": "Ingress (Envoy / Contour)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100 },
|
|
|
|
{
|
|
"type": "stat", "title": "Total RPS", "id": 1,
|
|
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(rate(envoy_http_downstream_rq_total{namespace=\"projectcontour\"}[5m]))", "legendFormat": "rps" }],
|
|
"fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "graphMode": "area" }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Active downstream connections", "id": 2,
|
|
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(envoy_http_downstream_cx_active{namespace=\"projectcontour\"})", "legendFormat": "connections" }],
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Requests by response class", "id": 3,
|
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (envoy_response_code_class) (rate(envoy_http_downstream_rq_xx{namespace=\"projectcontour\"}[5m]))",
|
|
"legendFormat": "{{envoy_response_code_class}}xx"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "list", "placement": "bottom" } }
|
|
},
|
|
|
|
{ "type": "row", "title": "Service level (by namespace)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 101 },
|
|
|
|
{
|
|
"type": "timeseries", "title": "CPU usage by namespace", "id": 10,
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))",
|
|
"legendFormat": "{{namespace}}"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "short", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Memory usage by namespace", "id": 11,
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{
|
|
"expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})",
|
|
"legendFormat": "{{namespace}}"
|
|
}],
|
|
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "fillOpacity": 10, "stacking": { "mode": "normal" } } }, "overrides": [] },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } }
|
|
},
|
|
{
|
|
"type": "table", "title": "Current usage per namespace", "id": 12,
|
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [
|
|
{ "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}[5m]))", "format": "table", "instant": true, "refId": "A" },
|
|
{ "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"})", "format": "table", "instant": true, "refId": "B" },
|
|
{ "expr": "count by (namespace) (count by (namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", container!=\"POD\"}))", "format": "table", "instant": true, "refId": "C" }
|
|
],
|
|
"transformations": [
|
|
{ "id": "merge", "options": {} },
|
|
{ "id": "organize", "options": {
|
|
"excludeByName": { "Time": true, "Time 1": true, "Time 2": true, "Time 3": true },
|
|
"renameByName": { "Value #A": "CPU (cores)", "Value #B": "Memory", "Value #C": "Pods" }
|
|
} }
|
|
],
|
|
"fieldConfig": { "defaults": {}, "overrides": [
|
|
{ "matcher": { "id": "byName", "options": "Memory" }, "properties": [{ "id": "unit", "value": "bytes" }] },
|
|
{ "matcher": { "id": "byName", "options": "CPU (cores)" }, "properties": [{ "id": "unit", "value": "short" }, { "id": "decimals", "value": 3 }] }
|
|
] }
|
|
},
|
|
|
|
{ "type": "row", "title": "Cluster utilization", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 102 },
|
|
|
|
{
|
|
"type": "gauge", "title": "CPU utilization", "id": 20,
|
|
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)" }],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100,
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }
|
|
] } }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "gauge", "title": "Memory utilization", "id": 21,
|
|
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)" }],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100,
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }
|
|
] } }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "timeseries", "title": "Cluster CPU utilization over time", "id": 22,
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [
|
|
{ "expr": "100 * sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m])) / sum(machine_cpu_cores)", "legendFormat": "CPU %" },
|
|
{ "expr": "100 * sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}) / sum(machine_memory_bytes)", "legendFormat": "Memory %" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0 }, "overrides": [] }
|
|
},
|
|
|
|
{ "type": "row", "title": "Total resources", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 103 },
|
|
|
|
{
|
|
"type": "stat", "title": "Node CPU capacity", "id": 30,
|
|
"gridPos": { "h": 5, "w": 4, "x": 0, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "max(machine_cpu_cores)" }],
|
|
"fieldConfig": { "defaults": { "unit": "short", "displayName": "cores" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Node memory capacity", "id": 31,
|
|
"gridPos": { "h": 5, "w": 4, "x": 4, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "max(machine_memory_bytes)" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "CPU used (cluster)", "id": 32,
|
|
"gridPos": { "h": 5, "w": 4, "x": 8, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\", container!=\"POD\"}[5m]))" }],
|
|
"fieldConfig": { "defaults": { "unit": "short", "displayName": "cores", "decimals": 2 }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Memory used (cluster)", "id": 33,
|
|
"gridPos": { "h": 5, "w": 4, "x": 12, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "sum(container_memory_working_set_bytes{container!=\"\", container!=\"POD\"})" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Running pods", "id": 34,
|
|
"gridPos": { "h": 5, "w": 4, "x": 16, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "count(count by (namespace, pod) (container_memory_working_set_bytes{container!=\"\", container!=\"POD\"}))" }],
|
|
"fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }
|
|
},
|
|
{
|
|
"type": "stat", "title": "Disk free (root)", "id": 35,
|
|
"gridPos": { "h": 5, "w": 4, "x": 20, "y": 33 },
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"targets": [{ "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}" }],
|
|
"fieldConfig": { "defaults": { "unit": "bytes",
|
|
"thresholds": { "mode": "absolute", "steps": [
|
|
{ "color": "red", "value": null }, { "color": "yellow", "value": 5000000000 }, { "color": "green", "value": 15000000000 }
|
|
] } }, "overrides": [] }
|
|
}
|
|
]
|
|
}
|
|
|
|
ingress:
|
|
enabled: true
|
|
ingressClassName: contour
|
|
annotations:
|
|
# Only the real domain appears in tls below — Let's Encrypt cannot
|
|
# issue for nip.io, and one certificate spanning both would fail
|
|
# outright rather than covering the half it can serve.
|
|
cert-manager.io/cluster-issuer: letsencrypt-prod
|
|
path: /
|
|
hosts:
|
|
- grafana.35.238.248.203.nip.io
|
|
- grafana.infra.deployshed.com
|
|
tls:
|
|
- secretName: grafana-tls
|
|
hosts:
|
|
- grafana.infra.deployshed.com
|