From 4a5f4d94abc4a72c3911a890b5bafdfbd02d9b30 Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Wed, 9 Sep 2026 12:35:35 +0530 Subject: [PATCH 1/2] Grant deployer Ingress rights in its own namespace, for custom domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overdue: this belongs with toolshed's custom-domains feature, which has already merged and shipped. Without it, deployer gets "forbidden" the moment anyone actually adds a custom domain — the exact failure mode internal/deploy/kubernetes.go's own package doc warns about, since its resource list and this file are unsynchronised copies in two repositories. Scoped as a namespaced Role/RoleBinding rather than widening the ClusterRole above it. A custom domain's Ingress and TLS secret live in the gateway's own namespace and never in an app's, so this only ever targets one fixed namespace — unlike the per-app namespaces the ClusterRole necessarily spans. Secrets are get/list/watch/delete only: cert-manager writes them, deployer just reads readiness and cleans up on removal. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Wajog7nELA3i8JWTjxYGHF --- extra-manifests/toolshed-deployer-rbac.yaml | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/extra-manifests/toolshed-deployer-rbac.yaml b/extra-manifests/toolshed-deployer-rbac.yaml index 8ee8ea2..d24a966 100644 --- a/extra-manifests/toolshed-deployer-rbac.yaml +++ b/extra-manifests/toolshed-deployer-rbac.yaml @@ -85,3 +85,47 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: toolshed-deployer +--- +# A custom domain's Ingress and TLS secret live in the gateway's own +# namespace (toolshed, same as above) — never an app's namespace. Scoped +# with a namespaced Role/RoleBinding rather than widening the ClusterRole +# above: Ingress management here only ever targets this one fixed +# namespace, unlike the per-app namespaces the ClusterRole necessarily +# spans. Added alongside toolshed's custom-domains feature — see +# internal/deploy.Client.EnsureDomainIngress/DomainCertReady/ +# RemoveDomainIngress and this file's own header comment about keeping it +# and deploy/helm/toolshed/templates/rbac.yaml in sync. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: toolshed-deployer-ingress + namespace: toolshed + labels: + app.kubernetes.io/part-of: toolshed +rules: + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # Reads the TLS secret cert-manager's ingress-shim populates for a domain + # Ingress, and deletes it (and the Ingress above) when a domain is + # removed. Never create/update — cert-manager, not deployer, writes this + # secret. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: toolshed-deployer-ingress + namespace: toolshed + labels: + app.kubernetes.io/part-of: toolshed +subjects: + - kind: ServiceAccount + name: toolshed-deployer + namespace: toolshed +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: toolshed-deployer-ingress From dcd35836569f902f7506e232a8bec706dbe3749f Mon Sep 17 00:00:00 2001 From: Mukul Sharma Date: Wed, 9 Sep 2026 12:35:46 +0530 Subject: [PATCH 2/2] Deploy Redis, backing toolshed's managed cache add-on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers the new hand-written redis chart (devops-infra-helm-charts, separate commit) and the ExternalSecret feeding its admin password from Vault. Own namespace, addressed over cluster DNS like every other platform component here: redis.redis.svc.cluster.local:6379 Only one consumer for the credential, unlike the Postgres one next door: the server itself, to seed its ACL file on first boot. toolshed's api gets it from the connection an operator configures in the dashboard, encrypted in toolshed's own database — so there is deliberately no second ExternalSecret into the toolshed namespace. Order matters: put the password in Vault at secret/toolshed/redis before syncing, or the init container sits in CreateContainerConfigError. The exact command, the reason the password must be alphanumeric (it is written into an ACL directive where a space or quote would split it), and the manual rotation procedure are all recorded in the ExternalSecret's own header. Nothing here needs to change for Postgres: toolshed's managed database add-on points at the existing postgresql.postgres.svc.cluster.local, whose POSTGRES_USER is the initdb superuser and so already has the CREATEDB and CREATEROLE that provisioning needs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Wajog7nELA3i8JWTjxYGHF --- secretstores/toolshed-redis-credentials.yaml | 54 +++++++++++++++++++ ...bator-infra-k8s-admin-prd-ase1-values.yaml | 21 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 secretstores/toolshed-redis-credentials.yaml diff --git a/secretstores/toolshed-redis-credentials.yaml b/secretstores/toolshed-redis-credentials.yaml new file mode 100644 index 0000000..cf02457 --- /dev/null +++ b/secretstores/toolshed-redis-credentials.yaml @@ -0,0 +1,54 @@ +# Redis admin password, backing toolshed's managed cache add-on. +# +# Only one consumer, unlike the Postgres credential next door: the Redis +# server itself needs it to seed its ACL file on first boot. toolshed's api +# reads it from the *connection* an operator configures in the dashboard +# (encrypted in toolshed's own database via the secretbox keyring), not from +# a Kubernetes Secret — so there is deliberately no second ExternalSecret +# into the toolshed namespace here. +# +# Put the credential in Vault BEFORE syncing this. External Secrets cannot +# create a Secret for a path that does not exist, and the Redis pod's init +# container will sit in CreateContainerConfigError until it can: +# +# kubectl -n vault exec -i vault-0 -- sh -lc ' +# vault login >/dev/null && +# vault kv put secret/toolshed/redis \ +# password=' +# +# Use an alphanumeric password. It is written into the ACL file as +# `user default on > ...` by the init container, where a space or +# a quote would split the directive and produce a server that either fails +# to start or, worse, starts with different rules than intended. +# +# Remember that `kubectl exec` into Vault is unauthenticated by default — +# without the `vault login` the commands fail with a "preflight capability +# check" error that reads like a permissions bug rather than a missing +# login. +# +# Rotating this password later does NOT propagate to a running server: the +# init container only ever writes the ACL file when it is absent, precisely +# so it cannot delete the per-app users toolshed has provisioned into it. +# To rotate, update Vault and then, against the running server: +# +# ACL SETUSER default >newpassword +# ACL SAVE +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: redis-credentials + namespace: redis +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: redis-credentials + creationPolicy: Owner + data: + - secretKey: password + remoteRef: + key: toolshed/redis + property: password diff --git a/values/incubator-infra-k8s-admin-prd-ase1-values.yaml b/values/incubator-infra-k8s-admin-prd-ase1-values.yaml index a00678a..c2f538a 100644 --- a/values/incubator-infra-k8s-admin-prd-ase1-values.yaml +++ b/values/incubator-infra-k8s-admin-prd-ase1-values.yaml @@ -132,6 +132,27 @@ appSpec: namespace: postgres chartDir: postgresql valuesDir: postgresql + - name: redis + # Backs toolshed's managed cache add-on — toolshed provisions a per-app + # ACL user, scoped to its own key prefix, on request. Own namespace for + # the same reason postgresql has one: addressed over cluster DNS like + # any other platform component, outliving whatever consumes it: + # redis.redis.svc.cluster.local:6379 + # + # Hand-written chart, not Bitnami's, for the same reason as postgresql + # (infra issue #4) — Redis ships no official chart either. + # + # Authentication is defined by an ACL file with no requirepass, which + # is a security property rather than a preference: see the chart's own + # values.yaml, where getting it wrong leaves the server open to + # unauthenticated access after its first restart. + # + # Requires secretstores/toolshed-redis-credentials.yaml to have synced + # first — the init container cannot seed the ACL file without it. + nameOverride: redis + namespace: redis + chartDir: redis + valuesDir: redis - name: victoria-metrics-single # Replaces the Prometheus server this entry briefly was (see git # history on this file) — same job, lower RAM/disk footprint for the