From 613f8b0a8f3a05bec0a8375b5f2870dddd94e4e9 Mon Sep 17 00:00:00 2001 From: Gabriel Radureau Date: Sat, 25 Jul 2026 23:43:26 +0200 Subject: [PATCH] =?UTF-8?q?fix(ops):=20pin=20the=20kube-context=20?= =?UTF-8?q?=E2=80=94=20never=20run=20destructive=20steps=20on=20the=20ambi?= =?UTF-8?q?ent=20one?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sandbox-lifecycle.sh scales deployments to zero, patches the ArgoCD Application and runs DROP OWNED ... CASCADE. Every one of those ran against whatever kube-context happened to be current. This workstation also carries a CLIENT production cluster. On 2026-07-25 a `checkpoint refresh` was issued while the current context was do-nyc3-kissmetrics-prod-k8s-cluster: the script patched the ArgoCD Application, scaled `erp-sandbox` to zero and copied a prod secret — all against the client's cluster. Nothing was damaged only because that cluster has no `application` CRD and no erp/erp-sandbox namespaces, so each call failed silently under `|| true`. That is luck, not a control. - ERP_KUBE_CONTEXT (default: "default") pins the target; every kubectl call now goes through K(), so nothing inherits the ambient context. - assert_arcodange_cluster() proves the target by positive fingerprint — the erp, erp-sandbox and argocd namespaces AND the erp-sandbox ArgoCD Application. A client cluster cannot match all four by accident. Wired into all three entry points, before any mutation. Verified: refuses the client context, refuses an unknown context, passes on the homelab and completes normally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VRShc4QhLLU73FLHx9vskh --- ops/sandbox/sandbox-lifecycle.sh | 70 ++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/ops/sandbox/sandbox-lifecycle.sh b/ops/sandbox/sandbox-lifecycle.sh index f17f624..7761d24 100755 --- a/ops/sandbox/sandbox-lifecycle.sh +++ b/ops/sandbox/sandbox-lifecycle.sh @@ -38,20 +38,48 @@ TMP_PROD_SECRET="prod-db-ro-temp" # transient copy of prod creds, deleted on e log() { printf '\033[1;36m==>\033[0m %s\n' "$*"; } die() { printf '\033[1;31mABORT:\033[0m %s\n' "$*" >&2; exit 1; } -sb_pod() { kubectl get pod -n "$SB_NS" -l app.kubernetes.io/instance=erp-sandbox -o name 2>/dev/null | head -1; } -prod_pod() { kubectl get pod -n "$PROD_NS" -l app.kubernetes.io/instance=erp -o name 2>/dev/null | head -1; } +sb_pod() { K get pod -n "$SB_NS" -l app.kubernetes.io/instance=erp-sandbox -o name 2>/dev/null | head -1; } +prod_pod() { K get pod -n "$PROD_NS" -l app.kubernetes.io/instance=erp -o name 2>/dev/null | head -1; } -cleanup_secret() { kubectl delete secret "$TMP_PROD_SECRET" -n "$SB_NS" --ignore-not-found >/dev/null 2>&1 || true; } +cleanup_secret() { K delete secret "$TMP_PROD_SECRET" -n "$SB_NS" --ignore-not-found >/dev/null 2>&1 || true; } # erp-sandbox is ArgoCD-managed with self-heal ON, which reverts `kubectl scale # --replicas=0` within seconds — so without pausing it the seed runs while Dolibarr # is still connected, and the restore collides with the app re-creating tables. # Pause self-heal for the duration so the scale-down holds; always re-arm it. ARGOCD_NS="argocd"; ARGOCD_APP="erp-sandbox" -set_selfheal() { kubectl patch application "$ARGOCD_APP" -n "$ARGOCD_NS" --type merge \ + +# --- Cluster guard ----------------------------------------------------------- +# This script scales deployments to zero, patches the ArgoCD Application and runs +# `DROP OWNED ... CASCADE`. Until now every one of those ran against whatever +# kube-context happened to be current — and this workstation also carries a +# CLIENT production cluster (observed 2026-07-25: the current context was the +# client's, and refresh only failed because that cluster has no `erp` namespace). +# Never trust the ambient context: resolve an explicit one and prove it is the +# Arcodange homelab before touching anything. +ERP_KUBE_CONTEXT="${ERP_KUBE_CONTEXT:-default}" +K() { kubectl --context "$ERP_KUBE_CONTEXT" "$@"; } + +assert_arcodange_cluster() { + kubectl config get-contexts -o name 2>/dev/null | grep -qx "$ERP_KUBE_CONTEXT" \ + || die "kube-context '$ERP_KUBE_CONTEXT' does not exist (set ERP_KUBE_CONTEXT)" + # Positive fingerprint: the three namespaces AND the ArgoCD Application this + # script drives. A client cluster cannot match all four by accident. + for ns in "$PROD_NS" "$SB_NS" "$ARGOCD_NS"; do + K get ns "$ns" >/dev/null 2>&1 \ + || die "kube-context '$ERP_KUBE_CONTEXT' has no '$ns' namespace — refusing to run against it. + This script is destructive (scale-to-0, DROP OWNED CASCADE, ArgoCD patch) and must only + ever target the Arcodange homelab. Current context is '$(kubectl config current-context 2>/dev/null)'. + Set ERP_KUBE_CONTEXT to the homelab context and retry." + done + K get application "$ARGOCD_APP" -n "$ARGOCD_NS" >/dev/null 2>&1 \ + || die "kube-context '$ERP_KUBE_CONTEXT' has no ArgoCD Application '$ARGOCD_APP' — refusing to run." + log "cluster guard OK — context '$ERP_KUBE_CONTEXT' (namespaces $PROD_NS/$SB_NS/$ARGOCD_NS + app $ARGOCD_APP)" +} +set_selfheal() { K patch application "$ARGOCD_APP" -n "$ARGOCD_NS" --type merge \ -p "{\"spec\":{\"syncPolicy\":{\"automated\":{\"selfHeal\":$1,\"prune\":true}}}}" >/dev/null 2>&1 || true; } # Safety net (EXIT trap): whatever happens, bring the app back, re-arm self-heal, drop the secret. -restore_state() { set_selfheal true; kubectl scale deploy erp-sandbox -n "$SB_NS" --replicas=1 >/dev/null 2>&1 || true; cleanup_secret; } +restore_state() { set_selfheal true; K scale deploy erp-sandbox -n "$SB_NS" --replicas=1 >/dev/null 2>&1 || true; cleanup_secret; } refresh_from_prod() { command -v python3 >/dev/null || die "python3 required to copy the prod secret without exposing it" @@ -61,17 +89,17 @@ refresh_from_prod() { set_selfheal false log "Copying prod DB creds into a transient, read-only-intent secret in $SB_NS (values stay base64)" - kubectl get secret vso-db-credentials -n "$PROD_NS" -o json \ + K get secret vso-db-credentials -n "$PROD_NS" -o json \ | python3 -c "import json,sys; d=json.load(sys.stdin); d['metadata']={'name':'$TMP_PROD_SECRET','namespace':'$SB_NS'}; d.pop('status',None); d['data']={k:d['data'][k] for k in ('username','password')}; print(json.dumps(d))" \ - | kubectl apply -f - >/dev/null + | K apply -f - >/dev/null log "Scaling erp-sandbox to 0 (exclusive DB access for the restore)" - kubectl scale deploy erp-sandbox -n "$SB_NS" --replicas=0 >/dev/null - kubectl wait --for=delete pod -l app.kubernetes.io/instance=erp-sandbox -n "$SB_NS" --timeout=120s >/dev/null 2>&1 || true + K scale deploy erp-sandbox -n "$SB_NS" --replicas=0 >/dev/null + K wait --for=delete pod -l app.kubernetes.io/instance=erp-sandbox -n "$SB_NS" --timeout=120s >/dev/null 2>&1 || true log "Running the seed Job (pg_dump prod read-only -> DROP OWNED -> pg_restore into sandbox)" - kubectl delete job sandbox-seed -n "$SB_NS" --ignore-not-found >/dev/null 2>&1 || true - kubectl apply -f - >/dev/null </dev/null 2>&1 || true + K apply -f - >/dev/null </dev/null 2>&1 \ - || die "seed Job did not complete — see: kubectl logs -n $SB_NS job/sandbox-seed" - kubectl logs -n "$SB_NS" job/sandbox-seed | sed 's/^/ /' - kubectl delete job sandbox-seed -n "$SB_NS" --ignore-not-found >/dev/null 2>&1 || true + K wait --for=condition=complete job/sandbox-seed -n "$SB_NS" --timeout=300s >/dev/null 2>&1 \ + || die "seed Job did not complete — see: K logs -n $SB_NS job/sandbox-seed" + K logs -n "$SB_NS" job/sandbox-seed | sed 's/^/ /' + K delete job sandbox-seed -n "$SB_NS" --ignore-not-found >/dev/null 2>&1 || true log "Restoring app (replicas=1) + re-arming ArgoCD self-heal" set_selfheal true - kubectl scale deploy erp-sandbox -n "$SB_NS" --replicas=1 >/dev/null + K scale deploy erp-sandbox -n "$SB_NS" --replicas=1 >/dev/null cleanup_secret; trap - EXIT log "Refresh complete. Run 'sync-documents' to also copy the company logo + uploads." } @@ -140,14 +168,14 @@ sync_documents() { [ -n "$pp" ] || die "no prod erp pod found" [ -n "$sp" ] || die "no erp-sandbox pod found" log "Syncing $DOC_ROOT/mycompany (logo + uploads) ${pp##*/} -> ${sp##*/} via tar pipe" - kubectl exec -n "$PROD_NS" "${pp#pod/}" -- tar -C "$DOC_ROOT" -cf - mycompany 2>/dev/null \ - | kubectl exec -i -n "$SB_NS" "${sp#pod/}" -- tar -C "$DOC_ROOT" -xf - + K exec -n "$PROD_NS" "${pp#pod/}" -- tar -C "$DOC_ROOT" -cf - mycompany 2>/dev/null \ + | K exec -i -n "$SB_NS" "${sp#pod/}" -- tar -C "$DOC_ROOT" -xf - log "Documents synced. (For a one-shot logo only, scope the tar to mycompany/logos.)" } case "${1:-}" in - refresh-from-prod) refresh_from_prod ;; - sync-documents) sync_documents ;; - refresh) refresh_from_prod; sync_documents ;; + refresh-from-prod) assert_arcodange_cluster; refresh_from_prod ;; + sync-documents) assert_arcodange_cluster; sync_documents ;; + refresh) assert_arcodange_cluster; refresh_from_prod; sync_documents ;; *) echo "usage: $0 {refresh-from-prod|sync-documents|refresh}" >&2; exit 2 ;; esac -- 2.54.0