From da93bbc9f343b6396a4e6f7c81029256560d92c2 Mon Sep 17 00:00:00 2001 From: Yuriy Panov Date: Thu, 17 Sep 2026 22:33:54 +0300 Subject: [PATCH] Remove the release directory a failed deploy leaves behind A deploy that died after mkdir -- a broken build, a missing lock file, a health check that never went green -- left its half-finished release in releases/. That is not just disk: it is the newest directory there, so the KEEP=3 rotation of the next successful deploy counted it among the three to keep and dropped a working release instead, thinning out exactly the rollback targets the runbook tells you to use. The release directory is now armed for cleanup the moment it is created and disarmed once the release is live and healthy, with the removal done by the same EXIT trap that drops the unpacked archive -- so it covers every way out: fatal, a command failing under set -e, an aborted build. The one case left alone is a release the swap did reach and that `current` still points at, i.e. a first-ever deploy with nothing to roll back to: removing that would leave a dangling symlink behind instead. Co-Authored-By: Claude Opus 5 --- deploy/README.md | 5 ++++- deploy/bin/exo-deploy | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 576837a..86f1032 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -267,7 +267,10 @@ cat /srv/exo//current/.deploy-source | Disk usage | `du -sh /srv/exo/*/releases/*` | `exo-deploy` health-checks after the swap and **rolls back automatically** if the new -release fails to answer `/api/health`. +release fails to answer `/api/health`. A deploy that fails anywhere — a broken build, a +missing lock file, a health check that never goes green — takes its half-finished release +directory with it, so the three kept releases are always three releases that actually ran +and any one of them is a safe rollback target. ## Monitoring diff --git a/deploy/bin/exo-deploy b/deploy/bin/exo-deploy index 0cb4a33..893c0a2 100755 --- a/deploy/bin/exo-deploy +++ b/deploy/bin/exo-deploy @@ -91,7 +91,21 @@ fi # ------------------------------------------------------- archive, if any STAGE="" -cleanup() { if [[ -n $STAGE ]]; then rm -rf "$STAGE"; fi; } +PENDING_REL="" # release being built; cleared once it is live and healthy +PENDING_LINK="" # its app's `current` symlink + +# A release that never went live is not just wasted disk: it is the newest +# directory in releases/, so the KEEP rotation below would keep it and drop a +# working release instead — the very one the rollback in the runbook needs. +# Anything the swap did reach is left alone, dangling symlinks included. +cleanup() { + if [[ -n $STAGE ]]; then rm -rf "$STAGE"; fi + if [[ -n $PENDING_REL && -d $PENDING_REL ]] && + [[ "$(readlink -f "$PENDING_LINK" 2>/dev/null || true)" != "$PENDING_REL" ]]; then + echo "==> removing incomplete release $PENDING_REL" + rm -rf "$PENDING_REL" + fi +} trap cleanup EXIT # Shallowest / directory in the archive that holds a package.json, so both @@ -168,6 +182,8 @@ deploy_app() { [[ -n $PORT ]] || fatal "PORT not set in $ENVFILE" mkdir -p "$REL" + PENDING_REL="$REL" + PENDING_LINK="$BASE/current" if [[ -n $ZIP ]]; then src="$(app_src "$APP")" || fatal "$ZIP: no $APP/ directory with a package.json inside" @@ -185,7 +201,7 @@ deploy_app() { echo "==> extracting $APP/ into $REL" git -C "$BASE/repo" archive "$REF" "$APP" | tar -x -C "$REL" --strip-components=1 - [[ -f $REL/package.json ]] || { rm -rf "$REL"; fatal "$APP/ not found at $REF"; } + [[ -f $REL/package.json ]] || fatal "$APP/ not found at $REF" desc="git $REF $(git -C "$BASE/repo" rev-parse --short "$REF")" fi @@ -251,6 +267,10 @@ deploy_app() { grep -q '"ok":true' <<<"$HEALTH" || fatal "health not ok" grep -q '"amo":true' <<<"$HEALTH" || echo " WARNING: amo=false — /api/leads/* will return 503. Check AMO_* in $ENVFILE." + # Live and healthy: from here on it is a release like any other. + PENDING_REL="" + PENDING_LINK="" + # Prune old releases, never the live one. CURRENT="$(readlink -f "$BASE/current")" ls -1dt "$BASE"/releases/*/ 2>/dev/null | tail -n "+$((KEEP+1))" | while read -r old; do