diff --git a/deploy/README.md b/deploy/README.md index 576837a..86f1032 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -267,7 +267,10 @@ cat /srv/exo//current/.deploy-source | Disk usage | `du -sh /srv/exo/*/releases/*` | `exo-deploy` health-checks after the swap and **rolls back automatically** if the new -release fails to answer `/api/health`. +release fails to answer `/api/health`. A deploy that fails anywhere — a broken build, a +missing lock file, a health check that never goes green — takes its half-finished release +directory with it, so the three kept releases are always three releases that actually ran +and any one of them is a safe rollback target. ## Monitoring diff --git a/deploy/bin/exo-deploy b/deploy/bin/exo-deploy index 0cb4a33..893c0a2 100755 --- a/deploy/bin/exo-deploy +++ b/deploy/bin/exo-deploy @@ -91,7 +91,21 @@ fi # ------------------------------------------------------- archive, if any STAGE="" -cleanup() { if [[ -n $STAGE ]]; then rm -rf "$STAGE"; fi; } +PENDING_REL="" # release being built; cleared once it is live and healthy +PENDING_LINK="" # its app's `current` symlink + +# A release that never went live is not just wasted disk: it is the newest +# directory in releases/, so the KEEP rotation below would keep it and drop a +# working release instead — the very one the rollback in the runbook needs. +# Anything the swap did reach is left alone, dangling symlinks included. +cleanup() { + if [[ -n $STAGE ]]; then rm -rf "$STAGE"; fi + if [[ -n $PENDING_REL && -d $PENDING_REL ]] && + [[ "$(readlink -f "$PENDING_LINK" 2>/dev/null || true)" != "$PENDING_REL" ]]; then + echo "==> removing incomplete release $PENDING_REL" + rm -rf "$PENDING_REL" + fi +} trap cleanup EXIT # Shallowest / directory in the archive that holds a package.json, so both @@ -168,6 +182,8 @@ deploy_app() { [[ -n $PORT ]] || fatal "PORT not set in $ENVFILE" mkdir -p "$REL" + PENDING_REL="$REL" + PENDING_LINK="$BASE/current" if [[ -n $ZIP ]]; then src="$(app_src "$APP")" || fatal "$ZIP: no $APP/ directory with a package.json inside" @@ -185,7 +201,7 @@ deploy_app() { echo "==> extracting $APP/ into $REL" git -C "$BASE/repo" archive "$REF" "$APP" | tar -x -C "$REL" --strip-components=1 - [[ -f $REL/package.json ]] || { rm -rf "$REL"; fatal "$APP/ not found at $REF"; } + [[ -f $REL/package.json ]] || fatal "$APP/ not found at $REF" desc="git $REF $(git -C "$BASE/repo" rev-parse --short "$REF")" fi @@ -251,6 +267,10 @@ deploy_app() { grep -q '"ok":true' <<<"$HEALTH" || fatal "health not ok" grep -q '"amo":true' <<<"$HEALTH" || echo " WARNING: amo=false — /api/leads/* will return 503. Check AMO_* in $ENVFILE." + # Live and healthy: from here on it is a release like any other. + PENDING_REL="" + PENDING_LINK="" + # Prune old releases, never the live one. CURRENT="$(readlink -f "$BASE/current")" ls -1dt "$BASE"/releases/*/ 2>/dev/null | tail -n "+$((KEEP+1))" | while read -r old; do