#!/usr/bin/env bash # test-upgrade-rollback.sh — the #791 B1 regression gate. # # A keep-mode upgrade takes a pre-update snapshot and installs an ERR/INT/TERM # trap that restores it if the sync aborts midway (install.sh: make_snapshot + # `trap restore_snapshot`). That trap is only reached if `set -E` (errtrace) is # active — otherwise a failure INSIDE sync_framework_keep() (which runs entirely # in a function) never fires the trap, and the upgrade aborts leaving a # half-written target with NO rollback. This test proves: # # Part A (the gate): the shipped installer rolls back a mid-sync failure — # the restore message fires, the corrupted file is put # back, AND the whole target is byte-identical to its # pre-upgrade state. # Part B (the control): the SAME installer with `-E` stripped does NOT roll back # (dead trap) — the mid-sync corruption survives, proving # errtrace is load-bearing. If anyone removes `set -E`, # Part A goes red. # # The mid-sync failure is injected with a PATH-shadowing `cp` shim rather than # file permissions. The earlier 0400/EACCES approach was NOT portable: Woodpecker # runs steps as root (node:24-alpine has no USER directive), and root overwrites a # 0400 file, so the failure never fired and this gate silently passed (#791 # blocker-3). The shim fails deterministically for one framework-owned # destination regardless of uid, and — like a real interrupted cp (disk-full # mid-write) — leaves a partially-written target behind, so rollback has real # damage to undo and the control has real damage to expose. # # Usage: bash test-upgrade-rollback.sh set -uo pipefail FW="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" # packages/mosaic/framework INSTALL="$FW/install.sh" ORIG_PATH="$PATH" # The `-E`-stripped control installer must live INSIDE $FW: install.sh derives # SOURCE_DIR from its own path and `source`s $SOURCE_DIR/tools/_lib/manifest.sh, # so a copy anywhere else aborts at the source line before ever reaching the sync # loop — which would make the control a false negative. A root dotfile is # operator-owned (unknown→operator), so the sync loop skips it. Clean up on exit. STRIPPED="$FW/.install-rollback-control.tmp.sh" NOEXIT="$FW/.install-noexit-control.tmp.sh" D1CTRL="$FW/.install-d1guard-control.tmp.sh" D2CTRL="$FW/.install-d2guard-control.tmp.sh" rm -f "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL" trap 'rm -f "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL"' EXIT pass=0; fail=0 chk() { if eval "$2"; then echo " ✓ $1"; pass=$((pass + 1)); else echo " ✗ $1"; fail=$((fail + 1)); fi; } SECRET='SUPER-SECRET-TOKEN-do-not-log-b1' # A framework-owned file the shim fails the copy of. The seeded target holds GOOD # bytes; source ships different bytes, so sync_framework_keep() attempts the copy # and the shim intercepts it. Root-level framework files sort before guides/, so # several framework files are already refreshed when the copy reaches this one. POISON_REL='guides/E2E-DELIVERY.md' GOOD='GOOD-REFERENCE-CONTENT-pre-upgrade-b1' GARBAGE='PARTIAL-WRITE-GARBAGE-mid-sync-b1' # A `cp` shim: for the poisoned destination, simulate an interrupted copy — write # partial garbage to the target, then fail — otherwise delegate to the real cp # (resolved via the ORIGINAL PATH so make_snapshot/restore still work). make_cp_shim() { local dir="$1" cat > "$dir/cp" < "\$dest" 2>/dev/null || true exit 1 ;; esac exec env PATH="$ORIG_PATH" cp "\$@" SHIM chmod +x "$dir/cp" } # A `find` shim that fails every enumeration scan (`-print0`) as if it hit an # EACCES/I/O error partway — it emits the real (here: complete) list first, then # exits non-zero, exactly the class of failure a `< <(find …)` process # substitution silently swallows. All non-`-print0` finds (e.g. the -delete # sweep) delegate to the real find on the original PATH. Used to prove #791 # blocker-D1: the shipped installer must honor find's exit status and roll back. make_find_fail_shim() { local dir="$1" cat > "$dir/find" <` (the restore's target # reset) and delegates every other rm to the real one. Used to prove #791 # blocker-D2: when the target reset inside restore_snapshot fails, the installer # must emit the manual-recovery pointer (snapshot path) instead of exiting # silently under `set -e`. FAIL_RM_TARGET is exported into the installer env. make_rm_fail_shim() { local dir="$1" cat > "$dir/rm" <<'SHIM' #!/usr/bin/env bash last="${@: -1}" if [ -n "${FAIL_RM_TARGET:-}" ] && [ "$last" = "$FAIL_RM_TARGET" ]; then exit 1 fi exec env PATH="$ORIG_PATH_FOR_RM" rm "$@" SHIM chmod +x "$dir/rm" } seed_home() { local H="$1" mkdir -p "$H/agents" "$H/tools/_lib" "$H/memory" "$H/guides" printf '# persona\n' > "$H/SOUL.md" # recognized install → keep mode + snapshot printf 'MODEL=opus\n' > "$H/agents/coder0.conf" printf '# operator memory\n' > "$H/memory/note.md" printf 'TOKEN=%s\n' "$SECRET" > "$H/tools/_lib/credentials.json" echo 3 > "$H/.framework-version" # Good pre-upgrade bytes; source ships different bytes, so cp is attempted. printf '%s' "$GOOD" > "$H/$POISON_REL" } # Run one keep-mode upgrade against $1=installer, seeding a fresh home and a # byte-for-byte reference of the pre-upgrade state, with the cp shim first on # PATH. Echoes: "\t\t\t". run_upgrade() { local installer="$1" shim_maker="${2:-make_cp_shim}" H REF OUT SHIM rc H=$(mktemp -d); REF=$(mktemp -d); OUT=$(mktemp); SHIM=$(mktemp -d) seed_home "$H" env PATH="$ORIG_PATH" cp -a "$H/." "$REF/" # pre-upgrade reference (real cp) "$shim_maker" "$SHIM" set +e PATH="$SHIM:$ORIG_PATH" \ MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 bash "$installer" >"$OUT" 2>&1 rc=$? set -e 2>/dev/null || true rm -rf "$SHIM" printf '%s\t%s\t%s\t%s\n' "$rc" "$OUT" "$REF" "$H" } echo "#791 upgrade rollback (B1 regression gate):" # ── Part A: the shipped installer must roll back a mid-sync failure ─────────── IFS=$'\t' read -r rcA OUTA REFA HA < <(run_upgrade "$INSTALL") chk "[shipped] upgrade aborts non-zero on the injected mid-sync failure" \ "[ '$rcA' -ne 0 ]" chk "[shipped] restore_snapshot fires (rollback message present)" \ "grep -q 'restoring previous state from snapshot' '$OUTA'" chk "[shipped] the corrupted file is restored to its pre-upgrade bytes" \ "[ \"\$(cat '$HA/$POISON_REL')\" = '$GOOD' ]" chk "[shipped] target rolled back byte-identical to pre-upgrade state" \ "diff -r '$REFA' '$HA' >/dev/null 2>&1" chk "[shipped] operator secret value absent from installer output" \ "! grep -q '$SECRET' '$OUTA'" # ── Part B: control — strip `-E`, the trap is dead, no rollback happens ─────── # Proves errtrace is what makes the trap reachable. If `set -E` is ever removed # from install.sh, Part A's rollback assertions fail exactly like this control. sed 's/^set -Eeuo pipefail/set -euo pipefail/' "$INSTALL" > "$STRIPPED" chk "[control] the -E strip actually changed the installer" \ "! cmp -s '$INSTALL' '$STRIPPED'" IFS=$'\t' read -r rcB OUTB REFB HB < <(run_upgrade "$STRIPPED") chk "[control] without -E the upgrade still aborts non-zero" \ "[ '$rcB' -ne 0 ]" # The load-bearing, deterministic proof of B1: without errtrace the ERR trap # never fires for a failure inside sync_framework_keep(), so no rollback runs. chk "[control] without -E the rollback message does NOT fire (dead trap)" \ "! grep -q 'restoring previous state from snapshot' '$OUTB'" chk "[control] without -E the mid-sync corruption survives (no rollback)" \ "[ \"\$(cat '$HB/$POISON_REL')\" = '$GARBAGE' ]" # ── Part C: an INT/TERM interrupt must terminate, not resume (blocker-A) ────── # A bash signal trap that merely returns lets the script continue past the # interrupt — restoring the snapshot, then resuming the sync and reporting # success. We inject a SIGTERM mid-sync with a cp that SUCCEEDS (so set -e never # fires and ONLY the signal path governs), and assert the shipped installer # restores AND exits without reporting success. The control strips `exit 1` from # the trap and shows the buggy resume-to-success. make_term_shim() { local dir="$1" cat > "$dir/cp" </dev/null # signal install.sh; the copy still succeeds exec env PATH="$ORIG_PATH" cp "\$@" ;; esac exec env PATH="$ORIG_PATH" cp "\$@" SHIM chmod +x "$dir/cp" } # Run one keep-mode upgrade with the SIGTERM shim. Echoes "\t\t". run_signal_upgrade() { local installer="$1" H OUT SHIM rc H=$(mktemp -d); OUT=$(mktemp); SHIM=$(mktemp -d) seed_home "$H" make_term_shim "$SHIM" set +e PATH="$SHIM:$ORIG_PATH" \ MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 bash "$installer" >"$OUT" 2>&1 rc=$? set -e 2>/dev/null || true rm -rf "$SHIM" printf '%s\t%s\t%s\n' "$rc" "$OUT" "$H" } IFS=$'\t' read -r rcC OUTC HC < <(run_signal_upgrade "$INSTALL") chk "[signal] SIGTERM mid-sync aborts non-zero (trap exits, does not resume)" \ "[ '$rcC' -ne 0 ]" chk "[signal] restore_snapshot fires on the interrupt" \ "grep -q 'restoring previous state from snapshot' '$OUTC'" chk "[signal] does NOT resume to report sync success after the interrupt" \ "! grep -q 'file phase complete' '$OUTC'" # Control: strip `exit 1` from the signal trap → the handler returns, the script # resumes past the interrupt and wrongly reports success. In $FW so SOURCE_DIR resolves. sed "s/trap 'restore_snapshot; exit 1' ERR INT TERM/trap 'restore_snapshot' ERR INT TERM/" \ "$INSTALL" > "$NOEXIT" chk "[control] the exit-strip actually changed the installer" \ "! cmp -s '$INSTALL' '$NOEXIT'" IFS=$'\t' read -r _rcD OUTD HD < <(run_signal_upgrade "$NOEXIT") chk "[control] without 'exit 1' the trap resumes and reports sync success (the bug)" \ "grep -q 'file phase complete' '$OUTD'" # ── Part D: a failed source/prune `find` scan must abort + roll back (D1) ───── # A `< <(find …)` process substitution discards find's exit status, so an # EACCES/I/O failure mid-scan would truncate the file list yet leave the reading # loop exiting 0 — a partial upgrade committed and reported as success, with the # ERR/restore trap never firing. The shipped installer captures the scan into a # checked temp file (_scan_or_die) and aborts on failure. We inject a `find` that # fails every `-print0` scan and assert the shipped installer rolls back. IFS=$'\t' read -r rcE OUTE REFE HE < <(run_upgrade "$INSTALL" make_find_fail_shim) chk "[find-fail] a failing framework scan aborts the upgrade non-zero" \ "[ '$rcE' -ne 0 ]" chk "[find-fail] restore_snapshot fires on the aborted scan" \ "grep -q 'restoring previous state from snapshot' '$OUTE'" chk "[find-fail] the abort is a fail-closed enumeration error (not a silent truncation)" \ "grep -q 'Could not enumerate framework files' '$OUTE'" chk "[find-fail] target rolled back byte-identical to pre-upgrade state" \ "diff -r '$REFE' '$HE' >/dev/null 2>&1" # Control: neuter the D1 guard (turn its `return 1` into a no-op) so a find # failure is swallowed exactly as `< <(find …)` would — the scan appears to # succeed and the upgrade reports completion with NO rollback. sed 's/return 1 # D1-GUARD/: # D1-GUARD-DISABLED/' "$INSTALL" > "$D1CTRL" chk "[control] the D1-guard strip actually changed the installer" \ "! cmp -s '$INSTALL' '$D1CTRL'" IFS=$'\t' read -r _rcF OUTF REFF HF < <(run_upgrade "$D1CTRL" make_find_fail_shim) chk "[control] with the D1 guard disabled the find failure is swallowed (no rollback)" \ "! grep -q 'restoring previous state from snapshot' '$OUTF'" chk "[control] with the D1 guard disabled the upgrade wrongly reports success" \ "grep -q 'file phase complete' '$OUTF'" # ── Part E: a failed target reset inside restore must not exit silently (D2) ── # restore_snapshot resets the target (`rm -rf; mkdir -p`) before rebuilding from # the snapshot. Under `set -e` (trap disarmed) a bare reset that fails would exit # the whole script immediately — after `rm` may have deleted part of the target — # WITHOUT printing where the snapshot lives. We trigger a rollback (cp poison) AND # fail the target reset (rm shim), then assert the shipped installer emits the # manual-recovery pointer and preserves the snapshot. run_rmfail_upgrade() { local installer="$1" H OUT SHIM rc H=$(mktemp -d); OUT=$(mktemp); SHIM=$(mktemp -d) seed_home "$H" make_cp_shim "$SHIM" # poison cp → triggers the abort + restore make_rm_fail_shim "$SHIM" # rm -rf fails → exercises the D2 reset guard set +e PATH="$SHIM:$ORIG_PATH" ORIG_PATH_FOR_RM="$ORIG_PATH" FAIL_RM_TARGET="$H" \ MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 bash "$installer" >"$OUT" 2>&1 rc=$? set -e 2>/dev/null || true rm -rf "$SHIM" printf '%s\t%s\t%s\n' "$rc" "$OUT" "$H" } IFS=$'\t' read -r rcG OUTG HG < <(run_rmfail_upgrade "$INSTALL") chk "[reset-fail] a failed target reset still aborts non-zero" \ "[ '$rcG' -ne 0 ]" chk "[reset-fail] the manual-recovery pointer is emitted (not a silent set -e exit)" \ "grep -q 'Snapshot restore could not reset' '$OUTG'" chk "[reset-fail] the recovery message points at a preserved snapshot dir" \ "grep -q 'preserved at: .*mosaic-snapshot' '$OUTG'" SNAP_E="$(grep -o '/[^ ]*mosaic-snapshot[^ ]*' "$OUTG" | head -1)" chk "[reset-fail] the named snapshot directory actually survives for recovery" \ "[ -n '$SNAP_E' ] && [ -d '$SNAP_E' ]" chk "[reset-fail] operator secret value never appears in installer output" \ "! grep -q '$SECRET' '$OUTG'" # Control: delete the D2 recovery line so a failed reset returns non-zero with NO # operator pointer — the observable defect (half-reset target, snapshot orphaned # in /tmp with no path told to the operator). Proves the message is load-bearing. sed '/Snapshot restore could not reset/d' "$INSTALL" > "$D2CTRL" chk "[control] the D2-recovery strip actually changed the installer" \ "! cmp -s '$INSTALL' '$D2CTRL'" IFS=$'\t' read -r _rcH OUTH HH < <(run_rmfail_upgrade "$D2CTRL") chk "[control] without the D2 recovery line the operator gets no snapshot pointer" \ "! grep -q 'Snapshot restore could not reset' '$OUTH'" [ -n "${SNAP_E:-}" ] && rm -rf "$SNAP_E" # Reap any snapshot the reset-fail runs left in /tmp (reset failed → never cleaned). grep -o '/[^ ]*mosaic-snapshot[^ ]*' "$OUTH" 2>/dev/null | head -1 | while read -r s; do rm -rf "$s"; done # Cleanup ($STRIPPED / $NOEXIT / $D1CTRL / $D2CTRL are also removed by the EXIT trap). for d in "$HA" "$REFA" "$HB" "$REFB" "$HC" "$HD" "$HE" "$REFE" "$HF" "$REFF" "$HG" "$HH"; do rm -rf "$d"; done rm -f "$OUTA" "$OUTB" "$OUTC" "$OUTD" "$OUTE" "$OUTF" "$OUTG" "$OUTH" \ "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL" echo echo "RESULT: $pass passed, $fail failed" [ "$fail" -eq 0 ]