test(ci): bound queue harness liveness

Refs #1019

packages/mosaic/framework/tools/quality/scripts/test-upgrade-rollback.sh

@@ -39,12 +39,11 @@ ORIG_PATH="$PATH"
 # loop — which would make the control a false negative. A root dotfile is
 # operator-owned (unknown→operator), so the sync loop skips it. Clean up on exit.
 STRIPPED="$FW/.install-rollback-control.tmp.sh"
-SIGNALED="$FW/.install-signal-control.tmp.sh"
 NOEXIT="$FW/.install-noexit-control.tmp.sh"
 D1CTRL="$FW/.install-d1guard-control.tmp.sh"
 D2CTRL="$FW/.install-d2guard-control.tmp.sh"
-rm -f "$STRIPPED" "$SIGNALED" "$NOEXIT" "$D1CTRL" "$D2CTRL"
+rm -f "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL"
-trap 'rm -f "$STRIPPED" "$SIGNALED" "$NOEXIT" "$D1CTRL" "$D2CTRL"' EXIT
+trap 'rm -f "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL"' EXIT
 
 pass=0; fail=0
 chk() { if eval "$2"; then echo "  ✓ $1"; pass=$((pass + 1)); else echo "  ✗ $1"; fail=$((fail + 1)); fi; }
@@ -181,86 +180,41 @@ chk "[control] without -E the mid-sync corruption survives (no rollback)" \
 # ── Part C: an INT/TERM interrupt must terminate, not resume (blocker-A) ──────
 # A bash signal trap that merely returns lets the script continue past the
 # interrupt — restoring the snapshot, then resuming the sync and reporting
-# success. The earlier test used a child cp shim to signal its parent, making
+# success. We inject a SIGTERM mid-sync with a cp that SUCCEEDS (so set -e never
-# child completion race Bash's interrupted wait. Concurrency is not part of the
+# fires and ONLY the signal path governs), and assert the shipped installer
-# guarded property: sync_framework_keep() runs in the installer's own Bash
+# restores AND exits without reporting success. The control strips `exit 1` from
-# process, and `kill` is a builtin. Generate two installer fixtures that signal
+# the trap and shows the buggy resume-to-success.
-# themselves at the same known mid-sync point. Their TERM handlers emit the same
+make_term_shim() {
-# observable before diverging, so missing signal delivery fails BOTH arms rather
+  local dir="$1"
-# than manufacturing a pass. The only semantic difference between fixtures is
+  cat > "$dir/cp" <<SHIM
-# the explicit `exit 1` whose load-bearing behavior this control proves.
+#!/usr/bin/env bash
-TERM_MARKER='[test-control] TERM handler entered'
+dest="\${@: -1}"
-HANDLER_WITH_EXIT="trap 'echo \"$TERM_MARKER\" >&2; restore_snapshot; exit 1' TERM  # TEST-TERM-HANDLER"
+case "\$dest" in
-HANDLER_WITHOUT_EXIT="trap 'echo \"$TERM_MARKER\" >&2; restore_snapshot' TERM  # TEST-TERM-HANDLER"
+  */$POISON_REL)
-
+    kill -TERM "\$PPID" 2>/dev/null   # signal install.sh; the copy still succeeds
-make_signal_installer() {
+    exec env PATH="$ORIG_PATH" cp "\$@" ;;
-  local output="$1" handler="$2"
+esac
-  local target_trap="trap 'restore_snapshot; exit 1' ERR INT TERM"
+exec env PATH="$ORIG_PATH" cp "\$@"
-  local target_cp='    cp "$abs" "$dst/$rel"'
+SHIM
-  local inject_open="    if [[ \"\$rel\" == \"$POISON_REL\" ]]; then"
+  chmod +x "$dir/cp"
-  local inject_kill='      kill -TERM "$$"  # TEST-TERM-INJECTION'
-  local inject_close='    fi'
-
-  if ! awk \
-    -v target_trap="$target_trap" -v target_cp="$target_cp" \
-    -v handler="$handler" -v inject_open="$inject_open" \
-    -v inject_kill="$inject_kill" -v inject_close="$inject_close" '
-      $0 == target_cp {
-        print inject_open
-        print inject_kill
-        print inject_close
-        injection_sites++
-      }
-      { print }
-      $0 == target_trap {
-        print handler
-        handler_sites++
-      }
-      END {
-        if (handler_sites != 1 || injection_sites != 1) exit 42
-      }
-    ' "$INSTALL" > "$output"; then
-    rm -f "$output"
-    fail "Could not construct the self-TERM control installer at the exact trap/copy sites"
-    exit 1
-  fi
-  chmod +x "$output"
 }
 
-make_signal_installer "$SIGNALED" "$HANDLER_WITH_EXIT"
+# Run one keep-mode upgrade with the SIGTERM shim. Echoes "<exit>\t<out>\t<home>".
-make_signal_installer "$NOEXIT" "$HANDLER_WITHOUT_EXIT"
-signal_fixture_ready() {
-  local fixture="$1" expected_handler="$2"
-  [[ "$(grep -cF '# TEST-TERM-INJECTION' "$fixture")" -eq 1 ]] \
-    && [[ "$(grep -cF '# TEST-TERM-HANDLER' "$fixture")" -eq 1 ]] \
-    && grep -Fqx "$expected_handler" "$fixture"
-}
-signaled_fixture_ready() { signal_fixture_ready "$SIGNALED" "$HANDLER_WITH_EXIT"; }
-noexit_fixture_ready() { signal_fixture_ready "$NOEXIT" "$HANDLER_WITHOUT_EXIT"; }
-chk "[signal] shipped fixture has exactly one self-TERM injection and marked handler" \
-  "signaled_fixture_ready"
-chk "[control] no-exit fixture has exactly one self-TERM injection and marked handler" \
-  "noexit_fixture_ready"
-chk "[control] removing the explicit TERM exit changes the fixture" \
-  "! cmp -s '$SIGNALED' '$NOEXIT'"
-
-# Run one keep-mode upgrade whose own shell delivers SIGTERM synchronously at
-# the selected copy. Echoes "<exit>\t<out>\t<home>".
 run_signal_upgrade() {
-  local installer="$1" H OUT rc
+  local installer="$1" H OUT SHIM rc
-  H=$(mktemp -d); OUT=$(mktemp)
+  H=$(mktemp -d); OUT=$(mktemp); SHIM=$(mktemp -d)
   seed_home "$H"
+  make_term_shim "$SHIM"
   set +e
-  PATH="$ORIG_PATH" \
+  PATH="$SHIM:$ORIG_PATH" \
     MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 bash "$installer" >"$OUT" 2>&1
   rc=$?
   set -e 2>/dev/null || true
+  rm -rf "$SHIM"
   printf '%s\t%s\t%s\n' "$rc" "$OUT" "$H"
 }
 
-IFS=$'\t' read -r rcC OUTC HC < <(run_signal_upgrade "$SIGNALED")
+IFS=$'\t' read -r rcC OUTC HC < <(run_signal_upgrade "$INSTALL")
-chk "[signal] TERM handler observable fires exactly once" \
-  "[ \"\$(grep -cF '$TERM_MARKER' '$OUTC')\" -eq 1 ]"
 chk "[signal] SIGTERM mid-sync aborts non-zero (trap exits, does not resume)" \
   "[ '$rcC' -ne 0 ]"
 chk "[signal] restore_snapshot fires on the interrupt" \
@@ -268,13 +222,13 @@ chk "[signal] restore_snapshot fires on the interrupt" \
 chk "[signal] does NOT resume to report sync success after the interrupt" \
   "! grep -q 'file phase complete' '$OUTC'"
 
-IFS=$'\t' read -r rcD OUTD HD < <(run_signal_upgrade "$NOEXIT")
+# Control: strip `exit 1` from the signal trap → the handler returns, the script
-chk "[control] TERM handler observable fires exactly once" \
+# resumes past the interrupt and wrongly reports success. In $FW so SOURCE_DIR resolves.
-  "[ \"\$(grep -cF '$TERM_MARKER' '$OUTD')\" -eq 1 ]"
+sed "s/trap 'restore_snapshot; exit 1' ERR INT TERM/trap 'restore_snapshot' ERR INT TERM/" \
-chk "[control] without 'exit 1' the handler restores before returning" \
+  "$INSTALL" > "$NOEXIT"
-  "grep -q 'restoring previous state from snapshot' '$OUTD'"
+chk "[control] the exit-strip actually changed the installer" \
-chk "[control] without 'exit 1' the installer exits zero after resuming" \
+  "! cmp -s '$INSTALL' '$NOEXIT'"
-  "[ '$rcD' -eq 0 ]"
+IFS=$'\t' read -r _rcD OUTD HD < <(run_signal_upgrade "$NOEXIT")
 chk "[control] without 'exit 1' the trap resumes and reports sync success (the bug)" \
   "grep -q 'file phase complete' '$OUTD'"
 
@@ -355,10 +309,10 @@ chk "[control] without the D2 recovery line the operator gets no snapshot pointe
 # Reap any snapshot the reset-fail runs left in /tmp (reset failed → never cleaned).
 grep -o '/[^ ]*mosaic-snapshot[^ ]*' "$OUTH" 2>/dev/null | head -1 | while read -r s; do rm -rf "$s"; done
 
-# Cleanup (generated installer controls are also removed by the EXIT trap).
+# Cleanup ($STRIPPED / $NOEXIT / $D1CTRL / $D2CTRL are also removed by the EXIT trap).
 for d in "$HA" "$REFA" "$HB" "$REFB" "$HC" "$HD" "$HE" "$REFE" "$HF" "$REFF" "$HG" "$HH"; do rm -rf "$d"; done
 rm -f "$OUTA" "$OUTB" "$OUTC" "$OUTD" "$OUTE" "$OUTF" "$OUTG" "$OUTH" \
-  "$STRIPPED" "$SIGNALED" "$NOEXIT" "$D1CTRL" "$D2CTRL"
+  "$STRIPPED" "$NOEXIT" "$D1CTRL" "$D2CTRL"
 
 echo
 echo "RESULT: $pass passed, $fail failed"