test(ci): bound queue harness liveness

Refs #1019

packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh

@@ -11,10 +11,49 @@ STUB_DIR="$WORK_DIR/stubs"
 AUDIT_LOG="$WORK_DIR/audit/ci-queue-wait.jsonl"
 STATUS_OBSERVED="$WORK_DIR/status-observed"
 CLOCK_LOG="$WORK_DIR/clock.log"
+WATCHDOG_PYTHON="/usr/bin/python3"
+WATCHDOG_SCRIPT="$WORK_DIR/real-clock-watchdog.py"
+WATCHDOG_TIMEOUT_SEC=5
+WATCHDOG_EXIT=90
 FEATURE_BRANCH="fix/rm-03-fixture"
 
+if [[ ! -x "$WATCHDOG_PYTHON" ]]; then
+    echo "FAIL setup: required real-clock watchdog runtime is unavailable at $WATCHDOG_PYTHON" >&2
+    exit 1
+fi
+
 rm -rf "$WORK_DIR"
 mkdir -p "$REPO_DIR" "$STUB_DIR"
+cat > "$WATCHDOG_SCRIPT" <<'PY'
+import os
+import signal
+import subprocess
+import sys
+
+if len(sys.argv) < 3:
+    raise SystemExit(2)
+
+timeout_seconds = float(sys.argv[1])
+process = subprocess.Popen(sys.argv[2:], start_new_session=True)
+try:
+    return_code = process.wait(timeout=timeout_seconds)
+except subprocess.TimeoutExpired:
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    process.wait()
+    print(
+        f"FAIL HANG watchdog: subject exceeded {timeout_seconds:g}s "
+        "before completing its intended path",
+        file=sys.stderr,
+    )
+    raise SystemExit(90)
+
+if return_code < 0:
+    raise SystemExit(128 - return_code)
+raise SystemExit(return_code)
+PY
 git -C "$REPO_DIR" init -q
 git -C "$REPO_DIR" checkout -q -b "$FEATURE_BRANCH"
 git -C "$REPO_DIR" remote add origin https://git.example.test/acme/widgets.git
@@ -35,6 +74,9 @@ printf '%s\n' "$url" >> "${MOSAIC_STUB_URL_LOG:?}"
 
 case "$url" in
     */branches/*)
+        if [[ "${MOSAIC_STUB_BRANCH_MODE:-ok}" == "hang-before-provider" ]]; then
+            while :; do :; done
+        fi
         if [[ "${MOSAIC_STUB_BRANCH_MODE:-ok}" == "unreachable" ]]; then
             exit 7
         fi
@@ -78,7 +120,7 @@ fi
 
 if [[ -e "${MOSAIC_STUB_STATUS_OBSERVED:?}" ]]; then
     printf 'date-phase=after-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}"
-    printf '2000\n'
+    printf '1002\n'
 else
     printf 'date-phase=before-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}"
     printf '1000\n'
@@ -117,7 +159,17 @@ run_guard() {
         export MOSAIC_CI_QUEUE_AUDIT_LOG="$audit_log"
         # Provider observation is the synchronization event. The one-second
         # timeout is subject semantics under virtual time, never a wall wait.
-        "$SCRIPT_DIR/ci-queue-wait.sh" --purpose "${MOSAIC_TEST_PURPOSE:-push}" -t 1 -i 1 "$@"
+        # The absolute Python runtime uses an internal monotonic wait and kills
+        # the subject's isolated process group. Neither operation can resolve
+        # to the virtual date/sleep stubs at the front of PATH.
+        local subject_rc
+        if "$WATCHDOG_PYTHON" "$WATCHDOG_SCRIPT" "$WATCHDOG_TIMEOUT_SEC" \
+            "$SCRIPT_DIR/ci-queue-wait.sh" --purpose "${MOSAIC_TEST_PURPOSE:-push}" -t 1 -i 1 "$@"; then
+            subject_rc=0
+        else
+            subject_rc=$?
+        fi
+        return "$subject_rc"
     )
 }
 
@@ -197,6 +249,27 @@ run_assertion large-payload not126 large-success 'state=terminal-success'
 run_assertion credential-unresolvable zero credential-unresolvable 'CANNOT_ASSERT'
 run_assertion provider-unreachable zero unreachable 'CANNOT_ASSERT'
 
+# Positive liveness control: a subject mutant hangs before the branch lookup
+# can reach the status provider. Only the independent real-clock watchdog may
+# terminate it, and its failure must be distinct from subject timeout rc=124.
+set +e
+watchdog_output=$(MOSAIC_STUB_BRANCH_MODE=hang-before-provider run_guard success "$AUDIT_LOG" 2>&1)
+watchdog_rc=$?
+set -e
+if [[ "$watchdog_rc" -ne "$WATCHDOG_EXIT" ]]; then
+    echo "FAIL watchdog-control: expected hang-specific rc=$WATCHDOG_EXIT, got rc=$watchdog_rc" >&2
+    failures=$((failures + 1))
+fi
+if [[ "$watchdog_output" != *"FAIL HANG watchdog:"* ]]; then
+    echo "FAIL watchdog-control: expected distinct hang-specific diagnostic" >&2
+    printf '%s\n' "$watchdog_output" >&2
+    failures=$((failures + 1))
+fi
+if [[ -e "$STATUS_OBSERVED" ]]; then
+    echo "FAIL watchdog-control: hanging mutant unexpectedly reached the status provider" >&2
+    failures=$((failures + 1))
+fi
+
 if [[ ! -s "$AUDIT_LOG" ]] || ! grep -q '"outcome":"CANNOT_ASSERT"' "$AUDIT_LOG"; then
     echo "FAIL provider-unreachable-audit: expected durable CANNOT_ASSERT JSONL record" >&2
     failures=$((failures + 1))