From 88f55d913541cee0297f4bd6abafc9bf74cfa3ec Mon Sep 17 00:00:00 2001
From: Jason Woltje <jason@diversecanvas.com>
Date: Wed, 2 Sep 2026 20:56:58 -0500
Subject: [PATCH] test(task): live failures self-report evidence; wrong-exit no
 longer masks (#15)

- live hello failure dumps latest run result.json + stderr tail before
  sandbox cleanup destroys them
- wrong-expectExact case asserts reason == expect-mismatch (was: any
  exit 1, which masked compose-level failures)
- repair dangling if/else from the docker-guard refactor

Closes #15
---
 scripts/test-task.sh | 41 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/scripts/test-task.sh b/scripts/test-task.sh
index 685c80fc..c6e421b4 100755
--- a/scripts/test-task.sh
+++ b/scripts/test-task.sh
@@ -93,10 +93,34 @@ M2=$(stat -c %Y "$SANDBOX/ok.json")
 [ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
 
 # ---------- live: real runs (Docker + credentials required) ----------
-if docker info >/dev/null 2>&1; then
-    expect_exit "live hello task succeeds with exact marker" 0 -- \
-        scripts/run-task.sh run "$SANDBOX/ok.json"
+# On failure, surface the run record + agent stderr BEFORE the sandbox
+# cleanup destroys them. Never let a wrong-exit mask the real reason.
+dump_latest_run() {
+    local latest
+    latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
+    if [ -n "$latest" ]; then
+        echo "--- latest run evidence: $latest ---" >&2
+        cat "$latest/result.json" 2>/dev/null >&2
+        echo "--- stderr.txt (tail) ---" >&2
+        tail -8 "$latest/stderr.txt" 2>/dev/null >&2
+    else
+        echo "--- no run dir was created at all ---" >&2
+    fi
+}
 
+latest_reason() {
+    local latest
+    latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
+    [ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
+}
+
+if docker info >/dev/null 2>&1; then
+    if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
+        PASS=$((PASS+1)); echo "ok   live hello task succeeds with exact marker"
+    else
+        FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
+        dump_latest_run
+    fi
     [ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
     R1="$(ls "$DATA_ROOT/runs" | head -1)"
     [ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
@@ -107,8 +131,15 @@ process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.e
     check "result.json contents are correct" $?
 
     printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
-    expect_exit "wrong expectExact fails with exit 1" 1 -- \
-        scripts/run-task.sh run "$SANDBOX/wrong.json"
+    scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
+    RC=$?
+    WRONG_REASON="$(latest_reason)"
+    if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
+        PASS=$((PASS+1)); echo "ok   wrong expectExact fails with exit 1 (reason: expect-mismatch)"
+    else
+        FAIL=$((FAIL+1)); echo "FAIL wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
+        dump_latest_run
+    fi
 
     RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
     [ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \