test(task): live failures self-report evidence; wrong-exit no longer masks (#15)

- live hello failure dumps latest run result.json + stderr tail before
  sandbox cleanup destroys them
- wrong-expectExact case asserts reason == expect-mismatch (was: any
  exit 1, which masked compose-level failures)
- repair dangling if/else from the docker-guard refactor

Closes #15
This commit is contained in:
2026-09-02 20:56:58 -05:00
parent e7e1bd26eb
commit 88f55d9135
+36 -5
View File
@@ -93,10 +93,34 @@ M2=$(stat -c %Y "$SANDBOX/ok.json")
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
# ---------- live: real runs (Docker + credentials required) ----------
if docker info >/dev/null 2>&1; then
expect_exit "live hello task succeeds with exact marker" 0 -- \
scripts/run-task.sh run "$SANDBOX/ok.json"
# On failure, surface the run record + agent stderr BEFORE the sandbox
# cleanup destroys them. Never let a wrong-exit mask the real reason.
dump_latest_run() {
local latest
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
if [ -n "$latest" ]; then
echo "--- latest run evidence: $latest ---" >&2
cat "$latest/result.json" 2>/dev/null >&2
echo "--- stderr.txt (tail) ---" >&2
tail -8 "$latest/stderr.txt" 2>/dev/null >&2
else
echo "--- no run dir was created at all ---" >&2
fi
}
latest_reason() {
local latest
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
}
if docker info >/dev/null 2>&1; then
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
PASS=$((PASS+1)); echo "ok live hello task succeeds with exact marker"
else
FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
dump_latest_run
fi
[ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
R1="$(ls "$DATA_ROOT/runs" | head -1)"
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
@@ -107,8 +131,15 @@ process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.e
check "result.json contents are correct" $?
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
expect_exit "wrong expectExact fails with exit 1" 1 -- \
scripts/run-task.sh run "$SANDBOX/wrong.json"
scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
RC=$?
WRONG_REASON="$(latest_reason)"
if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
PASS=$((PASS+1)); echo "ok wrong expectExact fails with exit 1 (reason: expect-mismatch)"
else
FAIL=$((FAIL+1)); echo "FAIL wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
dump_latest_run
fi
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \