test(task): live failures self-report evidence; wrong-exit no longer masks (#15)
- live hello failure dumps latest run result.json + stderr tail before sandbox cleanup destroys them - wrong-expectExact case asserts reason == expect-mismatch (was: any exit 1, which masked compose-level failures) - repair dangling if/else from the docker-guard refactor Closes #15
This commit is contained in:
+36
-5
@@ -93,10 +93,34 @@ M2=$(stat -c %Y "$SANDBOX/ok.json")
|
||||
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
|
||||
|
||||
# ---------- live: real runs (Docker + credentials required) ----------
|
||||
if docker info >/dev/null 2>&1; then
|
||||
expect_exit "live hello task succeeds with exact marker" 0 -- \
|
||||
scripts/run-task.sh run "$SANDBOX/ok.json"
|
||||
# On failure, surface the run record + agent stderr BEFORE the sandbox
|
||||
# cleanup destroys them. Never let a wrong-exit mask the real reason.
|
||||
dump_latest_run() {
|
||||
local latest
|
||||
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
||||
if [ -n "$latest" ]; then
|
||||
echo "--- latest run evidence: $latest ---" >&2
|
||||
cat "$latest/result.json" 2>/dev/null >&2
|
||||
echo "--- stderr.txt (tail) ---" >&2
|
||||
tail -8 "$latest/stderr.txt" 2>/dev/null >&2
|
||||
else
|
||||
echo "--- no run dir was created at all ---" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
latest_reason() {
|
||||
local latest
|
||||
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
||||
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
|
||||
}
|
||||
|
||||
if docker info >/dev/null 2>&1; then
|
||||
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
|
||||
PASS=$((PASS+1)); echo "ok live hello task succeeds with exact marker"
|
||||
else
|
||||
FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
|
||||
dump_latest_run
|
||||
fi
|
||||
[ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
|
||||
R1="$(ls "$DATA_ROOT/runs" | head -1)"
|
||||
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
|
||||
@@ -107,8 +131,15 @@ process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.e
|
||||
check "result.json contents are correct" $?
|
||||
|
||||
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
|
||||
expect_exit "wrong expectExact fails with exit 1" 1 -- \
|
||||
scripts/run-task.sh run "$SANDBOX/wrong.json"
|
||||
scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
|
||||
RC=$?
|
||||
WRONG_REASON="$(latest_reason)"
|
||||
if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
|
||||
PASS=$((PASS+1)); echo "ok wrong expectExact fails with exit 1 (reason: expect-mismatch)"
|
||||
else
|
||||
FAIL=$((FAIL+1)); echo "FAIL wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
|
||||
dump_latest_run
|
||||
fi
|
||||
|
||||
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
|
||||
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
|
||||
|
||||
Reference in New Issue
Block a user