test(task): mission/task selftests — schema negatives + live runs (#8)

18 cases: validation negatives (unknown keys, versions, ids, prompt,
expectExact NUL, timeout range, missing/invalid mission, writes-nothing)
plus live cases: exact-marker success, wrong expectExact fails, distinct
run dirs, result.json contents, list output.

Closes #8
This commit is contained in:
2026-09-02 19:57:34 -05:00
parent d2a9e26395
commit cfd2a19bd7
3 changed files with 142 additions and 0 deletions
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
# Sandboxed selftests for the mission/task layer.
#
# Fast cases are validation-only (no Docker, no network). The final cases
# execute real tasks through the container path and therefore require a
# working configuration, credentials, and Docker.
set -uo pipefail
cd "$(dirname "$0")/.."
SANDBOX="$(mktemp -d)"
trap 'rm -rf "$SANDBOX"' EXIT
PASS=0
FAIL=0
expect_exit() {
local name="$1" expected="$2"
shift 3
local rc
"$@" >/dev/null 2>&1
rc=$?
if [ "$rc" -eq "$expected" ]; then
PASS=$((PASS+1)); echo "ok $name (exit $rc)"
else
FAIL=$((FAIL+1)); echo "FAIL $name (exit $rc, expected $expected)"
fi
}
check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
}
CONFIG="$SANDBOX/config.json"
DATA_ROOT="$SANDBOX/data"
mkdir -p "$DATA_ROOT"
cat > "$CONFIG" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$DATA_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash"}}
EOF
export MOSAIC_CONFIG="$CONFIG"
TASK="node scripts/mosaic-task.mjs"
good_task() { # args: file [overrides as jq-less JSON fragments]
cat > "$1" <<EOF
{"taskVersion":1,"id":"t-ok","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_HELLO_OK"}
EOF
}
good_mission() {
cat > "$1" <<'EOF'
{"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]}
EOF
}
# ---------- fast: schema negatives ----------
good_task "$SANDBOX/ok.json"
expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json"
expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json"
printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json"
expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json"
printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json"
expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json"
printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json"
expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json"
expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json"
expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json"
expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json"
good_mission "$SANDBOX/m.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json"
expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json"
printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json"
expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json"
expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json"
# validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json
M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1
$TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
M2=$(stat -c %Y "$SANDBOX/ok.json")
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
# ---------- live: real runs (Docker + credentials required) ----------
if docker info >/dev/null 2>&1; then
expect_exit "live hello task succeeds with exact marker" 0 -- \
scripts/run-task.sh run "$SANDBOX/ok.json"
[ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
R1="$(ls "$DATA_ROOT/runs" | head -1)"
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
node -e '
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1);
' "$DATA_ROOT/runs/$R1/result.json"
check "result.json contents are correct" $?
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
expect_exit "wrong expectExact fails with exit 1" 1 -- \
scripts/run-task.sh run "$SANDBOX/wrong.json"
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
|| check "each run gets a distinct run dir (no clobber)" 1
COUNT=$($TASK list | wc -l)
[ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1
else
echo "skip live task cases (docker unavailable)"
fi
echo
echo "selftest: $PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]