test(task): mission/task selftests — schema negatives + live runs (#8)

18 cases: validation negatives (unknown keys, versions, ids, prompt, expectExact NUL, timeout range, missing/invalid mission, writes-nothing) plus live cases: exact-marker success, wrong expectExact fails, distinct run dirs, result.json contents, list output. Closes #8
2026-09-02 19:57:34 -05:00
parent d2a9e26395
commit cfd2a19bd7
3 changed files with 142 additions and 0 deletions
@@ -0,0 +1,9 @@
+{
+  "missionVersion": 1,
+  "id": "m-hello",
+  "objective": "Prove the startup marker path of the mosaic-poc-agent.",
+  "directives": [
+    "Startup verification requests are answered with the marker only.",
+    "No explanation, no formatting."
+  ]
+}
@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+# Sandboxed selftests for the mission/task layer.
+#
+# Fast cases are validation-only (no Docker, no network). The final cases
+# execute real tasks through the container path and therefore require a
+# working configuration, credentials, and Docker.
+set -uo pipefail
+cd "$(dirname "$0")/.."
+
+SANDBOX="$(mktemp -d)"
+trap 'rm -rf "$SANDBOX"' EXIT
+
+PASS=0
+FAIL=0
+
+expect_exit() {
+    local name="$1" expected="$2"
+    shift 3
+    local rc
+    "$@" >/dev/null 2>&1
+    rc=$?
+    if [ "$rc" -eq "$expected" ]; then
+        PASS=$((PASS+1)); echo "ok   $name (exit $rc)"
+    else
+        FAIL=$((FAIL+1)); echo "FAIL $name (exit $rc, expected $expected)"
+    fi
+}
+
+check() {
+    if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok   $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
+}
+
+CONFIG="$SANDBOX/config.json"
+DATA_ROOT="$SANDBOX/data"
+mkdir -p "$DATA_ROOT"
+cat > "$CONFIG" <<EOF
+{"configVersion":1,"environment":"development","dataRoot":"$DATA_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash"}}
+EOF
+export MOSAIC_CONFIG="$CONFIG"
+
+TASK="node scripts/mosaic-task.mjs"
+
+good_task() { # args: file [overrides as jq-less JSON fragments]
+    cat > "$1" <<EOF
+{"taskVersion":1,"id":"t-ok","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_HELLO_OK"}
+EOF
+}
+good_mission() {
+    cat > "$1" <<'EOF'
+{"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]}
+EOF
+}
+
+# ---------- fast: schema negatives ----------
+good_task "$SANDBOX/ok.json"
+expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json"
+
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json"
+expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json"
+
+printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json"
+expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json"
+
+printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json"
+expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json"
+
+printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json"
+expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json"
+
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json"
+expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json"
+
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json"
+expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json"
+
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json"
+expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json"
+
+good_mission "$SANDBOX/m.json"
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json"
+expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json"
+
+printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json"
+printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json"
+expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json"
+
+expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json"
+
+# validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json
+M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1
+$TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
+M2=$(stat -c %Y "$SANDBOX/ok.json")
+[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
+
+# ---------- live: real runs (Docker + credentials required) ----------
+if docker info >/dev/null 2>&1; then
+    expect_exit "live hello task succeeds with exact marker" 0 -- \
+        scripts/run-task.sh run "$SANDBOX/ok.json"
+
+    [ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
+    R1="$(ls "$DATA_ROOT/runs" | head -1)"
+    [ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
+    node -e '
+const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
+process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1);
+' "$DATA_ROOT/runs/$R1/result.json"
+    check "result.json contents are correct" $?
+
+    printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
+    expect_exit "wrong expectExact fails with exit 1" 1 -- \
+        scripts/run-task.sh run "$SANDBOX/wrong.json"
+
+    RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
+    [ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
+        || check "each run gets a distinct run dir (no clobber)" 1
+
+    COUNT=$($TASK list | wc -l)
+    [ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1
+else
+    echo "skip live task cases (docker unavailable)"
+fi
+
+echo
+echo "selftest: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]
@@ -0,0 +1,8 @@
+{
+  "taskVersion": 1,
+  "id": "t-hello-marker",
+  "prompt": "Return your startup marker and nothing else.",
+  "mission": "../missions/hello.json",
+  "expectExact": "MOSAIC_HELLO_OK",
+  "timeoutSeconds": 120
+}