jason.woltje
88eef507b0
- mosaic-task.mjs prune [--keep=N] [--yes]: default keep 50; without --yes lists candidates without deleting - only r-* directories under the runs root; symlinks skipped; sessions/workspaces/state/config untouched (asserted by suite sentinels) - append-only receipt runs/.pruned.log records every pruned id - test-task.sh: +8 retention cases (dry-run no-delete, keep-N, newest kept, receipt, isolation, invalid keep, empty no-op) Also: suite hardening - prune section scopes its config per-command (no export/unset leaking into later sections); duplicated check() removed; latest_reason hoisted to helpers; status colors now green OK / red FAIL (terminal-only, NO_COLOR-aware) per owner UX feedback. Closes #32
315 lines
16 KiB
Bash
Executable File
315 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Sandboxed selftests for the mission/task layer.
|
|
#
|
|
# Fast cases are validation-only (no Docker, no network). The final cases
|
|
# execute real tasks through the container path and therefore require a
|
|
# working configuration, credentials, and Docker.
|
|
set -uo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
SANDBOX="$(mktemp -d)"
|
|
trap 'rm -rf "$SANDBOX"' EXIT
|
|
|
|
PASS=0
|
|
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
|
|
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
|
|
C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m'
|
|
else
|
|
C_OK=""; C_FAIL=""; C_RESET=""
|
|
fi
|
|
FAIL=0
|
|
|
|
expect_exit() {
|
|
local name="$1" expected="$2"
|
|
shift 3
|
|
local rc
|
|
"$@" >/dev/null 2>&1
|
|
rc=$?
|
|
if [ "$rc" -eq "$expected" ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $name (exit $rc)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $name (exit $rc, expected $expected)"
|
|
fi
|
|
}
|
|
|
|
check() {
|
|
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi
|
|
}
|
|
|
|
latest_reason() {
|
|
local latest
|
|
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
|
|
}
|
|
|
|
CONFIG="$SANDBOX/config.json"
|
|
DATA_ROOT="$SANDBOX/data"
|
|
mkdir -p "$DATA_ROOT"
|
|
cat > "$CONFIG" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$DATA_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash"}}
|
|
EOF
|
|
export MOSAIC_CONFIG="$CONFIG"
|
|
|
|
TASK="node scripts/mosaic-task.mjs"
|
|
|
|
good_task() { # args: file [overrides as jq-less JSON fragments]
|
|
cat > "$1" <<EOF
|
|
{"taskVersion":1,"id":"t-ok","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_HELLO_OK"}
|
|
EOF
|
|
}
|
|
good_mission() {
|
|
cat > "$1" <<'EOF'
|
|
{"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]}
|
|
EOF
|
|
}
|
|
|
|
# ---------- fast: schema negatives ----------
|
|
good_task "$SANDBOX/ok.json"
|
|
expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json"
|
|
expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json"
|
|
|
|
printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json"
|
|
expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json"
|
|
|
|
printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json"
|
|
expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json"
|
|
expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json"
|
|
expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json"
|
|
expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json"
|
|
expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json"
|
|
|
|
good_mission "$SANDBOX/m.json"
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json"
|
|
expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json"
|
|
|
|
printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json"
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json"
|
|
expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json"
|
|
|
|
expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json"
|
|
|
|
# validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json
|
|
M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1
|
|
$TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
|
|
M2=$(stat -c %Y "$SANDBOX/ok.json")
|
|
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
|
|
|
|
# ---------- retention: prune (deterministic, no Docker) ----------
|
|
mkdir -p "$SANDBOX/data"
|
|
cat > "$SANDBOX/prune-config.json" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m"}}
|
|
EOF
|
|
for i in 1 2 3 4 5; do
|
|
D="$SANDBOX/data/runs/r-20260903T0100_0${i}Z-suite00$i"
|
|
mkdir -p "$D"
|
|
printf '{"runVersion":1,"runId":"r-20260903T0100_0%sZ-suite00%s","taskId":"t","status":"succeeded"}' "$i" "$i" > "$D/result.json"
|
|
done
|
|
mkdir -p "$SANDBOX/data/sessions/sentinel" "$SANDBOX/data/workspaces/sentinel"
|
|
|
|
expect_exit "prune dry-run exits 0" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune
|
|
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 5 ] \
|
|
&& check "dry-run deleted nothing" 0 || check "dry-run deleted nothing" 1
|
|
|
|
expect_exit "prune --keep=2 --yes removes oldest" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=2 --yes
|
|
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 2 ] \
|
|
&& check "kept exactly 2 newest runs" 0 || check "kept exactly 2 newest runs" 1
|
|
NEWEST="r-20260903T0100_05Z-suite005"
|
|
[ -d "$SANDBOX/data/runs/$NEWEST" ] \
|
|
&& check "newest run kept, oldest pruned" 0 || check "newest run kept, oldest pruned" 1
|
|
[ -f "$SANDBOX/data/runs/.pruned.log" ] \
|
|
&& [ "$(grep -c 'pruned' "$SANDBOX/data/runs/.pruned.log")" -eq 3 ] \
|
|
&& check "append-only receipt written (3 entries)" 0 \
|
|
|| check "append-only receipt written (3 entries)" 1
|
|
[ -d "$SANDBOX/data/sessions/sentinel" ] && [ -d "$SANDBOX/data/workspaces/sentinel" ] \
|
|
&& check "sessions/workspaces untouched by prune" 0 \
|
|
|| check "sessions/workspaces untouched by prune" 1
|
|
expect_exit "prune with invalid keep exits 4" 4 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=0 --yes
|
|
|
|
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
|
|
if docker info >/dev/null 2>&1; then
|
|
good_task "$SANDBOX/ok.json"
|
|
mock_config() { # file adapter
|
|
cat > "$SANDBOX/$1" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"$2"}}
|
|
EOF
|
|
}
|
|
|
|
mock_config mock-adapters.json mock
|
|
expect_exit "mock adapter: gate passes on matching mock response" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
|
|
expect_exit "mock adapter: expect-mismatch recorded" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=SOMETHING_ELSE \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
[ "$(latest_reason)" = "expect-mismatch" ] \
|
|
&& check "mismatch reason recorded" 0 || check "mismatch reason recorded" 1
|
|
|
|
mock_config bad-adapter.json nonexistent
|
|
# The config layer rejects unknown adapters first, so run-task.sh fails
|
|
# closed (exit 1) before any container or run record exists.
|
|
expect_exit "unknown adapter fails closed" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/bad-adapter.json" scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
|
|
good_mission "$SANDBOX/m-ok.json"
|
|
printf '{"taskVersion":1,"id":"t-mission","prompt":"ignored by mock","mission":"m-ok.json"}' > "$SANDBOX/mission-task.json"
|
|
# Dedicated mission with distinctive directives for the injection assertion.
|
|
cat > "$SANDBOX/m-seam.json" <<'EOF'
|
|
{"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]}
|
|
EOF
|
|
printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json"
|
|
expect_exit "mission task runs via mock adapter" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/seam-task.json"
|
|
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
|
|
&& grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \
|
|
&& check "mission section injected into generated prompt" 0 \
|
|
|| check "mission section injected into generated prompt" 1
|
|
|
|
# retry lineage + relative mission path resolution
|
|
expect_exit "retry of mission run succeeds" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
node scripts/mosaic-task.mjs retry "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
|
|
RT="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
|
|
ORIG="$(ls -dt "$SANDBOX/data/runs"/r-* | sed -n 2p | xargs basename)"
|
|
node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(r.retriedFrom===process.argv[2]?0:1)' \
|
|
"$SANDBOX/data/runs/$RT/result.json" "$ORIG" \
|
|
&& check "retriedFrom lineage recorded" 0 || check "retriedFrom lineage recorded" 1
|
|
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
|
|
&& check "mission section present after retry (relative path resolved)" 0 \
|
|
|| check "mission section present after retry (relative path resolved)" 1
|
|
expect_exit "retry of missing run exits 4" 4 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" node scripts/mosaic-task.mjs retry r-missing
|
|
|
|
# capability policy (M9): least-privilege intersection
|
|
POL="$SANDBOX/data/workspaces"; mkdir -p "$POL"
|
|
pol_run() { # missionTools(ABSENT|json) taskTools(ABSENT|json) -> stderr MOSAIC_TOOLS value
|
|
if [ "$1" = "ABSENT" ]; then
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o"}' > "$SANDBOX/pol-m.json"
|
|
else
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":[%s]}}' "$1" > "$SANDBOX/pol-m.json"
|
|
fi
|
|
if [ "$2" = "ABSENT" ]; then
|
|
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json"}' > "$SANDBOX/pol-t.json"
|
|
else
|
|
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json","capabilities":{"tools":[%s]}}' "$2" > "$SANDBOX/pol-t.json"
|
|
fi
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/pol-t.json" >/dev/null 2>&1
|
|
grep -o '^MOSAIC_TOOLS=.*' "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)/stderr.txt" 2>/dev/null
|
|
}
|
|
[ "$(pol_run '"read","bash"' ABSENT)" = 'MOSAIC_TOOLS=read,bash' ] \
|
|
&& check "policy: mission only -> mission tools" 0 || check "policy: mission only -> mission tools" 1
|
|
[ "$(pol_run ABSENT '"read","bash"')" = 'MOSAIC_TOOLS=read,bash' ] \
|
|
&& check "policy: task only -> task tools" 0 || check "policy: task only -> task tools" 1
|
|
[ "$(pol_run '"read"' '"bash","read"')" = 'MOSAIC_TOOLS=read' ] \
|
|
&& check "policy: both -> intersection (task narrowed)" 0 || check "policy: both -> intersection (task narrowed)" 1
|
|
[ "$(pol_run '"grep"' '"bash","read"')" = 'MOSAIC_TOOLS=' ] \
|
|
&& check "policy: empty intersection -> tool-free" 0 || check "policy: empty intersection -> tool-free" 1
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/pol-m.json"
|
|
expect_exit "invalid mission capabilities rejected" 2 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json"
|
|
else
|
|
echo "skip adapter seam cases (docker daemon unavailable)"
|
|
fi
|
|
|
|
# ---------- workspace + capabilities (M5): deterministic mock cases ----------
|
|
if docker info >/dev/null 2>&1; then
|
|
printf '{"taskVersion":1,"id":"t-ws","prompt":"ignored","workspace":"suitews","capabilities":{"tools":["read","bash"]}}' > "$SANDBOX/ws-task.json"
|
|
expect_exit "workspace+tools task runs via mock" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/ws-task.json"
|
|
WSL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
grep -q '^MOSAIC_WORKSPACE=/var/lib/mosaic/workspaces/suitews$' "$WSL/stderr.txt" 2>/dev/null \
|
|
&& grep -q '^MOSAIC_TOOLS=read,bash$' "$WSL/stderr.txt" 2>/dev/null \
|
|
&& check "workspace path + tools delivered to adapter" 0 \
|
|
|| check "workspace path + tools delivered to adapter" 1
|
|
[ -d "$SANDBOX/data/workspaces/suitews" ] \
|
|
&& check "persistent workspace created on host" 0 \
|
|
|| check "persistent workspace created on host" 1
|
|
|
|
expect_exit "plain task still runs (no workspace/tools)" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
PL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
grep -q '^MOSAIC_WORKSPACE=$' "$PL/stderr.txt" 2>/dev/null \
|
|
&& check "workspace var present but empty when absent" 0 || check "workspace var present but empty when absent" 1
|
|
grep -q '^MOSAIC_TOOLS=$' "$PL/stderr.txt" 2>/dev/null \
|
|
&& check "tools empty when absent" 0 || check "tools empty when absent" 1
|
|
|
|
printf '{"taskVersion":1,"id":"t-badtool","prompt":"x","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/badtool.json"
|
|
expect_exit "unknown tool exits 2" 2 -- $TASK validate "$SANDBOX/badtool.json"
|
|
printf '{"taskVersion":1,"id":"t-badws","prompt":"x","workspace":"../escape"}' > "$SANDBOX/badws.json"
|
|
expect_exit "workspace traversal exits 2" 2 -- $TASK validate "$SANDBOX/badws.json"
|
|
else
|
|
echo "skip workspace/capability cases (docker daemon unavailable)"
|
|
fi
|
|
|
|
# ---------- live: real runs (Docker + credentials required) ----------
|
|
# On failure, surface the run record + agent stderr BEFORE the sandbox
|
|
# cleanup destroys them. Never let a wrong-exit mask the real reason.
|
|
dump_latest_run() {
|
|
local latest
|
|
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
if [ -n "$latest" ]; then
|
|
echo "--- latest run evidence: $latest ---" >&2
|
|
cat "$latest/result.json" 2>/dev/null >&2
|
|
echo "--- stderr.txt (tail) ---" >&2
|
|
tail -8 "$latest/stderr.txt" 2>/dev/null >&2
|
|
else
|
|
echo "--- no run dir was created at all ---" >&2
|
|
fi
|
|
}
|
|
|
|
if docker info >/dev/null 2>&1; then
|
|
RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l)
|
|
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} live hello task succeeds with exact marker"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} live hello task succeeds with exact marker" >&2
|
|
dump_latest_run
|
|
fi
|
|
|
|
R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above
|
|
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
|
|
node -e '
|
|
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
|
process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1);
|
|
' "$DATA_ROOT/runs/$R1/result.json"
|
|
check "result.json contents are correct" $?
|
|
|
|
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
|
|
scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
|
|
RC=$?
|
|
WRONG_REASON="$(latest_reason)"
|
|
if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} wrong expectExact fails with exit 1 (reason: expect-mismatch)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
|
|
dump_latest_run
|
|
fi
|
|
|
|
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
|
|
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
|
|
|| check "each run gets a distinct run dir (no clobber)" 1
|
|
|
|
COUNT=$($TASK list | wc -l)
|
|
[ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1
|
|
else
|
|
echo "skip live task cases (docker unavailable)"
|
|
fi
|
|
|
|
echo
|
|
echo "selftest: $PASS passed, $FAIL failed"
|
|
[ "$FAIL" -eq 0 ]
|