#!/usr/bin/env bash # Sandboxed selftests for the mission/task layer. # # Fast cases are validation-only (no Docker, no network). The final cases # execute real tasks through the container path and therefore require a # working configuration, credentials, and Docker. set -uo pipefail cd "$(dirname "$0")/.." SANDBOX="$(mktemp -d)" trap 'rm -rf "$SANDBOX"' EXIT PASS=0 # Status colors: terminal-only, NO_COLOR-respecting; plain when piped. if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m' else C_OK=""; C_FAIL=""; C_RESET="" fi FAIL=0 expect_exit() { local name="$1" expected="$2" shift 3 local rc "$@" >/dev/null 2>&1 rc=$? if [ "$rc" -eq "$expected" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $name (exit $rc)" else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $name (exit $rc, expected $expected)" fi } check() { if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi } latest_reason() { local latest latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)" [ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null } CONFIG="$SANDBOX/config.json" DATA_ROOT="$SANDBOX/data" mkdir -p "$DATA_ROOT" cat > "$CONFIG" < "$1" < "$1" <<'EOF' {"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]} EOF } # ---------- fast: schema negatives ---------- good_task "$SANDBOX/ok.json" expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json" expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json" printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json" expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json" printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json" expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json" printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json" expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json" expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json" expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json" expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json" good_mission "$SANDBOX/m.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json" expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json" printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json" printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json" expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json" expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json" # validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1 $TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1 M2=$(stat -c %Y "$SANDBOX/ok.json") [ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1 # ---------- retention: prune (deterministic, no Docker) ---------- mkdir -p "$SANDBOX/data" cat > "$SANDBOX/prune-config.json" < "$D/result.json" done mkdir -p "$SANDBOX/data/sessions/sentinel" "$SANDBOX/data/workspaces/sentinel" expect_exit "prune dry-run exits 0" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune [ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 5 ] \ && check "dry-run deleted nothing" 0 || check "dry-run deleted nothing" 1 expect_exit "prune --keep=2 --yes removes oldest" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=2 --yes [ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 2 ] \ && check "kept exactly 2 newest runs" 0 || check "kept exactly 2 newest runs" 1 NEWEST="r-20260903T0100_05Z-suite005" [ -d "$SANDBOX/data/runs/$NEWEST" ] \ && check "newest run kept, oldest pruned" 0 || check "newest run kept, oldest pruned" 1 [ -f "$SANDBOX/data/runs/.pruned.log" ] \ && [ "$(grep -c 'pruned' "$SANDBOX/data/runs/.pruned.log")" -eq 3 ] \ && check "append-only receipt written (3 entries)" 0 \ || check "append-only receipt written (3 entries)" 1 [ -d "$SANDBOX/data/sessions/sentinel" ] && [ -d "$SANDBOX/data/workspaces/sentinel" ] \ && check "sessions/workspaces untouched by prune" 0 \ || check "sessions/workspaces untouched by prune" 1 expect_exit "prune with invalid keep exits 4" 4 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=0 --yes # ---------- adapter seam: deterministic mock cases (Docker, no provider) ---------- if docker info >/dev/null 2>&1; then good_task "$SANDBOX/ok.json" mock_config() { # file adapter cat > "$SANDBOX/$1" < "$SANDBOX/mission-task.json" # Dedicated mission with distinctive directives for the injection assertion. cat > "$SANDBOX/m-seam.json" <<'EOF' {"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]} EOF printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json" expect_exit "mission task runs via mock adapter" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \ scripts/run-task.sh run "$SANDBOX/seam-task.json" grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \ && grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \ && check "mission section injected into generated prompt" 0 \ || check "mission section injected into generated prompt" 1 # retry lineage + relative mission path resolution expect_exit "retry of mission run succeeds" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \ node scripts/mosaic-task.mjs retry "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)" RT="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)" ORIG="$(ls -dt "$SANDBOX/data/runs"/r-* | sed -n 2p | xargs basename)" node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(r.retriedFrom===process.argv[2]?0:1)' \ "$SANDBOX/data/runs/$RT/result.json" "$ORIG" \ && check "retriedFrom lineage recorded" 0 || check "retriedFrom lineage recorded" 1 grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \ && check "mission section present after retry (relative path resolved)" 0 \ || check "mission section present after retry (relative path resolved)" 1 expect_exit "retry of missing run exits 4" 4 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" node scripts/mosaic-task.mjs retry r-missing # skills lifecycle (M17) expect_exit "skill install bundled ms-tools" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools [ -d "$SANDBOX/data/skills-available/ms-tools" ] \ && check "installed to skills-available" 0 || check "installed to skills-available" 1 expect_exit "double install refuses" 1 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools expect_exit "activate enables skill" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh activate ms-tools [ -d "$SANDBOX/data/skills-enabled/ms-tools" ] \ && check "enabled dir populated" 0 || check "enabled dir populated" 1 expect_exit "uninstall while enabled refuses" 1 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools expect_exit "deactivate moves back to available" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh deactivate ms-tools expect_exit "uninstall removes from available" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools # seat skills dispatch (mock evidence) mkdir -p "$SANDBOX/agents/skillseat" "$SANDBOX/data/user" printf '# User\n\nname: Jason\n' > "$SANDBOX/data/user/USER.md" printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md" printf '{"agentVersion":1,"name":"skillseat","skills":["ms-tools"]}' > "$SANDBOX/agents/skillseat/agent.json" printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md" cat > "$SANDBOX/mock-config.json" < "$SANDBOX/data/state/active.json" scripts/skill.sh install ms-tools >/dev/null 2>&1 scripts/skill.sh activate ms-tools >/dev/null 2>&1 env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \ env MOSAIC_AGENTS_DIR="$SANDBOX/agents" \ scripts/agent.sh skillseat "$SANDBOX/seat-stdout.txt" 2>"$SANDBOX/seat-stderr.txt" RC=$? if [ "$RC" -eq 0 ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} seat with enabled skill launches" else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} seat with enabled skill launches (exit $RC)" >&2 echo "SEATCASE stderr:" >&2; cat "$SANDBOX/seat-stderr.txt" >&2 fi grep -q '^MOSAIC_SKILLS=/var/lib/mosaic/skills-enabled/ms-tools$' "$SANDBOX/seat-stderr.txt" 2>/dev/null \ && check "skill path delivered to adapter" 0 || check "skill path delivered to adapter" 1 # capability policy (M9): least-privilege intersection POL="$SANDBOX/data/workspaces"; mkdir -p "$POL" pol_run() { # missionTools(ABSENT|json) taskTools(ABSENT|json) -> stderr MOSAIC_TOOLS value if [ "$1" = "ABSENT" ]; then printf '{"missionVersion":1,"id":"m-pol","objective":"o"}' > "$SANDBOX/pol-m.json" else printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":[%s]}}' "$1" > "$SANDBOX/pol-m.json" fi if [ "$2" = "ABSENT" ]; then printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json"}' > "$SANDBOX/pol-t.json" else printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json","capabilities":{"tools":[%s]}}' "$2" > "$SANDBOX/pol-t.json" fi env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \ scripts/run-task.sh run "$SANDBOX/pol-t.json" >/dev/null 2>&1 grep -o '^MOSAIC_TOOLS=.*' "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)/stderr.txt" 2>/dev/null } [ "$(pol_run '"read","bash"' ABSENT)" = 'MOSAIC_TOOLS=read,bash' ] \ && check "policy: mission only -> mission tools" 0 || check "policy: mission only -> mission tools" 1 [ "$(pol_run ABSENT '"read","bash"')" = 'MOSAIC_TOOLS=read,bash' ] \ && check "policy: task only -> task tools" 0 || check "policy: task only -> task tools" 1 [ "$(pol_run '"read"' '"bash","read"')" = 'MOSAIC_TOOLS=read' ] \ && check "policy: both -> intersection (task narrowed)" 0 || check "policy: both -> intersection (task narrowed)" 1 [ "$(pol_run '"grep"' '"bash","read"')" = 'MOSAIC_TOOLS=' ] \ && check "policy: empty intersection -> tool-free" 0 || check "policy: empty intersection -> tool-free" 1 printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/pol-m.json" expect_exit "invalid mission capabilities rejected" 2 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json" env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json" # live user context (M14): dispatched to every launch without rebuild mkdir -p "$SANDBOX/data/user" printf '\nUSER-CANON-MARKER\n' >> "$SANDBOX/data/user/USER.md" expect_exit "task run with user layer present" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \ scripts/run-task.sh run "$SANDBOX/ok.json" grep -q 'USER CONTEXT: USER.md' "$SANDBOX/data/system-prompt.md" \ && grep -q 'USER-CANON-MARKER' "$SANDBOX/data/system-prompt.md" \ && check "user context dispatched into generated prompt" 0 \ || check "user context dispatched into generated prompt" 1 else echo "skip adapter seam cases (docker daemon unavailable)" fi # ---------- workspace + capabilities (M5): deterministic mock cases ---------- if docker info >/dev/null 2>&1; then printf '{"taskVersion":1,"id":"t-ws","prompt":"ignored","workspace":"suitews","capabilities":{"tools":["read","bash"]}}' > "$SANDBOX/ws-task.json" expect_exit "workspace+tools task runs via mock" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \ scripts/run-task.sh run "$SANDBOX/ws-task.json" WSL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)" grep -q '^MOSAIC_WORKSPACE=/var/lib/mosaic/workspaces/suitews$' "$WSL/stderr.txt" 2>/dev/null \ && grep -q '^MOSAIC_TOOLS=read,bash$' "$WSL/stderr.txt" 2>/dev/null \ && check "workspace path + tools delivered to adapter" 0 \ || check "workspace path + tools delivered to adapter" 1 [ -d "$SANDBOX/data/workspaces/suitews" ] \ && check "persistent workspace created on host" 0 \ || check "persistent workspace created on host" 1 expect_exit "plain task still runs (no workspace/tools)" 0 -- \ env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \ scripts/run-task.sh run "$SANDBOX/ok.json" PL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)" grep -q '^MOSAIC_WORKSPACE=$' "$PL/stderr.txt" 2>/dev/null \ && check "workspace var present but empty when absent" 0 || check "workspace var present but empty when absent" 1 grep -q '^MOSAIC_TOOLS=$' "$PL/stderr.txt" 2>/dev/null \ && check "tools empty when absent" 0 || check "tools empty when absent" 1 printf '{"taskVersion":1,"id":"t-badtool","prompt":"x","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/badtool.json" expect_exit "unknown tool exits 2" 2 -- $TASK validate "$SANDBOX/badtool.json" printf '{"taskVersion":1,"id":"t-badws","prompt":"x","workspace":"../escape"}' > "$SANDBOX/badws.json" expect_exit "workspace traversal exits 2" 2 -- $TASK validate "$SANDBOX/badws.json" else echo "skip workspace/capability cases (docker daemon unavailable)" fi # ---------- live: real runs (Docker + credentials required) ---------- # On failure, surface the run record + agent stderr BEFORE the sandbox # cleanup destroys them. Never let a wrong-exit mask the real reason. dump_latest_run() { local latest latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)" if [ -n "$latest" ]; then echo "--- latest run evidence: $latest ---" >&2 cat "$latest/result.json" 2>/dev/null >&2 echo "--- stderr.txt (tail) ---" >&2 tail -8 "$latest/stderr.txt" 2>/dev/null >&2 else echo "--- no run dir was created at all ---" >&2 fi } if docker info >/dev/null 2>&1; then RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l) if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} live hello task succeeds with exact marker" else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} live hello task succeeds with exact marker" >&2 dump_latest_run fi R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above [ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1 node -e ' const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1); ' "$DATA_ROOT/runs/$R1/result.json" check "result.json contents are correct" $? printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json" scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1 RC=$? WRONG_REASON="$(latest_reason)" if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} wrong expectExact fails with exit 1 (reason: expect-mismatch)" else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2 dump_latest_run fi RUNS2=$(ls "$DATA_ROOT/runs" | wc -l) [ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \ || check "each run gets a distinct run dir (no clobber)" 1 COUNT=$($TASK list | wc -l) [ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1 # session fork (M11): teach in base, fork into child, child recalls; # ancestor file count must be unchanged by the fork printf '{"taskVersion":1,"id":"t-fork-base","prompt":"Remember this code word for later: mosaico. Reply with exactly: REMEMBERED","session":"suite-base","expectExact":"REMEMBERED","timeoutSeconds":180}' > "$SANDBOX/fb.json" expect_exit "fork base: teach succeeds" 0 -- scripts/run-task.sh run "$SANDBOX/fb.json" BASECOUNT=$(ls "$SANDBOX/data/sessions/suite-base" | wc -l) printf '{"taskVersion":1,"id":"t-fork-child","prompt":"What code word did I ask you to remember? Reply with only the code word.","session":"suite-child","sessionForkFrom":"suite-base","timeoutSeconds":180}' > "$SANDBOX/fc.json" expect_exit "fork child recalls ancestor context" 0 -- scripts/run-task.sh run "$SANDBOX/fc.json" FR="$(ls -dt "$DATA_ROOT/runs"/r-* | head -1)" node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(String(r.response).toLowerCase().includes("mosaico")?0:1)' "$FR/result.json" \ && check "forked child recalled ancestor code word" 0 || check "forked child recalled ancestor code word" 1 [ "$(ls "$SANDBOX/data/sessions/suite-base" | wc -l)" -eq "$BASECOUNT" ] \ && check "ancestor session untouched by fork" 0 || check "ancestor session untouched by fork" 1 [ "$(ls "$SANDBOX/data/sessions/suite-child" | wc -l)" -ge 1 ] \ && check "child session dir has its own branch file" 0 || check "child session dir has its own branch file" 1 else echo "skip live task cases (docker unavailable)" fi # ---------- onboarding (M16): deterministic, no Docker ---------- ONB="$SANDBOX/data" cat > "$SANDBOX/onb-config.json" < "$SANDBOX/recall-task.json" RC=0 scripts/run-task.sh run "$SANDBOX/recall-task.json" >/dev/null 2>&1 || RC=$? [ "$RC" -eq 0 ] && check "user recall run succeeds" 0 || check "user recall run succeeds (exit $RC)" 1 FR="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)" RESP="$(node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.response||"")}catch{console.log("")}' "$FR/result.json" 2>/dev/null)" echo "$RESP" | grep -qi "jason" \ && check "recalled user name (response: $RESP)" 0 || check "recalled user name (response: $RESP)" 1 # headless run must NOT carry an agent identity section ! grep -q 'AGENT IDENTITY' "$SANDBOX/data/system-prompt.md" 2>/dev/null \ && check "no agent identity on headless run" 0 || check "no agent identity on headless run" 1 git -C "$SANDBOX/repo" rev-parse >/dev/null 2>&1 || true unset MOSAIC_CONFIG echo echo "selftest: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ]