mosaicstack/stack
21
0
Fork 1
Code Issues 338 Pull Requests 13 Actions Packages Projects Releases 15 Wiki Activity
Files
ca8135d70c0f8f852e4b58d9d491135bbda0f0bb
stack/scripts/test-task.sh
T
jason.woltje ca8135d70c feat(roles): M18 seat-role progressive capability restriction (#45) 
Role contracts (roles/<role>.json): roleVersion, name bound to filename,
tools ceiling (subset of pi built-ins), network declared (none|api-only|
open; enforced when network policy lands). Strict schema, fail closed -
a non-role document refuses resolution.

mosaic-task.mjs resolve-role: config-free contract validation, emits
MOSAIC_ROLE_TOOLS / MOSAIC_ROLE_NETWORK.

agent.sh: a declared role binds to its contract. Missing/invalid contract
refuses the launch (exit 2, names the role - the under-equipped-seat
failure mode, mirroring M17 skills). Effective tools = ceiling ∩ requested
(CLI --tools or agent.json caps); no request -> ceiling stands; narrowing
and tool-free outcomes loud on stderr. Adapters unchanged; headless M9
chain (mission ∩ task) untouched.

Ships roles/researcher.json (existing seat declares the role; without the
contract the fail-closed gate would refuse its launch).

Task suite 74 -> 88: contract resolution, wrong-kind/name/network/
duplicate/unsupported/missing refusals, ceiling narrowing E2E (mock
adapter), tool-free E2E, missing-contract refusal. Test-authoring
correction recorded in BUILD-LOG (a check that registered on one path
only, caught by count arithmetic).

Suites 24/88/14/17 + verify green.
2026-09-03 17:25:17 -05:00

485 lines
29 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Sandboxed selftests for the mission/task layer.
#
# Fast cases are validation-only (no Docker, no network). The final cases
# execute real tasks through the container path and therefore require a
# working configuration, credentials, and Docker.
set -uo pipefail
cd "$(dirname "$0")/.."
SANDBOX="$(mktemp -d)"
trap 'rm -rf "$SANDBOX"' EXIT
PASS=0
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m'
else
C_OK=""; C_FAIL=""; C_RESET=""
fi
FAIL=0
expect_exit() {
local name="$1" expected="$2"
shift 3
local rc
"$@" >/dev/null 2>&1
rc=$?
if [ "$rc" -eq "$expected" ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $name (exit $rc)"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $name (exit $rc, expected $expected)"
fi
}
check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi
}
latest_reason() {
local latest
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
}
CONFIG="$SANDBOX/config.json"
DATA_ROOT="$SANDBOX/data"
mkdir -p "$DATA_ROOT"
cat > "$CONFIG" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$DATA_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash"}}
EOF
export MOSAIC_CONFIG="$CONFIG"
TASK="node scripts/mosaic-task.mjs"
good_task() { # args: file [overrides as jq-less JSON fragments]
cat > "$1" <<EOF
{"taskVersion":1,"id":"t-ok","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_HELLO_OK"}
EOF
}
good_mission() {
cat > "$1" <<'EOF'
{"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]}
EOF
}
# ---------- fast: schema negatives ----------
good_task "$SANDBOX/ok.json"
expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json"
expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json"
printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json"
expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json"
printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json"
expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json"
printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json"
expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json"
expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json"
expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json"
expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json"
good_mission "$SANDBOX/m.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json"
expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json"
printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json"
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json"
expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json"
expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json"
# validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json
M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1
$TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
M2=$(stat -c %Y "$SANDBOX/ok.json")
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
# ---------- retention: prune (deterministic, no Docker) ----------
mkdir -p "$SANDBOX/data"
cat > "$SANDBOX/prune-config.json" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m"}}
EOF
for i in 1 2 3 4 5; do
D="$SANDBOX/data/runs/r-20260903T0100_0${i}Z-suite00$i"
mkdir -p "$D"
printf '{"runVersion":1,"runId":"r-20260903T0100_0%sZ-suite00%s","taskId":"t","status":"succeeded"}' "$i" "$i" > "$D/result.json"
done
mkdir -p "$SANDBOX/data/sessions/sentinel" "$SANDBOX/data/workspaces/sentinel"
expect_exit "prune dry-run exits 0" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 5 ] \
&& check "dry-run deleted nothing" 0 || check "dry-run deleted nothing" 1
expect_exit "prune --keep=2 --yes removes oldest" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=2 --yes
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 2 ] \
&& check "kept exactly 2 newest runs" 0 || check "kept exactly 2 newest runs" 1
NEWEST="r-20260903T0100_05Z-suite005"
[ -d "$SANDBOX/data/runs/$NEWEST" ] \
&& check "newest run kept, oldest pruned" 0 || check "newest run kept, oldest pruned" 1
[ -f "$SANDBOX/data/runs/.pruned.log" ] \
&& [ "$(grep -c 'pruned' "$SANDBOX/data/runs/.pruned.log")" -eq 3 ] \
&& check "append-only receipt written (3 entries)" 0 \
|| check "append-only receipt written (3 entries)" 1
[ -d "$SANDBOX/data/sessions/sentinel" ] && [ -d "$SANDBOX/data/workspaces/sentinel" ] \
&& check "sessions/workspaces untouched by prune" 0 \
|| check "sessions/workspaces untouched by prune" 1
expect_exit "prune with invalid keep exits 4" 4 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=0 --yes
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
if docker info >/dev/null 2>&1; then
good_task "$SANDBOX/ok.json"
mock_config() { # file adapter
cat > "$SANDBOX/$1" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"$2"}}
EOF
}
mock_config mock-adapters.json mock
expect_exit "mock adapter: gate passes on matching mock response" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
scripts/run-task.sh run "$SANDBOX/ok.json"
expect_exit "mock adapter: expect-mismatch recorded" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=SOMETHING_ELSE \
scripts/run-task.sh run "$SANDBOX/ok.json"
[ "$(latest_reason)" = "expect-mismatch" ] \
&& check "mismatch reason recorded" 0 || check "mismatch reason recorded" 1
mock_config bad-adapter.json nonexistent
# The config layer rejects unknown adapters first, so run-task.sh fails
# closed (exit 1) before any container or run record exists.
expect_exit "unknown adapter fails closed" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/bad-adapter.json" scripts/run-task.sh run "$SANDBOX/ok.json"
good_mission "$SANDBOX/m-ok.json"
printf '{"taskVersion":1,"id":"t-mission","prompt":"ignored by mock","mission":"m-ok.json"}' > "$SANDBOX/mission-task.json"
# Dedicated mission with distinctive directives for the injection assertion.
cat > "$SANDBOX/m-seam.json" <<'EOF'
{"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]}
EOF
printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json"
expect_exit "mission task runs via mock adapter" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
scripts/run-task.sh run "$SANDBOX/seam-task.json"
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
&& grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \
&& check "mission section injected into generated prompt" 0 \
|| check "mission section injected into generated prompt" 1
# retry lineage + relative mission path resolution
expect_exit "retry of mission run succeeds" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
node scripts/mosaic-task.mjs retry "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
RT="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
ORIG="$(ls -dt "$SANDBOX/data/runs"/r-* | sed -n 2p | xargs basename)"
node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(r.retriedFrom===process.argv[2]?0:1)' \
"$SANDBOX/data/runs/$RT/result.json" "$ORIG" \
&& check "retriedFrom lineage recorded" 0 || check "retriedFrom lineage recorded" 1
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
&& check "mission section present after retry (relative path resolved)" 0 \
|| check "mission section present after retry (relative path resolved)" 1
expect_exit "retry of missing run exits 4" 4 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" node scripts/mosaic-task.mjs retry r-missing
# skills lifecycle (M17)
expect_exit "skill install bundled ms-tools" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools
[ -d "$SANDBOX/data/skills-available/ms-tools" ] \
&& check "installed to skills-available" 0 || check "installed to skills-available" 1
expect_exit "double install refuses" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools
expect_exit "activate enables skill" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh activate ms-tools
[ -d "$SANDBOX/data/skills-enabled/ms-tools" ] \
&& check "enabled dir populated" 0 || check "enabled dir populated" 1
expect_exit "uninstall while enabled refuses" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools
expect_exit "deactivate moves back to available" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh deactivate ms-tools
expect_exit "uninstall removes from available" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools
# seat skills dispatch (mock evidence)
mkdir -p "$SANDBOX/agents/skillseat" "$SANDBOX/data/user"
printf '# User\n\nname: Jason\n' > "$SANDBOX/data/user/USER.md"
printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md"
printf '{"agentVersion":1,"name":"skillseat","skills":["ms-tools"]}' > "$SANDBOX/agents/skillseat/agent.json"
printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md"
cat > "$SANDBOX/mock-config.json" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"mock"}}
EOF
REL="$(tr -d '[:space:]' < RELEASE)"
mkdir -p "$SANDBOX/data/state"
printf '{"pointerVersion":1,"release":"%s","imageTag":"mosaic-poc-agent:0.84.4-r%s","activatedAt":"2026-01-01T00:00:00Z"}\n' "$REL" "$REL" > "$SANDBOX/data/state/active.json"
scripts/skill.sh install ms-tools >/dev/null 2>&1
scripts/skill.sh activate ms-tools >/dev/null 2>&1
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
env MOSAIC_AGENTS_DIR="$SANDBOX/agents" \
scripts/agent.sh skillseat </dev/null >"$SANDBOX/seat-stdout.txt" 2>"$SANDBOX/seat-stderr.txt"
RC=$?
if [ "$RC" -eq 0 ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} seat with enabled skill launches"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} seat with enabled skill launches (exit $RC)" >&2
echo "SEATCASE stderr:" >&2; cat "$SANDBOX/seat-stderr.txt" >&2
fi
grep -q '^MOSAIC_SKILLS=/var/lib/mosaic/skills-enabled/ms-tools$' "$SANDBOX/seat-stderr.txt" 2>/dev/null \
&& check "skill path delivered to adapter" 0 || check "skill path delivered to adapter" 1
# seat role ceiling (M18): role contracts cap seat capabilities.
# Contract validation is CLI-exercised (no docker); the seat wiring gets
# two mock-adapter launches (narrowing, tool-free) plus one pre-docker
# refusal (missing contract).
ROLE_OUT="$(node scripts/mosaic-task.mjs resolve-role roles/researcher.json 2>/dev/null)" \
&& printf '%s\n' "$ROLE_OUT" | grep -q '^MOSAIC_ROLE_TOOLS=read,grep,find,ls,bash$' \
&& printf '%s\n' "$ROLE_OUT" | grep -q '^MOSAIC_ROLE_NETWORK=none$' \
&& check "shipped researcher contract resolves (ceiling + network)" 0 || check "shipped researcher contract resolves (ceiling + network)" 1
expect_exit "resolve-role refuses a non-role document (conductor policy)" 2 -- \
node scripts/mosaic-task.mjs resolve-role roles/conductor-policy.json
mkdir -p "$SANDBOX/roles" "$SANDBOX/agents/roleseat" "$SANDBOX/no-roles"
printf '{"roleVersion":1,"name":"other","tools":["read"]}' > "$SANDBOX/roles/mismatch.json"
expect_exit "resolve-role refuses name/filename mismatch" 2 -- \
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/mismatch.json"
printf '{"roleVersion":1,"name":"badnet","tools":["read"],"network":"everywhere"}' > "$SANDBOX/roles/badnet.json"
expect_exit "resolve-role refuses unknown network declaration" 2 -- \
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/badnet.json"
printf '{"roleVersion":1,"name":"dupe","tools":["read","read"]}' > "$SANDBOX/roles/dupe.json"
expect_exit "resolve-role refuses duplicate role tool" 2 -- \
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/dupe.json"
printf '{"roleVersion":1,"name":"aliens","tools":["read","render3d"]}' > "$SANDBOX/roles/aliens.json"
expect_exit "resolve-role refuses unsupported tool" 2 -- \
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/aliens.json"
expect_exit "resolve-role refuses missing contract file" 4 -- \
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/absent.json"
printf '# SOUL - roleseat\n\nVerifies before claiming.\n' > "$SANDBOX/agents/roleseat/SOUL.md"
printf '{"agentVersion":1,"name":"roleseat","role":"analyst","capabilities":{"tools":["read","write","bash"]}}' > "$SANDBOX/agents/roleseat/agent.json"
printf '{"roleVersion":1,"name":"analyst","tools":["read","grep","bash"],"network":"none"}' > "$SANDBOX/roles/analyst.json"
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/roles" \
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
RC=$?
if [ "$RC" -eq 0 ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} seat launches under role ceiling"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} seat launches under role ceiling (exit $RC)" >&2
echo "ROLESEAT stderr:" >&2; cat "$SANDBOX/roleseat-stderr.txt" >&2
fi
grep -q '^MOSAIC_TOOLS=read,bash$' "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
&& check "role ceiling narrows seat tools (read,write,bash -> read,bash)" 0 || check "role ceiling narrows seat tools (read,write,bash -> read,bash)" 1
grep -q "role 'analyst' ceiling narrowed tools -> read,bash" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
&& check "narrowing recorded loudly" 0 || check "narrowing recorded loudly" 1
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/no-roles" \
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
RC=$?
if [ "$RC" -eq 2 ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} missing role contract refuses launch (exit 2)"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} missing role contract refuses launch (exit $RC)" >&2
echo "ROLESEAT stderr:" >&2; cat "$SANDBOX/roleseat-stderr.txt" >&2
fi
grep -q "role 'analyst' is declared but has no contract" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
&& check "missing-contract refusal names the role" 0 || check "missing-contract refusal names the role" 1
printf '{"agentVersion":1,"name":"roleseat","role":"analyst","capabilities":{"tools":["write","edit"]}}' > "$SANDBOX/agents/roleseat/agent.json"
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/roles" \
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
grep -q '^MOSAIC_TOOLS=$' "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
&& check "empty ceiling intersection -> tool-free seat" 0 || check "empty ceiling intersection -> tool-free seat" 1
grep -q "nothing in common -> tool-free seat" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
&& check "tool-free outcome recorded loudly" 0 || check "tool-free outcome recorded loudly" 1
# capability policy (M9): least-privilege intersection
POL="$SANDBOX/data/workspaces"; mkdir -p "$POL"
pol_run() { # missionTools(ABSENT|json) taskTools(ABSENT|json) -> stderr MOSAIC_TOOLS value
if [ "$1" = "ABSENT" ]; then
printf '{"missionVersion":1,"id":"m-pol","objective":"o"}' > "$SANDBOX/pol-m.json"
else
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":[%s]}}' "$1" > "$SANDBOX/pol-m.json"
fi
if [ "$2" = "ABSENT" ]; then
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json"}' > "$SANDBOX/pol-t.json"
else
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json","capabilities":{"tools":[%s]}}' "$2" > "$SANDBOX/pol-t.json"
fi
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
scripts/run-task.sh run "$SANDBOX/pol-t.json" >/dev/null 2>&1
grep -o '^MOSAIC_TOOLS=.*' "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)/stderr.txt" 2>/dev/null
}
[ "$(pol_run '"read","bash"' ABSENT)" = 'MOSAIC_TOOLS=read,bash' ] \
&& check "policy: mission only -> mission tools" 0 || check "policy: mission only -> mission tools" 1
[ "$(pol_run ABSENT '"read","bash"')" = 'MOSAIC_TOOLS=read,bash' ] \
&& check "policy: task only -> task tools" 0 || check "policy: task only -> task tools" 1
[ "$(pol_run '"read"' '"bash","read"')" = 'MOSAIC_TOOLS=read' ] \
&& check "policy: both -> intersection (task narrowed)" 0 || check "policy: both -> intersection (task narrowed)" 1
[ "$(pol_run '"grep"' '"bash","read"')" = 'MOSAIC_TOOLS=' ] \
&& check "policy: empty intersection -> tool-free" 0 || check "policy: empty intersection -> tool-free" 1
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/pol-m.json"
expect_exit "invalid mission capabilities rejected" 2 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json"
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json"
# live user context (M14): dispatched to every launch without rebuild
mkdir -p "$SANDBOX/data/user"
printf '\nUSER-CANON-MARKER\n' >> "$SANDBOX/data/user/USER.md"
expect_exit "task run with user layer present" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
scripts/run-task.sh run "$SANDBOX/ok.json"
grep -q 'USER CONTEXT: USER.md' "$SANDBOX/data/system-prompt.md" \
&& grep -q 'USER-CANON-MARKER' "$SANDBOX/data/system-prompt.md" \
&& check "user context dispatched into generated prompt" 0 \
|| check "user context dispatched into generated prompt" 1
else
echo "skip adapter seam cases (docker daemon unavailable)"
fi
# ---------- workspace + capabilities (M5): deterministic mock cases ----------
if docker info >/dev/null 2>&1; then
printf '{"taskVersion":1,"id":"t-ws","prompt":"ignored","workspace":"suitews","capabilities":{"tools":["read","bash"]}}' > "$SANDBOX/ws-task.json"
expect_exit "workspace+tools task runs via mock" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
scripts/run-task.sh run "$SANDBOX/ws-task.json"
WSL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
grep -q '^MOSAIC_WORKSPACE=/var/lib/mosaic/workspaces/suitews$' "$WSL/stderr.txt" 2>/dev/null \
&& grep -q '^MOSAIC_TOOLS=read,bash$' "$WSL/stderr.txt" 2>/dev/null \
&& check "workspace path + tools delivered to adapter" 0 \
|| check "workspace path + tools delivered to adapter" 1
[ -d "$SANDBOX/data/workspaces/suitews" ] \
&& check "persistent workspace created on host" 0 \
|| check "persistent workspace created on host" 1
expect_exit "plain task still runs (no workspace/tools)" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
scripts/run-task.sh run "$SANDBOX/ok.json"
PL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
grep -q '^MOSAIC_WORKSPACE=$' "$PL/stderr.txt" 2>/dev/null \
&& check "workspace var present but empty when absent" 0 || check "workspace var present but empty when absent" 1
grep -q '^MOSAIC_TOOLS=$' "$PL/stderr.txt" 2>/dev/null \
&& check "tools empty when absent" 0 || check "tools empty when absent" 1
printf '{"taskVersion":1,"id":"t-badtool","prompt":"x","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/badtool.json"
expect_exit "unknown tool exits 2" 2 -- $TASK validate "$SANDBOX/badtool.json"
printf '{"taskVersion":1,"id":"t-badws","prompt":"x","workspace":"../escape"}' > "$SANDBOX/badws.json"
expect_exit "workspace traversal exits 2" 2 -- $TASK validate "$SANDBOX/badws.json"
else
echo "skip workspace/capability cases (docker daemon unavailable)"
fi
# ---------- live: real runs (Docker + credentials required) ----------
# On failure, surface the run record + agent stderr BEFORE the sandbox
# cleanup destroys them. Never let a wrong-exit mask the real reason.
dump_latest_run() {
local latest
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
if [ -n "$latest" ]; then
echo "--- latest run evidence: $latest ---" >&2
cat "$latest/result.json" 2>/dev/null >&2
echo "--- stderr.txt (tail) ---" >&2
tail -8 "$latest/stderr.txt" 2>/dev/null >&2
else
echo "--- no run dir was created at all ---" >&2
fi
}
if docker info >/dev/null 2>&1; then
RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l)
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} live hello task succeeds with exact marker"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} live hello task succeeds with exact marker" >&2
dump_latest_run
fi
R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
node -e '
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1);
' "$DATA_ROOT/runs/$R1/result.json"
check "result.json contents are correct" $?
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
RC=$?
WRONG_REASON="$(latest_reason)"
if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} wrong expectExact fails with exit 1 (reason: expect-mismatch)"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
dump_latest_run
fi
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
|| check "each run gets a distinct run dir (no clobber)" 1
COUNT=$($TASK list | wc -l)
[ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1
# session fork (M11): teach in base, fork into child, child recalls;
# ancestor file count must be unchanged by the fork
printf '{"taskVersion":1,"id":"t-fork-base","prompt":"Remember this code word for later: mosaico. Reply with exactly: REMEMBERED","session":"suite-base","expectExact":"REMEMBERED","timeoutSeconds":180}' > "$SANDBOX/fb.json"
expect_exit "fork base: teach succeeds" 0 -- scripts/run-task.sh run "$SANDBOX/fb.json"
BASECOUNT=$(ls "$SANDBOX/data/sessions/suite-base" | wc -l)
printf '{"taskVersion":1,"id":"t-fork-child","prompt":"What code word did I ask you to remember? Reply with only the code word.","session":"suite-child","sessionForkFrom":"suite-base","timeoutSeconds":180}' > "$SANDBOX/fc.json"
expect_exit "fork child recalls ancestor context" 0 -- scripts/run-task.sh run "$SANDBOX/fc.json"
FR="$(ls -dt "$DATA_ROOT/runs"/r-* | head -1)"
node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(String(r.response).toLowerCase().includes("mosaico")?0:1)' "$FR/result.json" \
&& check "forked child recalled ancestor code word" 0 || check "forked child recalled ancestor code word" 1
[ "$(ls "$SANDBOX/data/sessions/suite-base" | wc -l)" -eq "$BASECOUNT" ] \
&& check "ancestor session untouched by fork" 0 || check "ancestor session untouched by fork" 1
[ "$(ls "$SANDBOX/data/sessions/suite-child" | wc -l)" -ge 1 ] \
&& check "child session dir has its own branch file" 0 || check "child session dir has its own branch file" 1
else
echo "skip live task cases (docker unavailable)"
fi
# ---------- onboarding (M16): deterministic, no Docker ----------
ONB="$SANDBOX/data"
cat > "$SANDBOX/onb-config.json" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$ONB","execution":{"backend":"docker","provider":"zai","model":"m"}}
EOF
expect_exit "onboard without name exits 4 (non-interactive)" 4 -- \
env MOSAIC_CONFIG="$SANDBOX/onb-config.json" scripts/onboard.sh --profession x
expect_exit "onboard --name renders profile" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/onb-config.json" scripts/onboard.sh --name "Jason" --timezone "Europe/Berlin"
[ -f "$ONB/user/USER.md" ] \
&& check "profile written" 0 || check "profile written" 1
grep -q "name: Jason" "$ONB/user/USER.md" \
&& grep -q "(not provided)" "$ONB/user/USER.md" \
&& check "canon structure: required filled, optional placeholdered" 0 \
|| check "canon structure: required filled, optional placeholdered" 1
grep -q "## Skillset" "$ONB/user/USER.md" && grep -q "## Pets" "$ONB/user/USER.md" \
&& check "canon sections present" 0 || check "canon sections present" 1
# Live recall: real pi, real phrasing - assert containment, not exactness
printf '{"taskVersion":1,"id":"t-user","prompt":"What is the user name? Reply with only the name.","session":"onb-check","timeoutSeconds":180}' > "$SANDBOX/recall-task.json"
RC=0
scripts/run-task.sh run "$SANDBOX/recall-task.json" >/dev/null 2>&1 || RC=$?
[ "$RC" -eq 0 ] && check "user recall run succeeds" 0 || check "user recall run succeeds (exit $RC)" 1
FR="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)"
RESP="$(node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.response||"")}catch{console.log("")}' "$FR/result.json" 2>/dev/null)"
echo "$RESP" | grep -qi "jason" \
&& check "recalled user name (response: $RESP)" 0 || check "recalled user name (response: $RESP)" 1
# headless run must NOT carry an agent identity section
! grep -q 'AGENT IDENTITY' "$SANDBOX/data/system-prompt.md" 2>/dev/null \
&& check "no agent identity on headless run" 0 || check "no agent identity on headless run" 1
git -C "$SANDBOX/repo" rev-parse >/dev/null 2>&1 || true
unset MOSAIC_CONFIG
echo
echo "selftest: $PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]
Reference in New Issue View Git Blame Copy Permalink