jason.woltje
d1d7b5598d
Owner decision after live verification of M18: an explicit agents-dir override that cannot resolve the named seat now refuses the launch (exit 4, names the seat and dir) instead of launching seatless and unbounded. Unsetting the override keeps the M13 plain governed TUI. MOSAIC_ROLES_DIR needs no symmetric change - the M18 gate already refuses unresolvable role contracts. Task suite 88 -> 90 (refusal + refusal-names-the-seat). TOOLS.md Agent section documents the refusal. Suites 24/90/14/17 + verify green.
499 lines
30 KiB
Bash
Executable File
499 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Sandboxed selftests for the mission/task layer.
|
|
#
|
|
# Fast cases are validation-only (no Docker, no network). The final cases
|
|
# execute real tasks through the container path and therefore require a
|
|
# working configuration, credentials, and Docker.
|
|
set -uo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
SANDBOX="$(mktemp -d)"
|
|
trap 'rm -rf "$SANDBOX"' EXIT
|
|
|
|
PASS=0
|
|
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
|
|
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
|
|
C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m'
|
|
else
|
|
C_OK=""; C_FAIL=""; C_RESET=""
|
|
fi
|
|
FAIL=0
|
|
|
|
expect_exit() {
|
|
local name="$1" expected="$2"
|
|
shift 3
|
|
local rc
|
|
"$@" >/dev/null 2>&1
|
|
rc=$?
|
|
if [ "$rc" -eq "$expected" ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $name (exit $rc)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $name (exit $rc, expected $expected)"
|
|
fi
|
|
}
|
|
|
|
check() {
|
|
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi
|
|
}
|
|
|
|
latest_reason() {
|
|
local latest
|
|
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
|
|
}
|
|
|
|
CONFIG="$SANDBOX/config.json"
|
|
DATA_ROOT="$SANDBOX/data"
|
|
mkdir -p "$DATA_ROOT"
|
|
cat > "$CONFIG" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$DATA_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash"}}
|
|
EOF
|
|
export MOSAIC_CONFIG="$CONFIG"
|
|
|
|
TASK="node scripts/mosaic-task.mjs"
|
|
|
|
good_task() { # args: file [overrides as jq-less JSON fragments]
|
|
cat > "$1" <<EOF
|
|
{"taskVersion":1,"id":"t-ok","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_HELLO_OK"}
|
|
EOF
|
|
}
|
|
good_mission() {
|
|
cat > "$1" <<'EOF'
|
|
{"missionVersion":1,"id":"m-ok","objective":"Prove the marker path.","directives":["Be terse."]}
|
|
EOF
|
|
}
|
|
|
|
# ---------- fast: schema negatives ----------
|
|
good_task "$SANDBOX/ok.json"
|
|
expect_exit "valid task validates" 0 -- $TASK validate "$SANDBOX/ok.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","extra":1}' > "$SANDBOX/unknownkey.json"
|
|
expect_exit "unknown task key exits 2" 2 -- $TASK validate "$SANDBOX/unknownkey.json"
|
|
|
|
printf '{"taskVersion":2,"id":"t-x","prompt":"hi"}' > "$SANDBOX/badver.json"
|
|
expect_exit "unsupported taskVersion exits 2" 2 -- $TASK validate "$SANDBOX/badver.json"
|
|
|
|
printf '{"taskVersion":1,"id":"BAD ID","prompt":"hi"}' > "$SANDBOX/badid.json"
|
|
expect_exit "invalid task id exits 2" 2 -- $TASK validate "$SANDBOX/badid.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":""}' > "$SANDBOX/emptyprompt.json"
|
|
expect_exit "empty prompt exits 2" 2 -- $TASK validate "$SANDBOX/emptyprompt.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","expectExact":"bad\\u0000nul"}' > "$SANDBOX/badexpect.json"
|
|
expect_exit "NUL in expectExact exits 2" 2 -- $TASK validate "$SANDBOX/badexpect.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","timeoutSeconds":9999}' > "$SANDBOX/badtimeout.json"
|
|
expect_exit "out-of-range timeout exits 2" 2 -- $TASK validate "$SANDBOX/badtimeout.json"
|
|
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"missing.json"}' > "$SANDBOX/missmission.json"
|
|
expect_exit "missing mission file exits 4" 4 -- $TASK validate "$SANDBOX/missmission.json"
|
|
|
|
good_mission "$SANDBOX/m.json"
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"m.json"}' > "$SANDBOX/withmission.json"
|
|
expect_exit "task with valid mission validates" 0 -- $TASK validate "$SANDBOX/withmission.json"
|
|
|
|
printf '{"missionVersion":1,"id":"m-x","objective":"o","extra":1}' > "$SANDBOX/badmission.json"
|
|
printf '{"taskVersion":1,"id":"t-x","prompt":"hi","mission":"badmission.json"}' > "$SANDBOX/withbadmission.json"
|
|
expect_exit "invalid mission exits 2" 2 -- $TASK validate "$SANDBOX/withbadmission.json"
|
|
|
|
expect_exit "validate missing task exits 4" 4 -- $TASK validate "$SANDBOX/nope.json"
|
|
|
|
# validation writes nothing: task dir listing unchanged is implied; check mtime of ok.json
|
|
M1=$(stat -c %Y "$SANDBOX/ok.json"); sleep 1.1
|
|
$TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
|
|
M2=$(stat -c %Y "$SANDBOX/ok.json")
|
|
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
|
|
|
|
# ---------- retention: prune (deterministic, no Docker) ----------
|
|
mkdir -p "$SANDBOX/data"
|
|
cat > "$SANDBOX/prune-config.json" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m"}}
|
|
EOF
|
|
for i in 1 2 3 4 5; do
|
|
D="$SANDBOX/data/runs/r-20260903T0100_0${i}Z-suite00$i"
|
|
mkdir -p "$D"
|
|
printf '{"runVersion":1,"runId":"r-20260903T0100_0%sZ-suite00%s","taskId":"t","status":"succeeded"}' "$i" "$i" > "$D/result.json"
|
|
done
|
|
mkdir -p "$SANDBOX/data/sessions/sentinel" "$SANDBOX/data/workspaces/sentinel"
|
|
|
|
expect_exit "prune dry-run exits 0" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune
|
|
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 5 ] \
|
|
&& check "dry-run deleted nothing" 0 || check "dry-run deleted nothing" 1
|
|
|
|
expect_exit "prune --keep=2 --yes removes oldest" 0 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=2 --yes
|
|
[ "$(ls "$SANDBOX/data/runs" | grep -c '^r-')" -eq 2 ] \
|
|
&& check "kept exactly 2 newest runs" 0 || check "kept exactly 2 newest runs" 1
|
|
NEWEST="r-20260903T0100_05Z-suite005"
|
|
[ -d "$SANDBOX/data/runs/$NEWEST" ] \
|
|
&& check "newest run kept, oldest pruned" 0 || check "newest run kept, oldest pruned" 1
|
|
[ -f "$SANDBOX/data/runs/.pruned.log" ] \
|
|
&& [ "$(grep -c 'pruned' "$SANDBOX/data/runs/.pruned.log")" -eq 3 ] \
|
|
&& check "append-only receipt written (3 entries)" 0 \
|
|
|| check "append-only receipt written (3 entries)" 1
|
|
[ -d "$SANDBOX/data/sessions/sentinel" ] && [ -d "$SANDBOX/data/workspaces/sentinel" ] \
|
|
&& check "sessions/workspaces untouched by prune" 0 \
|
|
|| check "sessions/workspaces untouched by prune" 1
|
|
expect_exit "prune with invalid keep exits 4" 4 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=0 --yes
|
|
|
|
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
|
|
if docker info >/dev/null 2>&1; then
|
|
good_task "$SANDBOX/ok.json"
|
|
mock_config() { # file adapter
|
|
cat > "$SANDBOX/$1" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"$2"}}
|
|
EOF
|
|
}
|
|
|
|
mock_config mock-adapters.json mock
|
|
expect_exit "mock adapter: gate passes on matching mock response" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
|
|
expect_exit "mock adapter: expect-mismatch recorded" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=SOMETHING_ELSE \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
[ "$(latest_reason)" = "expect-mismatch" ] \
|
|
&& check "mismatch reason recorded" 0 || check "mismatch reason recorded" 1
|
|
|
|
mock_config bad-adapter.json nonexistent
|
|
# The config layer rejects unknown adapters first, so run-task.sh fails
|
|
# closed (exit 1) before any container or run record exists.
|
|
expect_exit "unknown adapter fails closed" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/bad-adapter.json" scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
|
|
good_mission "$SANDBOX/m-ok.json"
|
|
printf '{"taskVersion":1,"id":"t-mission","prompt":"ignored by mock","mission":"m-ok.json"}' > "$SANDBOX/mission-task.json"
|
|
# Dedicated mission with distinctive directives for the injection assertion.
|
|
cat > "$SANDBOX/m-seam.json" <<'EOF'
|
|
{"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]}
|
|
EOF
|
|
printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json"
|
|
expect_exit "mission task runs via mock adapter" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/seam-task.json"
|
|
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
|
|
&& grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \
|
|
&& check "mission section injected into generated prompt" 0 \
|
|
|| check "mission section injected into generated prompt" 1
|
|
|
|
# retry lineage + relative mission path resolution
|
|
expect_exit "retry of mission run succeeds" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
node scripts/mosaic-task.mjs retry "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
|
|
RT="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1 | xargs basename)"
|
|
ORIG="$(ls -dt "$SANDBOX/data/runs"/r-* | sed -n 2p | xargs basename)"
|
|
node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(r.retriedFrom===process.argv[2]?0:1)' \
|
|
"$SANDBOX/data/runs/$RT/result.json" "$ORIG" \
|
|
&& check "retriedFrom lineage recorded" 0 || check "retriedFrom lineage recorded" 1
|
|
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
|
|
&& check "mission section present after retry (relative path resolved)" 0 \
|
|
|| check "mission section present after retry (relative path resolved)" 1
|
|
expect_exit "retry of missing run exits 4" 4 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" node scripts/mosaic-task.mjs retry r-missing
|
|
|
|
# skills lifecycle (M17)
|
|
expect_exit "skill install bundled ms-tools" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools
|
|
[ -d "$SANDBOX/data/skills-available/ms-tools" ] \
|
|
&& check "installed to skills-available" 0 || check "installed to skills-available" 1
|
|
expect_exit "double install refuses" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh install ms-tools
|
|
expect_exit "activate enables skill" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh activate ms-tools
|
|
[ -d "$SANDBOX/data/skills-enabled/ms-tools" ] \
|
|
&& check "enabled dir populated" 0 || check "enabled dir populated" 1
|
|
expect_exit "uninstall while enabled refuses" 1 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools
|
|
expect_exit "deactivate moves back to available" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh deactivate ms-tools
|
|
expect_exit "uninstall removes from available" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" scripts/skill.sh uninstall ms-tools
|
|
|
|
# seat skills dispatch (mock evidence)
|
|
mkdir -p "$SANDBOX/agents/skillseat" "$SANDBOX/data/user"
|
|
printf '# User\n\nname: Jason\n' > "$SANDBOX/data/user/USER.md"
|
|
printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md"
|
|
printf '{"agentVersion":1,"name":"skillseat","skills":["ms-tools"]}' > "$SANDBOX/agents/skillseat/agent.json"
|
|
printf '# SOUL - skillseat\n\nMethodical. Verifies before claiming.\n' > "$SANDBOX/agents/skillseat/SOUL.md"
|
|
cat > "$SANDBOX/mock-config.json" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"mock"}}
|
|
EOF
|
|
REL="$(tr -d '[:space:]' < RELEASE)"
|
|
mkdir -p "$SANDBOX/data/state"
|
|
printf '{"pointerVersion":1,"release":"%s","imageTag":"mosaic-poc-agent:0.84.4-r%s","activatedAt":"2026-01-01T00:00:00Z"}\n' "$REL" "$REL" > "$SANDBOX/data/state/active.json"
|
|
scripts/skill.sh install ms-tools >/dev/null 2>&1
|
|
scripts/skill.sh activate ms-tools >/dev/null 2>&1
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
env MOSAIC_AGENTS_DIR="$SANDBOX/agents" \
|
|
scripts/agent.sh skillseat </dev/null >"$SANDBOX/seat-stdout.txt" 2>"$SANDBOX/seat-stderr.txt"
|
|
RC=$?
|
|
if [ "$RC" -eq 0 ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} seat with enabled skill launches"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} seat with enabled skill launches (exit $RC)" >&2
|
|
echo "SEATCASE stderr:" >&2; cat "$SANDBOX/seat-stderr.txt" >&2
|
|
fi
|
|
grep -q '^MOSAIC_SKILLS=/var/lib/mosaic/skills-enabled/ms-tools$' "$SANDBOX/seat-stderr.txt" 2>/dev/null \
|
|
&& check "skill path delivered to adapter" 0 || check "skill path delivered to adapter" 1
|
|
|
|
# seat role ceiling (M18): role contracts cap seat capabilities.
|
|
# Contract validation is CLI-exercised (no docker); the seat wiring gets
|
|
# two mock-adapter launches (narrowing, tool-free) plus one pre-docker
|
|
# refusal (missing contract).
|
|
ROLE_OUT="$(node scripts/mosaic-task.mjs resolve-role roles/researcher.json 2>/dev/null)" \
|
|
&& printf '%s\n' "$ROLE_OUT" | grep -q '^MOSAIC_ROLE_TOOLS=read,grep,find,ls,bash$' \
|
|
&& printf '%s\n' "$ROLE_OUT" | grep -q '^MOSAIC_ROLE_NETWORK=none$' \
|
|
&& check "shipped researcher contract resolves (ceiling + network)" 0 || check "shipped researcher contract resolves (ceiling + network)" 1
|
|
expect_exit "resolve-role refuses a non-role document (conductor policy)" 2 -- \
|
|
node scripts/mosaic-task.mjs resolve-role roles/conductor-policy.json
|
|
mkdir -p "$SANDBOX/roles" "$SANDBOX/agents/roleseat" "$SANDBOX/no-roles"
|
|
printf '{"roleVersion":1,"name":"other","tools":["read"]}' > "$SANDBOX/roles/mismatch.json"
|
|
expect_exit "resolve-role refuses name/filename mismatch" 2 -- \
|
|
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/mismatch.json"
|
|
printf '{"roleVersion":1,"name":"badnet","tools":["read"],"network":"everywhere"}' > "$SANDBOX/roles/badnet.json"
|
|
expect_exit "resolve-role refuses unknown network declaration" 2 -- \
|
|
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/badnet.json"
|
|
printf '{"roleVersion":1,"name":"dupe","tools":["read","read"]}' > "$SANDBOX/roles/dupe.json"
|
|
expect_exit "resolve-role refuses duplicate role tool" 2 -- \
|
|
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/dupe.json"
|
|
printf '{"roleVersion":1,"name":"aliens","tools":["read","render3d"]}' > "$SANDBOX/roles/aliens.json"
|
|
expect_exit "resolve-role refuses unsupported tool" 2 -- \
|
|
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/aliens.json"
|
|
expect_exit "resolve-role refuses missing contract file" 4 -- \
|
|
node scripts/mosaic-task.mjs resolve-role "$SANDBOX/roles/absent.json"
|
|
printf '# SOUL - roleseat\n\nVerifies before claiming.\n' > "$SANDBOX/agents/roleseat/SOUL.md"
|
|
printf '{"agentVersion":1,"name":"roleseat","role":"analyst","capabilities":{"tools":["read","write","bash"]}}' > "$SANDBOX/agents/roleseat/agent.json"
|
|
printf '{"roleVersion":1,"name":"analyst","tools":["read","grep","bash"],"network":"none"}' > "$SANDBOX/roles/analyst.json"
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/roles" \
|
|
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
|
|
RC=$?
|
|
if [ "$RC" -eq 0 ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} seat launches under role ceiling"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} seat launches under role ceiling (exit $RC)" >&2
|
|
echo "ROLESEAT stderr:" >&2; cat "$SANDBOX/roleseat-stderr.txt" >&2
|
|
fi
|
|
grep -q '^MOSAIC_TOOLS=read,bash$' "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
|
|
&& check "role ceiling narrows seat tools (read,write,bash -> read,bash)" 0 || check "role ceiling narrows seat tools (read,write,bash -> read,bash)" 1
|
|
grep -q "role 'analyst' ceiling narrowed tools -> read,bash" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
|
|
&& check "narrowing recorded loudly" 0 || check "narrowing recorded loudly" 1
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/no-roles" \
|
|
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
|
|
RC=$?
|
|
if [ "$RC" -eq 2 ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} missing role contract refuses launch (exit 2)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} missing role contract refuses launch (exit $RC)" >&2
|
|
echo "ROLESEAT stderr:" >&2; cat "$SANDBOX/roleseat-stderr.txt" >&2
|
|
fi
|
|
grep -q "role 'analyst' is declared but has no contract" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
|
|
&& check "missing-contract refusal names the role" 0 || check "missing-contract refusal names the role" 1
|
|
printf '{"agentVersion":1,"name":"roleseat","role":"analyst","capabilities":{"tools":["write","edit"]}}' > "$SANDBOX/agents/roleseat/agent.json"
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
MOSAIC_AGENTS_DIR="$SANDBOX/agents" MOSAIC_ROLES_DIR="$SANDBOX/roles" \
|
|
scripts/agent.sh roleseat </dev/null >"$SANDBOX/roleseat-stdout.txt" 2>"$SANDBOX/roleseat-stderr.txt"
|
|
grep -q '^MOSAIC_TOOLS=$' "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
|
|
&& check "empty ceiling intersection -> tool-free seat" 0 || check "empty ceiling intersection -> tool-free seat" 1
|
|
grep -q "nothing in common -> tool-free seat" "$SANDBOX/roleseat-stderr.txt" 2>/dev/null \
|
|
&& check "tool-free outcome recorded loudly" 0 || check "tool-free outcome recorded loudly" 1
|
|
|
|
# fail-closed seat resolution (#46, owner decision): an explicit
|
|
# agents-dir override that cannot resolve the named seat refuses.
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-config.json" MOSAIC_AGENTS_DIR="$SANDBOX/agents" \
|
|
scripts/agent.sh nobody </dev/null >"$SANDBOX/nobody-stdout.txt" 2>"$SANDBOX/nobody-stderr.txt"
|
|
RC=$?
|
|
if [ "$RC" -eq 4 ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} overridden agents dir without seat definition refuses (exit 4)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} overridden agents dir without seat definition refuses (exit $RC)" >&2
|
|
echo "NOBODY stderr:" >&2; cat "$SANDBOX/nobody-stderr.txt" >&2
|
|
fi
|
|
grep -q "no seat definition for 'nobody'" "$SANDBOX/nobody-stderr.txt" 2>/dev/null \
|
|
&& check "seat-resolution refusal names the seat" 0 || check "seat-resolution refusal names the seat" 1
|
|
|
|
# capability policy (M9): least-privilege intersection
|
|
POL="$SANDBOX/data/workspaces"; mkdir -p "$POL"
|
|
pol_run() { # missionTools(ABSENT|json) taskTools(ABSENT|json) -> stderr MOSAIC_TOOLS value
|
|
if [ "$1" = "ABSENT" ]; then
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o"}' > "$SANDBOX/pol-m.json"
|
|
else
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":[%s]}}' "$1" > "$SANDBOX/pol-m.json"
|
|
fi
|
|
if [ "$2" = "ABSENT" ]; then
|
|
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json"}' > "$SANDBOX/pol-t.json"
|
|
else
|
|
printf '{"taskVersion":1,"id":"t-pol","prompt":"x","mission":"pol-m.json","capabilities":{"tools":[%s]}}' "$2" > "$SANDBOX/pol-t.json"
|
|
fi
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/pol-t.json" >/dev/null 2>&1
|
|
grep -o '^MOSAIC_TOOLS=.*' "$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)/stderr.txt" 2>/dev/null
|
|
}
|
|
[ "$(pol_run '"read","bash"' ABSENT)" = 'MOSAIC_TOOLS=read,bash' ] \
|
|
&& check "policy: mission only -> mission tools" 0 || check "policy: mission only -> mission tools" 1
|
|
[ "$(pol_run ABSENT '"read","bash"')" = 'MOSAIC_TOOLS=read,bash' ] \
|
|
&& check "policy: task only -> task tools" 0 || check "policy: task only -> task tools" 1
|
|
[ "$(pol_run '"read"' '"bash","read"')" = 'MOSAIC_TOOLS=read' ] \
|
|
&& check "policy: both -> intersection (task narrowed)" 0 || check "policy: both -> intersection (task narrowed)" 1
|
|
[ "$(pol_run '"grep"' '"bash","read"')" = 'MOSAIC_TOOLS=' ] \
|
|
&& check "policy: empty intersection -> tool-free" 0 || check "policy: empty intersection -> tool-free" 1
|
|
printf '{"missionVersion":1,"id":"m-pol","objective":"o","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/pol-m.json"
|
|
expect_exit "invalid mission capabilities rejected" 2 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json"
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" $TASK validate "$SANDBOX/pol-t.json"
|
|
|
|
# live user context (M14): dispatched to every launch without rebuild
|
|
mkdir -p "$SANDBOX/data/user"
|
|
printf '\nUSER-CANON-MARKER\n' >> "$SANDBOX/data/user/USER.md"
|
|
expect_exit "task run with user layer present" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
grep -q 'USER CONTEXT: USER.md' "$SANDBOX/data/system-prompt.md" \
|
|
&& grep -q 'USER-CANON-MARKER' "$SANDBOX/data/system-prompt.md" \
|
|
&& check "user context dispatched into generated prompt" 0 \
|
|
|| check "user context dispatched into generated prompt" 1
|
|
else
|
|
echo "skip adapter seam cases (docker daemon unavailable)"
|
|
fi
|
|
|
|
# ---------- workspace + capabilities (M5): deterministic mock cases ----------
|
|
if docker info >/dev/null 2>&1; then
|
|
printf '{"taskVersion":1,"id":"t-ws","prompt":"ignored","workspace":"suitews","capabilities":{"tools":["read","bash"]}}' > "$SANDBOX/ws-task.json"
|
|
expect_exit "workspace+tools task runs via mock" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
|
scripts/run-task.sh run "$SANDBOX/ws-task.json"
|
|
WSL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
grep -q '^MOSAIC_WORKSPACE=/var/lib/mosaic/workspaces/suitews$' "$WSL/stderr.txt" 2>/dev/null \
|
|
&& grep -q '^MOSAIC_TOOLS=read,bash$' "$WSL/stderr.txt" 2>/dev/null \
|
|
&& check "workspace path + tools delivered to adapter" 0 \
|
|
|| check "workspace path + tools delivered to adapter" 1
|
|
[ -d "$SANDBOX/data/workspaces/suitews" ] \
|
|
&& check "persistent workspace created on host" 0 \
|
|
|| check "persistent workspace created on host" 1
|
|
|
|
expect_exit "plain task still runs (no workspace/tools)" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
|
PL="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
grep -q '^MOSAIC_WORKSPACE=$' "$PL/stderr.txt" 2>/dev/null \
|
|
&& check "workspace var present but empty when absent" 0 || check "workspace var present but empty when absent" 1
|
|
grep -q '^MOSAIC_TOOLS=$' "$PL/stderr.txt" 2>/dev/null \
|
|
&& check "tools empty when absent" 0 || check "tools empty when absent" 1
|
|
|
|
printf '{"taskVersion":1,"id":"t-badtool","prompt":"x","capabilities":{"tools":["sudo"]}}' > "$SANDBOX/badtool.json"
|
|
expect_exit "unknown tool exits 2" 2 -- $TASK validate "$SANDBOX/badtool.json"
|
|
printf '{"taskVersion":1,"id":"t-badws","prompt":"x","workspace":"../escape"}' > "$SANDBOX/badws.json"
|
|
expect_exit "workspace traversal exits 2" 2 -- $TASK validate "$SANDBOX/badws.json"
|
|
else
|
|
echo "skip workspace/capability cases (docker daemon unavailable)"
|
|
fi
|
|
|
|
# ---------- live: real runs (Docker + credentials required) ----------
|
|
# On failure, surface the run record + agent stderr BEFORE the sandbox
|
|
# cleanup destroys them. Never let a wrong-exit mask the real reason.
|
|
dump_latest_run() {
|
|
local latest
|
|
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
|
if [ -n "$latest" ]; then
|
|
echo "--- latest run evidence: $latest ---" >&2
|
|
cat "$latest/result.json" 2>/dev/null >&2
|
|
echo "--- stderr.txt (tail) ---" >&2
|
|
tail -8 "$latest/stderr.txt" 2>/dev/null >&2
|
|
else
|
|
echo "--- no run dir was created at all ---" >&2
|
|
fi
|
|
}
|
|
|
|
if docker info >/dev/null 2>&1; then
|
|
RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l)
|
|
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} live hello task succeeds with exact marker"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} live hello task succeeds with exact marker" >&2
|
|
dump_latest_run
|
|
fi
|
|
|
|
R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above
|
|
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
|
|
node -e '
|
|
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
|
process.exit(r.status === "succeeded" && r.response === "MOSAIC_HELLO_OK" && r.expectedExact === "MOSAIC_HELLO_OK" ? 0 : 1);
|
|
' "$DATA_ROOT/runs/$R1/result.json"
|
|
check "result.json contents are correct" $?
|
|
|
|
printf '{"taskVersion":1,"id":"t-wrong","prompt":"Return your startup marker and nothing else.","expectExact":"MOSAIC_NOT_OK"}' > "$SANDBOX/wrong.json"
|
|
scripts/run-task.sh run "$SANDBOX/wrong.json" >/dev/null 2>&1
|
|
RC=$?
|
|
WRONG_REASON="$(latest_reason)"
|
|
if [ "$RC" -eq 1 ] && [ "$WRONG_REASON" = "expect-mismatch" ]; then
|
|
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} wrong expectExact fails with exit 1 (reason: expect-mismatch)"
|
|
else
|
|
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} wrong expectExact (exit $RC, reason: '${WRONG_REASON:-none}')" >&2
|
|
dump_latest_run
|
|
fi
|
|
|
|
RUNS2=$(ls "$DATA_ROOT/runs" | wc -l)
|
|
[ "$RUNS2" -gt "${RUNS1:-0}" ] && check "each run gets a distinct run dir (no clobber)" 0 \
|
|
|| check "each run gets a distinct run dir (no clobber)" 1
|
|
|
|
COUNT=$($TASK list | wc -l)
|
|
[ "$COUNT" -ge 2 ] && check "list shows both runs" 0 || check "list shows both runs" 1
|
|
|
|
# session fork (M11): teach in base, fork into child, child recalls;
|
|
# ancestor file count must be unchanged by the fork
|
|
printf '{"taskVersion":1,"id":"t-fork-base","prompt":"Remember this code word for later: mosaico. Reply with exactly: REMEMBERED","session":"suite-base","expectExact":"REMEMBERED","timeoutSeconds":180}' > "$SANDBOX/fb.json"
|
|
expect_exit "fork base: teach succeeds" 0 -- scripts/run-task.sh run "$SANDBOX/fb.json"
|
|
BASECOUNT=$(ls "$SANDBOX/data/sessions/suite-base" | wc -l)
|
|
printf '{"taskVersion":1,"id":"t-fork-child","prompt":"What code word did I ask you to remember? Reply with only the code word.","session":"suite-child","sessionForkFrom":"suite-base","timeoutSeconds":180}' > "$SANDBOX/fc.json"
|
|
expect_exit "fork child recalls ancestor context" 0 -- scripts/run-task.sh run "$SANDBOX/fc.json"
|
|
FR="$(ls -dt "$DATA_ROOT/runs"/r-* | head -1)"
|
|
node -e 'const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));process.exit(String(r.response).toLowerCase().includes("mosaico")?0:1)' "$FR/result.json" \
|
|
&& check "forked child recalled ancestor code word" 0 || check "forked child recalled ancestor code word" 1
|
|
[ "$(ls "$SANDBOX/data/sessions/suite-base" | wc -l)" -eq "$BASECOUNT" ] \
|
|
&& check "ancestor session untouched by fork" 0 || check "ancestor session untouched by fork" 1
|
|
[ "$(ls "$SANDBOX/data/sessions/suite-child" | wc -l)" -ge 1 ] \
|
|
&& check "child session dir has its own branch file" 0 || check "child session dir has its own branch file" 1
|
|
else
|
|
echo "skip live task cases (docker unavailable)"
|
|
fi
|
|
|
|
# ---------- onboarding (M16): deterministic, no Docker ----------
|
|
ONB="$SANDBOX/data"
|
|
cat > "$SANDBOX/onb-config.json" <<EOF
|
|
{"configVersion":1,"environment":"development","dataRoot":"$ONB","execution":{"backend":"docker","provider":"zai","model":"m"}}
|
|
EOF
|
|
|
|
expect_exit "onboard without name exits 4 (non-interactive)" 4 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/onb-config.json" scripts/onboard.sh --profession x
|
|
expect_exit "onboard --name renders profile" 0 -- \
|
|
env MOSAIC_CONFIG="$SANDBOX/onb-config.json" scripts/onboard.sh --name "Jason" --timezone "Europe/Berlin"
|
|
[ -f "$ONB/user/USER.md" ] \
|
|
&& check "profile written" 0 || check "profile written" 1
|
|
grep -q "name: Jason" "$ONB/user/USER.md" \
|
|
&& grep -q "(not provided)" "$ONB/user/USER.md" \
|
|
&& check "canon structure: required filled, optional placeholdered" 0 \
|
|
|| check "canon structure: required filled, optional placeholdered" 1
|
|
grep -q "## Skillset" "$ONB/user/USER.md" && grep -q "## Pets" "$ONB/user/USER.md" \
|
|
&& check "canon sections present" 0 || check "canon sections present" 1
|
|
|
|
# Live recall: real pi, real phrasing - assert containment, not exactness
|
|
printf '{"taskVersion":1,"id":"t-user","prompt":"What is the user name? Reply with only the name.","session":"onb-check","timeoutSeconds":180}' > "$SANDBOX/recall-task.json"
|
|
RC=0
|
|
scripts/run-task.sh run "$SANDBOX/recall-task.json" >/dev/null 2>&1 || RC=$?
|
|
[ "$RC" -eq 0 ] && check "user recall run succeeds" 0 || check "user recall run succeeds (exit $RC)" 1
|
|
FR="$(ls -dt "$SANDBOX/data/runs"/r-* | head -1)"
|
|
RESP="$(node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.response||"")}catch{console.log("")}' "$FR/result.json" 2>/dev/null)"
|
|
echo "$RESP" | grep -qi "jason" \
|
|
&& check "recalled user name (response: $RESP)" 0 || check "recalled user name (response: $RESP)" 1
|
|
# headless run must NOT carry an agent identity section
|
|
! grep -q 'AGENT IDENTITY' "$SANDBOX/data/system-prompt.md" 2>/dev/null \
|
|
&& check "no agent identity on headless run" 0 || check "no agent identity on headless run" 1
|
|
|
|
git -C "$SANDBOX/repo" rev-parse >/dev/null 2>&1 || true
|
|
unset MOSAIC_CONFIG
|
|
|
|
echo
|
|
echo "selftest: $PASS passed, $FAIL failed"
|
|
[ "$FAIL" -eq 0 ]
|