mosaicstack/stack
21
0
Fork 1
Code Issues 338 Pull Requests 13 Actions Packages Projects Releases 15 Wiki Activity

test(adapters): seam selftests — deterministic mock cases + mission injection (#18)

Browse Source 
- test-config: absent adapter defaults to pi; mock validates; unknown
  adapter exits 2; env exports MOSAIC_ADAPTER (24 cases total)
- test-task: mock adapter gate pass/mismatch (no provider needed),
  mismatch reason asserted, unknown adapter fails closed, mission section
  injected into generated prompt asserted by content (24 cases total)
- harness fixes: helpers defined before use; per-case config files (no
  cross-case leakage); newest-run selection for the live case; deduped
  accidentally duplicated live block

Closes #18
This commit is contained in:
jason.woltje
2026-09-02 21:28:57 -05:00
parent 4ebb123ba3
commit 3c1ffd2c2d
3 changed files with 80 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/test-config.sh
+4
View File
@@ -12,6 +12,10 @@ trap 'rm -rf "$SANDBOX"' EXIT
PASS=0
FAIL=0
check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
}
# expect_exit NAME EXPECTED_RC -- command...
expect_exit() {
local name="$1" expected="$2"
scripts/test-task.sh
+57 -2
View File
@@ -30,6 +30,16 @@ check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
}
latest_reason() {
local latest
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
}
check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
}
CONFIG="$SANDBOX/config.json"
DATA_ROOT="$SANDBOX/data"
mkdir -p "$DATA_ROOT"
@@ -92,6 +102,50 @@ $TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
M2=$(stat -c %Y "$SANDBOX/ok.json")
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
if docker info >/dev/null 2>&1; then
good_task "$SANDBOX/ok.json"
mock_config() { # file adapter
cat > "$SANDBOX/$1" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"$2"}}
EOF
}
mock_config mock-adapters.json mock
expect_exit "mock adapter: gate passes on matching mock response" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
scripts/run-task.sh run "$SANDBOX/ok.json"
expect_exit "mock adapter: expect-mismatch recorded" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=SOMETHING_ELSE \
scripts/run-task.sh run "$SANDBOX/ok.json"
[ "$(latest_reason)" = "expect-mismatch" ] \
&& check "mismatch reason recorded" 0 || check "mismatch reason recorded" 1
mock_config bad-adapter.json nonexistent
# The config layer rejects unknown adapters first, so run-task.sh fails
# closed (exit 1) before any container or run record exists.
expect_exit "unknown adapter fails closed" 1 -- \
env MOSAIC_CONFIG="$SANDBOX/bad-adapter.json" scripts/run-task.sh run "$SANDBOX/ok.json"
good_mission "$SANDBOX/m-ok.json"
printf '{"taskVersion":1,"id":"t-mission","prompt":"ignored by mock","mission":"m-ok.json"}' > "$SANDBOX/mission-task.json"
# Dedicated mission with distinctive directives for the injection assertion.
cat > "$SANDBOX/m-seam.json" <<'EOF'
{"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]}
EOF
printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json"
expect_exit "mission task runs via mock adapter" 0 -- \
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
scripts/run-task.sh run "$SANDBOX/seam-task.json"
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
&& grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \
&& check "mission section injected into generated prompt" 0 \
|| check "mission section injected into generated prompt" 1
else
echo "skip adapter seam cases (docker daemon unavailable)"
fi
# ---------- live: real runs (Docker + credentials required) ----------
# On failure, surface the run record + agent stderr BEFORE the sandbox
# cleanup destroys them. Never let a wrong-exit mask the real reason.
@@ -115,14 +169,15 @@ latest_reason() {
}
if docker info >/dev/null 2>&1; then
RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l)
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
PASS=$((PASS+1)); echo "ok live hello task succeeds with exact marker"
else
FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
dump_latest_run
fi
[ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
R1="$(ls "$DATA_ROOT/runs" | head -1)"
R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
node -e '
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));