test(adapters): seam selftests — deterministic mock cases + mission injection (#18)
- test-config: absent adapter defaults to pi; mock validates; unknown adapter exits 2; env exports MOSAIC_ADAPTER (24 cases total) - test-task: mock adapter gate pass/mismatch (no provider needed), mismatch reason asserted, unknown adapter fails closed, mission section injected into generated prompt asserted by content (24 cases total) - harness fixes: helpers defined before use; per-case config files (no cross-case leakage); newest-run selection for the live case; deduped accidentally duplicated live block Closes #18
This commit is contained in:
@@ -101,6 +101,24 @@ scripts/test-release.sh # release selftests
|
|||||||
|
|
||||||
A failed health check never activates; the previously active release remains deployed. Updating the software therefore cannot corrupt the running installation: package beside, gate, then flip. Verified by the update/refusal/rollback drills in BUILD-LOG Phase 7.
|
A failed health check never activates; the previously active release remains deployed. Updating the software therefore cannot corrupt the running installation: package beside, gate, then flip. Verified by the update/refusal/rollback drills in BUILD-LOG Phase 7.
|
||||||
|
|
||||||
|
## Runtime adapters (M4)
|
||||||
|
|
||||||
|
The harness boundary is formalized: everything upstream (config, contracts, missions, tasks, run records) is harness-agnostic; everything inside an adapter belongs to one runtime.
|
||||||
|
|
||||||
|
```text
|
||||||
|
adapters/<name>/adapter.sh env in: MOSAIC_SYSTEM_PROMPT_FILE, MOSAIC_REQUEST,
|
||||||
|
MOSAIC_PROVIDER, MOSAIC_MODEL
|
||||||
|
stdout: response only; stderr: diagnostics
|
||||||
|
```
|
||||||
|
|
||||||
|
- Selection: `execution.adapter` in config.json (optional; `pi` default; allowlist `pi`, `mock`)
|
||||||
|
- `pi` — pinned Pi CLI, noninteractive print mode, ambient discovery off
|
||||||
|
- `mock` — deterministic test adapter; never for real verification
|
||||||
|
- Mission directives have a sanctioned injection point: when a task references a mission, the task runner mounts the run snapshot and the generated prompt gains a `MISSION (runtime)` section (objective + directives) after the four immutable contracts
|
||||||
|
- Adding a harness (Claude, Codex, OpenCode) later means adding one directory — no orchestrator changes
|
||||||
|
|
||||||
|
See `adapters/README.md` for the full contract.
|
||||||
|
|
||||||
See `docs/plans/2026-09-02_atomic-mosaic-foundation.md` for the full plan.
|
See `docs/plans/2026-09-02_atomic-mosaic-foundation.md` for the full plan.
|
||||||
|
|
||||||
Inside the container:
|
Inside the container:
|
||||||
@@ -137,7 +155,7 @@ scripts/verify.sh # full gated test; exit 0 only on exact MOSAIC_HELLO_OK
|
|||||||
scripts/run-task.sh # run a mission/task file (see Missions & tasks)
|
scripts/run-task.sh # run a mission/task file (see Missions & tasks)
|
||||||
scripts/release.sh # package / activate / rollback / status (see Release model)
|
scripts/release.sh # package / activate / rollback / status (see Release model)
|
||||||
scripts/test-config.sh # fast config-layer selftests (no Docker)
|
scripts/test-config.sh # fast config-layer selftests (no Docker)
|
||||||
scripts/test-task.sh # mission/task selftests (schema + live runs)
|
scripts/test-task.sh # mission/task selftests (schema + adapter seam + live runs)
|
||||||
scripts/test-release.sh # release selftests
|
scripts/test-release.sh # release selftests
|
||||||
scripts/reset.sh # delete the configured data root (safety-checked)
|
scripts/reset.sh # delete the configured data root (safety-checked)
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ trap 'rm -rf "$SANDBOX"' EXIT
|
|||||||
PASS=0
|
PASS=0
|
||||||
FAIL=0
|
FAIL=0
|
||||||
|
|
||||||
|
check() {
|
||||||
|
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
|
||||||
|
}
|
||||||
|
|
||||||
# expect_exit NAME EXPECTED_RC -- command...
|
# expect_exit NAME EXPECTED_RC -- command...
|
||||||
expect_exit() {
|
expect_exit() {
|
||||||
local name="$1" expected="$2"
|
local name="$1" expected="$2"
|
||||||
|
|||||||
+57
-2
@@ -30,6 +30,16 @@ check() {
|
|||||||
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
|
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
latest_reason() {
|
||||||
|
local latest
|
||||||
|
latest="$(ls -dt "$SANDBOX/data/runs"/r-* 2>/dev/null | head -1)"
|
||||||
|
[ -n "$latest" ] && node -e 'try{const r=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));console.log(r.reason??"")}catch{console.log("")}' "$latest/result.json" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
check() {
|
||||||
|
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "ok $1"; else FAIL=$((FAIL+1)); echo "FAIL $1"; fi
|
||||||
|
}
|
||||||
|
|
||||||
CONFIG="$SANDBOX/config.json"
|
CONFIG="$SANDBOX/config.json"
|
||||||
DATA_ROOT="$SANDBOX/data"
|
DATA_ROOT="$SANDBOX/data"
|
||||||
mkdir -p "$DATA_ROOT"
|
mkdir -p "$DATA_ROOT"
|
||||||
@@ -92,6 +102,50 @@ $TASK validate "$SANDBOX/ok.json" >/dev/null 2>&1
|
|||||||
M2=$(stat -c %Y "$SANDBOX/ok.json")
|
M2=$(stat -c %Y "$SANDBOX/ok.json")
|
||||||
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
|
[ "$M1" = "$M2" ] && check "validation does not modify the task file" 0 || check "validation does not modify the task file" 1
|
||||||
|
|
||||||
|
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
|
||||||
|
if docker info >/dev/null 2>&1; then
|
||||||
|
good_task "$SANDBOX/ok.json"
|
||||||
|
mock_config() { # file adapter
|
||||||
|
cat > "$SANDBOX/$1" <<EOF
|
||||||
|
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"$2"}}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
mock_config mock-adapters.json mock
|
||||||
|
expect_exit "mock adapter: gate passes on matching mock response" 0 -- \
|
||||||
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK \
|
||||||
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
||||||
|
|
||||||
|
expect_exit "mock adapter: expect-mismatch recorded" 1 -- \
|
||||||
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=SOMETHING_ELSE \
|
||||||
|
scripts/run-task.sh run "$SANDBOX/ok.json"
|
||||||
|
[ "$(latest_reason)" = "expect-mismatch" ] \
|
||||||
|
&& check "mismatch reason recorded" 0 || check "mismatch reason recorded" 1
|
||||||
|
|
||||||
|
mock_config bad-adapter.json nonexistent
|
||||||
|
# The config layer rejects unknown adapters first, so run-task.sh fails
|
||||||
|
# closed (exit 1) before any container or run record exists.
|
||||||
|
expect_exit "unknown adapter fails closed" 1 -- \
|
||||||
|
env MOSAIC_CONFIG="$SANDBOX/bad-adapter.json" scripts/run-task.sh run "$SANDBOX/ok.json"
|
||||||
|
|
||||||
|
good_mission "$SANDBOX/m-ok.json"
|
||||||
|
printf '{"taskVersion":1,"id":"t-mission","prompt":"ignored by mock","mission":"m-ok.json"}' > "$SANDBOX/mission-task.json"
|
||||||
|
# Dedicated mission with distinctive directives for the injection assertion.
|
||||||
|
cat > "$SANDBOX/m-seam.json" <<'EOF'
|
||||||
|
{"missionVersion":1,"id":"m-seam","objective":"Prove the mission injection point.","directives":["Seam directive A.","Seam directive B."]}
|
||||||
|
EOF
|
||||||
|
printf '{"taskVersion":1,"id":"t-mission-seam","prompt":"ignored by mock","mission":"m-seam.json"}' > "$SANDBOX/seam-task.json"
|
||||||
|
expect_exit "mission task runs via mock adapter" 0 -- \
|
||||||
|
env MOSAIC_CONFIG="$SANDBOX/mock-adapters.json" MOSAIC_MOCK_RESPONSE=MOCKED \
|
||||||
|
scripts/run-task.sh run "$SANDBOX/seam-task.json"
|
||||||
|
grep -q 'MISSION (runtime)' "$SANDBOX/data/system-prompt.md" \
|
||||||
|
&& grep -q 'Seam directive A.' "$SANDBOX/data/system-prompt.md" \
|
||||||
|
&& check "mission section injected into generated prompt" 0 \
|
||||||
|
|| check "mission section injected into generated prompt" 1
|
||||||
|
else
|
||||||
|
echo "skip adapter seam cases (docker daemon unavailable)"
|
||||||
|
fi
|
||||||
|
|
||||||
# ---------- live: real runs (Docker + credentials required) ----------
|
# ---------- live: real runs (Docker + credentials required) ----------
|
||||||
# On failure, surface the run record + agent stderr BEFORE the sandbox
|
# On failure, surface the run record + agent stderr BEFORE the sandbox
|
||||||
# cleanup destroys them. Never let a wrong-exit mask the real reason.
|
# cleanup destroys them. Never let a wrong-exit mask the real reason.
|
||||||
@@ -115,14 +169,15 @@ latest_reason() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if docker info >/dev/null 2>&1; then
|
if docker info >/dev/null 2>&1; then
|
||||||
|
RUNS1=$(ls "$DATA_ROOT/runs" 2>/dev/null | wc -l)
|
||||||
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
|
if scripts/run-task.sh run "$SANDBOX/ok.json" >/dev/null 2>&1; then
|
||||||
PASS=$((PASS+1)); echo "ok live hello task succeeds with exact marker"
|
PASS=$((PASS+1)); echo "ok live hello task succeeds with exact marker"
|
||||||
else
|
else
|
||||||
FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
|
FAIL=$((FAIL+1)); echo "FAIL live hello task succeeds with exact marker" >&2
|
||||||
dump_latest_run
|
dump_latest_run
|
||||||
fi
|
fi
|
||||||
[ -f "$DATA_ROOT/runs" ] && RUNS1=$(ls "$DATA_ROOT/runs" | wc -l)
|
|
||||||
R1="$(ls "$DATA_ROOT/runs" | head -1)"
|
R1="$(ls -t "$DATA_ROOT/runs" | head -1)" # newest = the live run above
|
||||||
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
|
[ -f "$DATA_ROOT/runs/$R1/result.json" ] && check "result.json written in run dir" 0 || check "result.json written in run dir" 1
|
||||||
node -e '
|
node -e '
|
||||||
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||||
|
|||||||
Reference in New Issue
Block a user