305 lines
18 KiB
Bash
Executable File
305 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The organizations CLI E2E (business-design r4 section 7), Bootstrap to
|
|
# Console, on a fresh instance `orge2e` in a scratch HOME. A real worker turn
|
|
# runs through the real launcher and sandboxed runner against the
|
|
# deterministic fixture model on loopback; there are no real credentials, no
|
|
# model proxy and no network beyond loopback.
|
|
#
|
|
# Usage: scripts/e2e-orgs-cli.sh
|
|
#
|
|
# Every command runs under `env -i` with a fixed list: PATH, HOME (the
|
|
# scratch HOME), MOSAIC_CONFIG, TMPDIR (the scratch HOME), LANG, and any
|
|
# agent markers the caller carries. No provider credential of the caller
|
|
# reaches the instance. Agent markers are kept, not cleared, so human verbs
|
|
# go through --tester as the broker requires.
|
|
#
|
|
# Each step prints its receipt and read-back, then checks it; a failed check
|
|
# exits nonzero. The trap stops only what this script started: the bus host
|
|
# it spawned and the fixture endpoint it initialised. The scratch HOME is
|
|
# kept for inspection; its path is printed at the start and the end.
|
|
#
|
|
# The bus host's process tree is sampled from /proc during the worker turn;
|
|
# the script fails if any of its sockets has a peer outside loopback, and
|
|
# fails if the sampler did not see the worker's socket to the fixture.
|
|
#
|
|
# Rows the candidate cannot run yet are printed as SKIP with the reason:
|
|
# the dev proxy key-check (S2's tests cover key survival), a second bus
|
|
# beside the first, bus --detach, bus status by organization, and Console.
|
|
set -euo pipefail
|
|
|
|
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
NAME=orge2e
|
|
H="$(mktemp -d /mnt/storage/scratch/tmp/XXXXXX)"
|
|
TEST_AREA="$H/area"
|
|
mkdir -m 700 "$TEST_AREA"
|
|
CONFIG="$H/.config/mosaic-$NAME/config.json"
|
|
BUS_PID=""
|
|
WATCH_PID=""
|
|
FIXTURE_UP=0
|
|
|
|
RUN_ENV=(env -i "PATH=$REPO/scripts:$PATH" "HOME=$H" "MOSAIC_CONFIG=$CONFIG" "TMPDIR=$H" "LANG=C.UTF-8")
|
|
TESTER=()
|
|
for k in CLAUDECODE CLAUDE_CODE_ENTRYPOINT CODEX_THREAD_ID MOSAIC_BUS_CAP MOSAIC_RUN_ID MOSAIC_AGENT_RUN PI_AGENT_DIR; do
|
|
if [ -n "${!k:-}" ]; then RUN_ENV+=("$k=${!k}"); TESTER=(--tester); fi
|
|
done
|
|
|
|
mosaic() { "${RUN_ENV[@]}" "$REPO/scripts/mosaic" "$@"; }
|
|
|
|
cleanup() {
|
|
local rc=$?
|
|
set +e
|
|
if [ -n "$WATCH_PID" ]; then kill -TERM "$WATCH_PID" 2>/dev/null; wait "$WATCH_PID" 2>/dev/null; fi
|
|
if [ -n "$BUS_PID" ] && kill -0 "$BUS_PID" 2>/dev/null; then
|
|
kill -TERM "$BUS_PID"
|
|
for _ in $(seq 1 50); do kill -0 "$BUS_PID" 2>/dev/null || break; sleep 0.2; done
|
|
kill -0 "$BUS_PID" 2>/dev/null && kill -KILL "$BUS_PID"
|
|
echo "cleanup: stopped bus host pid $BUS_PID"
|
|
fi
|
|
if [ "$FIXTURE_UP" = 1 ]; then
|
|
mosaic dev fixture stop --json >"$TEST_AREA/fixture-stop.json" 2>&1 && echo "cleanup: fixture stopped: $(jq -c . "$TEST_AREA/fixture-stop.json")"
|
|
fi
|
|
echo "scratch HOME kept: $H"
|
|
if [ $rc -eq 0 ]; then echo "E2E PASS"; else echo "E2E FAIL (exit $rc)"; fi
|
|
exit $rc
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT TERM
|
|
|
|
STEP=""
|
|
step() { STEP="$1"; printf '\n== %s\n' "$1"; }
|
|
fail() { echo "FAIL [$STEP]: $*" >&2; exit 1; }
|
|
# check FILE JQ-FILTER DESCRIPTION: the filter must yield true.
|
|
check() { jq -e "$2" "$1" >/dev/null || fail "$3 ($2 on $1)"; echo "ok: $3"; }
|
|
# show NAME CMD...: run, keep stdout in $TEST_AREA/NAME, print it.
|
|
show() { local out="$TEST_AREA/$1"; shift; "$@" >"$out" || fail "exit $? from: $*"; cat "$out"; }
|
|
skip() { printf '\n== SKIP %s\n%s\n' "$1" "$2"; }
|
|
|
|
# netwatch ROOT OUT: until ROOT exits, every 5 ms read /proc for ROOT and its
|
|
# descendants, map their socket fds to /proc/net/{tcp,tcp6,udp,udp6} by inode,
|
|
# and write each connected socket once, as "pid comm proto local -> peer", to
|
|
# OUT.loopback (peer on loopback) or OUT.remote (any other peer). OUT.polls
|
|
# counts samples, OUT.pids lists every pid seen in the tree, OUT.sockets every
|
|
# socket inode seen, OUT.fderr fd directories that could not be read (ENOENT
|
|
# is a process that exited mid-sample; EACCES would be a blind spot). Listening and
|
|
# unconnected sockets are skipped. A connection opened and closed between two
|
|
# samples is missed: this is a check with a positive control (the worker's
|
|
# socket to the fixture port must show up), not a proof.
|
|
NETWATCH_JS='
|
|
import fs from "node:fs";
|
|
const [root, out] = process.argv.slice(1);
|
|
const seen = new Set(), pids = new Set(), fderr = new Map(), socks = new Set();
|
|
let polls = 0;
|
|
const v4 = (h) => [3, 2, 1, 0].map((i) => parseInt(h.slice(i * 2, i * 2 + 2), 16)).join(".");
|
|
const addr = (hex) => {
|
|
const [h, port] = hex.split(":");
|
|
const p = parseInt(port, 16);
|
|
if (h.length === 8) return { ip: v4(h), p };
|
|
const words = [0, 1, 2, 3].map((w) => h.slice(w * 8, w * 8 + 8));
|
|
if (words[0] === "00000000" && words[1] === "00000000" && words[2] === "FFFF0000") return { ip: v4(words[3]), p };
|
|
if (h === "00000000000000000000000001000000") return { ip: "::1", p };
|
|
if (/^0+$/.test(h)) return { ip: "::", p };
|
|
return { ip: "[" + words.map((w) => w.match(/../g).reverse().join("")).join("") + "]", p };
|
|
};
|
|
const loop = (ip) => ip.startsWith("127.") || ip === "::1";
|
|
const unset = (ip) => ip === "0.0.0.0" || ip === "::";
|
|
const read = (f) => { try { return fs.readFileSync(f, "utf8"); } catch { return ""; } };
|
|
const tick = () => {
|
|
polls++;
|
|
// Walk down from ROOT through the children list of every thread: cheaper than
|
|
// reading the stat of every process on the host each sample.
|
|
const tree = [], comm = new Map(), queue = [String(root)];
|
|
while (queue.length) {
|
|
const pid = queue.shift();
|
|
let tasks = [];
|
|
try { tasks = fs.readdirSync(`/proc/${pid}/task`); } catch { continue; }
|
|
tree.push(pid);
|
|
const st = read(`/proc/${pid}/stat`);
|
|
comm.set(pid, st.slice(st.indexOf("(") + 1, st.lastIndexOf(")")));
|
|
for (const t of tasks) for (const c of read(`/proc/${pid}/task/${t}/children`).split(" ")) if (c.trim()) queue.push(c.trim());
|
|
}
|
|
const owner = new Map();
|
|
for (const pid of tree) {
|
|
pids.add(pid);
|
|
let fds = [];
|
|
try { fds = fs.readdirSync(`/proc/${pid}/fd`); } catch (e) { fderr.set(`${pid} ${comm.get(pid)}`, e.code); continue; }
|
|
for (const fd of fds) {
|
|
let l = "";
|
|
try { l = fs.readlinkSync(`/proc/${pid}/fd/${fd}`); } catch { continue; }
|
|
const m = /^socket:\[(\d+)\]$/.exec(l);
|
|
if (m) { owner.set(m[1], pid); socks.add(`${pid} ${comm.get(pid)} ${m[1]}`); }
|
|
}
|
|
}
|
|
if (owner.size === 0) return;
|
|
for (const proto of ["tcp", "tcp6", "udp", "udp6"]) {
|
|
for (const line of read(`/proc/net/${proto}`).split("\n").slice(1)) {
|
|
const f = line.trim().split(/\s+/);
|
|
if (f.length < 10 || !owner.has(f[9])) continue;
|
|
const l = addr(f[1]), r = addr(f[2]);
|
|
if (unset(r.ip) && r.p === 0) continue;
|
|
const pid = owner.get(f[9]);
|
|
const text = `${pid} ${comm.get(pid)} ${proto} ${l.ip}:${l.p} -> ${r.ip}:${r.p}`;
|
|
if (seen.has(text)) continue;
|
|
seen.add(text);
|
|
fs.appendFileSync(`${out}.${loop(r.ip) ? "loopback" : "remote"}`, text + "\n");
|
|
}
|
|
}
|
|
};
|
|
const sleep = new Int32Array(new SharedArrayBuffer(4));
|
|
fs.writeFileSync(`${out}.loopback`, ""); fs.writeFileSync(`${out}.remote`, "");
|
|
const done = () => {
|
|
fs.writeFileSync(`${out}.polls`, String(polls));
|
|
fs.writeFileSync(`${out}.pids`, [...pids].join("\n") + "\n");
|
|
fs.writeFileSync(`${out}.fderr`, [...fderr].map(([k, v]) => `${k} ${v}`).join("\n") + (fderr.size ? "\n" : ""));
|
|
fs.writeFileSync(`${out}.sockets`, [...socks].join("\n") + (socks.size ? "\n" : ""));
|
|
};
|
|
process.on("SIGTERM", () => { done(); process.exit(0); });
|
|
while (fs.existsSync(`/proc/${root}`)) { tick(); Atomics.wait(sleep, 0, 0, 5); }
|
|
done();
|
|
'
|
|
netwatch() { node --input-type=module -e "$NETWATCH_JS" "$1" "$2"; }
|
|
|
|
echo "scratch HOME: $H"
|
|
echo "instance: $NAME"
|
|
echo "env passed to every command: ${RUN_ENV[*]:2}" | sed -E 's/(PATH=)[^ ]*/\1.../'
|
|
held="$(env | cut -d= -f1 | /usr/bin/grep -E '^(ANTHROPIC_|OPENAI_|OPENROUTER_|ZAI_|GEMINI_|GOOGLE_API|CLAUDE_CODE_OAUTH)' | tr '\n' ' ' || true)"
|
|
[ -n "$held" ] && echo "caller provider variables not passed (names only): $held"
|
|
|
|
step "Bootstrap"
|
|
[ -e "$H/.config/mosaic-$NAME" ] && fail "instance $NAME already exists"
|
|
"${RUN_ENV[@]}" "$REPO/scripts/bootstrap.sh" --instance "$NAME" | tee "$TEST_AREA/bootstrap.out"
|
|
[ -f "$CONFIG" ] || fail "no config at $CONFIG"
|
|
DATA_ROOT="$(jq -r .dataRoot "$CONFIG")"
|
|
echo "config: $CONFIG, dataRoot: $DATA_ROOT"
|
|
[ "$DATA_ROOT" = "$H/.mosaic-$NAME" ] || fail "data root $DATA_ROOT is not $H/.mosaic-$NAME"
|
|
check "$DATA_ROOT/instance.json" ".instance == \"$NAME\" or .name == \"$NAME\"" "marker names instance $NAME"
|
|
skip "mosaic instance show --json" "no such verb in this candidate; the config and marker above are the read-back"
|
|
|
|
step "Fixture"
|
|
show fixture-init.json mosaic dev fixture init --instance "$NAME" --model deterministic --json
|
|
FIXTURE_UP=1
|
|
check "$TEST_AREA/fixture-init.json" '.receipt == "dev-fixture-init" and .loopback == true and .external == false and .upstream == "none"' "fixture receipt: loopback, no external upstream"
|
|
check "$TEST_AREA/fixture-init.json" '.endpoint | test("^http://127\\.0\\.0\\.1:[0-9]+$")' "endpoint on 127.0.0.1, OS-assigned port"
|
|
check "$TEST_AREA/fixture-init.json" '.principal == "local-dev:local-dev-owner"' "test principal is local-dev"
|
|
show fixture-status.json mosaic dev fixture status --json
|
|
check "$TEST_AREA/fixture-status.json" '.ready == true and .upstream == "none" and .external == false and .health.upstream == "none"' "status ready, upstream none"
|
|
|
|
# The S0 template (--fixture local) has no plugins, launch or tester flag yet.
|
|
# This adds what a worker turn on the fixture needs; it is the only change.
|
|
patch_doc() {
|
|
"${RUN_ENV[@]}" node --input-type=module -e '
|
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
const [, provider, file] = process.argv;
|
|
const { FIXTURE_ENTRY, FIXTURE_KEY_NAME } = await import(provider);
|
|
const doc = JSON.parse(readFileSync(file, "utf8"));
|
|
doc.plugins = { channel: ["cli", "webui"], harness: ["pi"], "model-provider": ["fixture"] };
|
|
doc.launch = { by: "chief", instances: ["assistant"], models: { [FIXTURE_KEY_NAME]: FIXTURE_ENTRY } };
|
|
doc.roles.assistant.vars = { ...(doc.roles.assistant.vars ?? {}), harness: "pi", model: FIXTURE_KEY_NAME, thinking: "off" };
|
|
doc.vars = { ...(doc.vars ?? {}), "bus.tester": true };
|
|
writeFileSync(file, `${JSON.stringify(doc, null, 2)}\n`, { mode: 0o600 });
|
|
' "$REPO/packages/plugins/model-provider/fixture/index.mjs" "$1" || fail "could not patch $1"
|
|
}
|
|
|
|
org_setup() {
|
|
local id="$1" doc="$TEST_AREA/$1.json"
|
|
step "$id document"
|
|
mosaic org template render "$id" --fixture local --output "$doc" | tee "$TEST_AREA/$id-render.out"
|
|
patch_doc "$doc"
|
|
echo "patched: plugins (channel, harness, model-provider), launch, assistant vars, bus.tester"
|
|
check "$doc" ".id == \"$id\" and (.projects.work | type) == \"object\"" "$id document names project work"
|
|
if /usr/bin/grep -Eiq '(api[_-]?key|token|secret|password)"[[:space:]]*:' "$doc"; then fail "$doc carries a secret-looking key"; fi
|
|
echo "ok: no secret-looking keys in $doc"
|
|
mosaic org validate "$doc" | tee "$TEST_AREA/$id-validate.out"
|
|
|
|
step "$id create"
|
|
mosaic org create "$id" --from "$doc" | tee "$TEST_AREA/$id-create.out"
|
|
show "$id-show.json" mosaic org show "$id" --json
|
|
check "$TEST_AREA/$id-show.json" ".id == \"$id\"" "org show reads back $id"
|
|
|
|
step "$id Git"
|
|
mosaic org queue init "$id" --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-queue-init.out"
|
|
mosaic org project repo init "$id" work --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-repo-init.out"
|
|
show "$id-queue.json" mosaic org queue show "$id" --json
|
|
show "$id-repo.json" mosaic org project repo show "$id" work --json
|
|
local root; root="$(jq -r .root "$TEST_AREA/$id-show.json")"
|
|
check "$TEST_AREA/$id-queue.json" ".org == \"$id\" and .mode == \"local-only\" and .riskAccepted == true and .root == \"$root/checkout\"" "$id queue: local-only, risk accepted, root under the org root"
|
|
check "$TEST_AREA/$id-repo.json" ".org == \"$id\" and .project == \"work\" and .mode == \"local-only\" and .riskAccepted == true and .root == \"$root/projects/work\"" "$id work repo: local-only, risk accepted, root derived from $id and work"
|
|
local kind f dir
|
|
for kind in queue repo; do
|
|
f="$TEST_AREA/$id-$kind.json"; dir="$(jq -r .root "$f")"
|
|
[ "$(git -C "$dir" rev-parse HEAD)" = "$(jq -r .commit "$f")" ] || fail "$dir HEAD is not the recorded commit"
|
|
[ -z "$(git -C "$dir" remote)" ] || fail "$dir has a remote"
|
|
echo "ok: $dir is a Git checkout at the recorded commit with no remote"
|
|
done
|
|
}
|
|
|
|
org_setup alpha
|
|
|
|
step "alpha bus"
|
|
# The bus refuses to start without a notifier config; this instance runs without DMs.
|
|
mkdir -p -m 700 "$DATA_ROOT/notify" "$DATA_ROOT/notify/alpha"
|
|
(umask 077; printf '{"notifyVersion": 1, "binding": null}\n' >"$DATA_ROOT/notify/alpha/notify.json")
|
|
echo "wrote $DATA_ROOT/notify/alpha/notify.json (binding null)"
|
|
"${RUN_ENV[@]}" "$REPO/scripts/mosaic" bus start alpha --dev-local-owner >"$TEST_AREA/bus-alpha.out" 2>&1 &
|
|
BUS_PID=$!
|
|
netwatch "$BUS_PID" "$TEST_AREA/netwatch" &
|
|
WATCH_PID=$!
|
|
up=0
|
|
for _ in $(seq 1 100); do
|
|
kill -0 "$BUS_PID" 2>/dev/null || { cat "$TEST_AREA/bus-alpha.out"; fail "bus host exited"; }
|
|
if /usr/bin/grep -q '^bus host up' "$TEST_AREA/bus-alpha.out"; then up=1; break; fi
|
|
sleep 0.2
|
|
done
|
|
cat "$TEST_AREA/bus-alpha.out"
|
|
[ $up = 1 ] || fail "bus host not up after 20s"
|
|
show bus-status-alpha.json mosaic bus status --json
|
|
check "$TEST_AREA/bus-status-alpha.json" ".host.business == \"alpha\" and .host.live == true and .socket != null and .socket != false" "bus status: alpha host live, socket present"
|
|
if /usr/bin/grep -q "$(jq -r .endpoint "$TEST_AREA/fixture-init.json")" "$TEST_AREA/bus-alpha.out"; then echo "note: bus output names the fixture endpoint"; fi
|
|
skip "bus start --detach, bus status alpha" "S1 is not in this candidate; the bus runs in the background of this script and bus status takes no organization"
|
|
|
|
step "alpha worker"
|
|
show project.out mosaic project create work --repo work --business alpha "${TESTER[@]}"
|
|
show thread.out mosaic thread create work trial --business alpha "${TESTER[@]}"
|
|
show talk.out mosaic talk --thread work/trial 'Reply with fixture word' --wait 60 --business alpha "${TESTER[@]}"
|
|
show thread-read.json mosaic thread read work/trial --business alpha --json "${TESTER[@]}"
|
|
/usr/bin/grep -q PINEAPPLE "$TEST_AREA/thread-read.json" || fail "thread read has no fixture reply"
|
|
echo "ok: thread read carries the fixture reply PINEAPPLE"
|
|
show evidence.json mosaic run evidence alpha --thread work/trial --verify --json "${TESTER[@]}"
|
|
check "$TEST_AREA/evidence.json" '.verified == true and (.runs | length) >= 1' "run evidence verified"
|
|
check "$TEST_AREA/evidence.json" '.org.id == "alpha" and (.runs[0].run | test("^r[0-9a-f]{12}$")) and (.runs[0].launch.sha256 | length) == 64' "evidence names org, run and launch line"
|
|
check "$TEST_AREA/evidence.json" '[.runs[0].stderr[] | .sha256 | length == 64] | all' "stderr paths hashed"
|
|
check "$TEST_AREA/evidence.json" '.runs[0].launch.model == "fixture-deterministic" and .runs[0].launch.harness == "pi"' "the launch used pi and the fixture model"
|
|
check "$TEST_AREA/evidence.json" '(.runs[0].snapshot.files | length) >= 1 and ([.runs[0].snapshot.files[].ok] | all)' "bundle digests recomputed and equal"
|
|
show fixture-status-after.json mosaic dev fixture status --json
|
|
check "$TEST_AREA/fixture-status-after.json" '.health.counts.messages >= 1' "the fixture endpoint answered at least one model call"
|
|
|
|
step "alpha bus stop and evidence after"
|
|
kill -TERM "$BUS_PID"
|
|
for _ in $(seq 1 100); do kill -0 "$BUS_PID" 2>/dev/null || break; sleep 0.2; done
|
|
kill -0 "$BUS_PID" 2>/dev/null && fail "bus host pid $BUS_PID still running 20s after SIGTERM"
|
|
wait "$BUS_PID" || true
|
|
BUS_PID=""
|
|
wait "$WATCH_PID" || true
|
|
WATCH_PID=""
|
|
tail -1 "$TEST_AREA/bus-alpha.out"
|
|
echo "netwatch: $(cat "$TEST_AREA/netwatch.polls") samples, $(wc -l <"$TEST_AREA/netwatch.pids") pids seen in the bus host tree, $(sort -u "$TEST_AREA/netwatch.loopback" | wc -l) distinct loopback socket lines, $(sort -u "$TEST_AREA/netwatch.remote" | wc -l) non-loopback"
|
|
/usr/bin/grep -v ' ENOENT$' "$TEST_AREA/netwatch.fderr" && fail "the watcher could not read the fds of a process in the bus host tree"
|
|
[ -s "$TEST_AREA/netwatch.remote" ] && { sort -u "$TEST_AREA/netwatch.remote"; fail "the bus host tree held a socket with a peer outside loopback"; }
|
|
echo "ok: no socket of the bus host tree had a peer outside loopback while sampled"
|
|
FIXTURE_PORT="$(jq -r .port "$TEST_AREA/fixture-init.json")"
|
|
/usr/bin/grep -E -- "-> 127\.0\.0\.1:$FIXTURE_PORT\$" "$TEST_AREA/netwatch.loopback" || fail "positive control: the watcher saw no socket from the bus host tree to the fixture port $FIXTURE_PORT, so its non-loopback result proves nothing"
|
|
echo "ok: positive control, the watcher saw the worker's socket to the fixture port $FIXTURE_PORT"
|
|
THREAD_ID="$(jq -r .thread.id "$TEST_AREA/evidence.json")"
|
|
show evidence-after.json mosaic run evidence alpha --thread "$THREAD_ID" --verify --json "${TESTER[@]}"
|
|
check "$TEST_AREA/evidence-after.json" '.verified == true and (.runs[0].end | length) >= 1' "with the bus stopped, evidence by thread id verifies and has the run's end record"
|
|
check "$TEST_AREA/evidence-after.json" ".runs[0].launch.sha256 == \"$(jq -r '.runs[0].launch.sha256' "$TEST_AREA/evidence.json")\"" "the launch line is unchanged"
|
|
|
|
org_setup beta
|
|
step "beta distinct from alpha"
|
|
check "$TEST_AREA/beta-show.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-show.json")\"" "beta root differs from alpha root"
|
|
check "$TEST_AREA/beta-repo.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-repo.json")\"" "beta work repo root differs from alpha's"
|
|
check "$TEST_AREA/beta-queue.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-queue.json")\"" "beta queue root differs from alpha's"
|
|
|
|
skip "Both buses" "S1 is not in this candidate: the socket is <dataRoot>/bus/broker.sock, one per instance, so beta's bus cannot run beside alpha's"
|
|
skip "dev proxy key-check" "dropped from this E2E by the S3 brief; S2's tests cover key survival after a second bus starts"
|
|
skip "Console" "mosaic console start --detach --port auto is S1 work and is not in this candidate"
|