Verify shared proxy and organization integration
This commit is contained in:
1 parent
62261ca1fc
commit
8f800c1361
5 files changed
+229
-117
No files matched your search
+171
-114
@@ -2,65 +2,47 @@
|
||||
# The organizations CLI E2E (business-design r4 section 7), Bootstrap to
|
||||
# Console, on a fresh instance `orge2e` in a scratch HOME. A real worker turn
|
||||
# runs through the real launcher and sandboxed runner against the
|
||||
# deterministic fixture model on loopback; there are no real credentials, no
|
||||
# model proxy and no network beyond loopback.
|
||||
# deterministic fixture model on loopback. A separate fake-proxy phase checks
|
||||
# shared keys and fencing; all endpoints are loopback and credentials are synthetic.
|
||||
#
|
||||
# Usage: scripts/e2e-orgs-cli.sh
|
||||
#
|
||||
# Every command runs under `env -i` with a fixed list: PATH, HOME (the
|
||||
# scratch HOME), MOSAIC_CONFIG, TMPDIR (the scratch HOME), LANG, and any
|
||||
# agent markers the caller carries. No provider credential of the caller
|
||||
# reaches the instance. Agent markers are kept, not cleared, so human verbs
|
||||
# go through --tester as the broker requires.
|
||||
#
|
||||
# Each step prints its receipt and read-back, then checks it; a failed check
|
||||
# exits nonzero. The trap stops only what this script started: the bus host
|
||||
# it spawned and the fixture endpoint it initialised. The scratch HOME is
|
||||
# kept for inspection; its path is printed at the start and the end.
|
||||
#
|
||||
# The bus host's process tree is sampled from /proc during the worker turn;
|
||||
# the script fails if any of its sockets has a peer outside loopback, and
|
||||
# fails if the sampler did not see the worker's socket to the fixture.
|
||||
#
|
||||
# Rows the candidate cannot run yet are printed as SKIP with the reason:
|
||||
# the dev proxy key-check (S2's tests cover key survival), a second bus
|
||||
# beside the first, bus --detach, bus status by organization, and Console.
|
||||
# Bus, Console and human verbs run through the Lead-owned devstack launcher.
|
||||
# Scratch HOME and test artifacts are retained for inspection. The trap stops
|
||||
# only this run's processes and checks that its devstack units are gone.
|
||||
# A /proc sampler checks worker sockets, with the fixture as a positive control.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
NAME=orge2e
|
||||
H="$(mktemp -d /mnt/storage/scratch/tmp/XXXXXX)"
|
||||
TEST_AREA="$H/area"
|
||||
mkdir -m 700 "$TEST_AREA"
|
||||
CONFIG="$H/.config/mosaic-$NAME/config.json"
|
||||
BUS_PID=""
|
||||
WATCH_PID=""
|
||||
WATCH_PIDS=()
|
||||
FIXTURE_UP=0
|
||||
|
||||
PROXY_PID=''
|
||||
DEVSTACK=/mnt/storage/scratch/mosaic-lead/uidev/org-release/devstack.sh
|
||||
export HOME="$H" MOSAIC_CONFIG="$CONFIG" TMPDIR="$H"
|
||||
RUN_ENV=(env -i "PATH=$REPO/scripts:$PATH" "HOME=$H" "MOSAIC_CONFIG=$CONFIG" "TMPDIR=$H" "LANG=C.UTF-8")
|
||||
TESTER=()
|
||||
for k in CLAUDECODE CLAUDE_CODE_ENTRYPOINT CODEX_THREAD_ID MOSAIC_BUS_CAP MOSAIC_RUN_ID MOSAIC_AGENT_RUN PI_AGENT_DIR; do
|
||||
if [ -n "${!k:-}" ]; then RUN_ENV+=("$k=${!k}"); TESTER=(--tester); fi
|
||||
done
|
||||
|
||||
mosaic() { "${RUN_ENV[@]}" "$REPO/scripts/mosaic" "$@"; }
|
||||
|
||||
stack() { "$DEVSTACK" "$@"; }
|
||||
mosaic() { stack exec -- "$REPO/scripts/mosaic" "$@"; }
|
||||
fixture() { "${RUN_ENV[@]}" "$REPO/scripts/mosaic" dev fixture "$@"; }
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
if [ -n "$WATCH_PID" ]; then kill -TERM "$WATCH_PID" 2>/dev/null; wait "$WATCH_PID" 2>/dev/null; fi
|
||||
if [ -n "$BUS_PID" ] && kill -0 "$BUS_PID" 2>/dev/null; then
|
||||
kill -TERM "$BUS_PID"
|
||||
for _ in $(seq 1 50); do kill -0 "$BUS_PID" 2>/dev/null || break; sleep 0.2; done
|
||||
kill -0 "$BUS_PID" 2>/dev/null && kill -KILL "$BUS_PID"
|
||||
echo "cleanup: stopped bus host pid $BUS_PID"
|
||||
fi
|
||||
if [ "$FIXTURE_UP" = 1 ]; then
|
||||
mosaic dev fixture stop --json >"$TEST_AREA/fixture-stop.json" 2>&1 && echo "cleanup: fixture stopped: $(jq -c . "$TEST_AREA/fixture-stop.json")"
|
||||
fi
|
||||
for pid in "${WATCH_PIDS[@]}"; do kill -TERM "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; done
|
||||
for name in alpha beta gamma; do mosaic bus stop "$name" >/dev/null 2>&1; done
|
||||
mosaic console stop >/dev/null 2>&1
|
||||
stack stop >/dev/null 2>&1
|
||||
if [ -n "$PROXY_PID" ]; then kill -TERM "$PROXY_PID" 2>/dev/null || rc=1; wait "$PROXY_PID" 2>/dev/null || rc=1; fi
|
||||
if [ "$FIXTURE_UP" = 1 ]; then fixture stop --json >"$TEST_AREA/fixture-stop.json" 2>&1 || rc=1; fi
|
||||
stack list >"$TEST_AREA/devstack-final.txt" 2>&1 || rc=1
|
||||
if [ -s "$TEST_AREA/devstack-final.txt" ]; then cat "$TEST_AREA/devstack-final.txt"; rc=1; fi
|
||||
echo "scratch HOME kept: $H"
|
||||
if [ $rc -eq 0 ]; then echo "E2E PASS"; else echo "E2E FAIL (exit $rc)"; fi
|
||||
exit $rc
|
||||
if [ "$rc" -eq 0 ]; then echo "E2E PASS"; else echo "E2E FAIL (exit $rc)"; fi
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
@@ -72,7 +54,6 @@ fail() { echo "FAIL [$STEP]: $*" >&2; exit 1; }
|
||||
check() { jq -e "$2" "$1" >/dev/null || fail "$3 ($2 on $1)"; echo "ok: $3"; }
|
||||
# show NAME CMD...: run, keep stdout in $TEST_AREA/NAME, print it.
|
||||
show() { local out="$TEST_AREA/$1"; shift; "$@" >"$out" || fail "exit $? from: $*"; cat "$out"; }
|
||||
skip() { printf '\n== SKIP %s\n%s\n' "$1" "$2"; }
|
||||
|
||||
# netwatch ROOT OUT: until ROOT exits, every 5 ms read /proc for ROOT and its
|
||||
# descendants, map their socket fds to /proc/net/{tcp,tcp6,udp,udp6} by inode,
|
||||
@@ -144,7 +125,7 @@ const tick = () => {
|
||||
}
|
||||
}
|
||||
};
|
||||
const sleep = new Int32Array(new SharedArrayBuffer(4));
|
||||
|
||||
fs.writeFileSync(`${out}.loopback`, ""); fs.writeFileSync(`${out}.remote`, "");
|
||||
const done = () => {
|
||||
fs.writeFileSync(`${out}.polls`, String(polls));
|
||||
@@ -153,10 +134,10 @@ const done = () => {
|
||||
fs.writeFileSync(`${out}.sockets`, [...socks].join("\n") + (socks.size ? "\n" : ""));
|
||||
};
|
||||
process.on("SIGTERM", () => { done(); process.exit(0); });
|
||||
while (fs.existsSync(`/proc/${root}`)) { tick(); Atomics.wait(sleep, 0, 0, 5); }
|
||||
while (fs.existsSync(`/proc/${root}`)) { tick(); await new Promise(resolve => setTimeout(resolve, 5)); }
|
||||
done();
|
||||
'
|
||||
netwatch() { node --input-type=module -e "$NETWATCH_JS" "$1" "$2"; }
|
||||
netwatch() { exec "${RUN_ENV[@]}" node --input-type=module -e "$NETWATCH_JS" "$1" "$2"; }
|
||||
|
||||
echo "scratch HOME: $H"
|
||||
echo "instance: $NAME"
|
||||
@@ -172,15 +153,15 @@ DATA_ROOT="$(jq -r .dataRoot "$CONFIG")"
|
||||
echo "config: $CONFIG, dataRoot: $DATA_ROOT"
|
||||
[ "$DATA_ROOT" = "$H/.mosaic-$NAME" ] || fail "data root $DATA_ROOT is not $H/.mosaic-$NAME"
|
||||
check "$DATA_ROOT/instance.json" ".instance == \"$NAME\" or .name == \"$NAME\"" "marker names instance $NAME"
|
||||
skip "mosaic instance show --json" "no such verb in this candidate; the config and marker above are the read-back"
|
||||
echo "ok: instance identity read from config and marker"
|
||||
|
||||
step "Fixture"
|
||||
show fixture-init.json mosaic dev fixture init --instance "$NAME" --model deterministic --json
|
||||
show fixture-init.json fixture init --instance "$NAME" --model deterministic --json
|
||||
FIXTURE_UP=1
|
||||
check "$TEST_AREA/fixture-init.json" '.receipt == "dev-fixture-init" and .loopback == true and .external == false and .upstream == "none"' "fixture receipt: loopback, no external upstream"
|
||||
check "$TEST_AREA/fixture-init.json" '.endpoint | test("^http://127\\.0\\.0\\.1:[0-9]+$")' "endpoint on 127.0.0.1, OS-assigned port"
|
||||
check "$TEST_AREA/fixture-init.json" '.principal == "local-dev:local-dev-owner"' "test principal is local-dev"
|
||||
show fixture-status.json mosaic dev fixture status --json
|
||||
show fixture-status.json fixture status --json
|
||||
check "$TEST_AREA/fixture-status.json" '.ready == true and .upstream == "none" and .external == false and .health.upstream == "none"' "status ready, upstream none"
|
||||
|
||||
org_setup() {
|
||||
@@ -197,6 +178,9 @@ org_setup() {
|
||||
show "$id-show.json" mosaic org show "$id" --json
|
||||
check "$TEST_AREA/$id-show.json" ".id == \"$id\"" "org show reads back $id"
|
||||
|
||||
check "$TEST_AREA/$id-create.out" '.notifications == "none"' "$id receipt records notifications: none"
|
||||
check "$(jq -r .root "$TEST_AREA/$id-show.json")/notify/$id/notify.json" '.notifyVersion == 1 and .binding == null' "$id notifier records explicit null"
|
||||
|
||||
step "$id Git"
|
||||
mosaic org queue init "$id" --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-queue-init.out"
|
||||
mosaic org project repo init "$id" work --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-repo-init.out"
|
||||
@@ -214,70 +198,143 @@ org_setup() {
|
||||
done
|
||||
}
|
||||
|
||||
# A unit owns each detached child until its matching CLI stop completes.
|
||||
cat >"$TEST_AREA/unit.sh" <<'UNIT'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
kind=$1; name=$2; out=$3
|
||||
if [ "$kind" = bus ]; then
|
||||
"$REPO_ROOT/scripts/mosaic" bus start "$name" --dev-local-owner --detach >"$out"
|
||||
else
|
||||
"$REPO_ROOT/scripts/mosaic" console start --dev-local-owner --detach --port auto >"$out"
|
||||
fi
|
||||
pid=$(jq -r .pid "$out")
|
||||
while kill -0 "$pid" 2>/dev/null; do sleep 0.2; done
|
||||
UNIT
|
||||
chmod 700 "$TEST_AREA/unit.sh"
|
||||
# devstack passes its own small environment, so put the checkout path in the
|
||||
# unit script itself rather than depending on a caller's environment.
|
||||
sed -i "s|\$REPO_ROOT|$REPO|g" "$TEST_AREA/unit.sh"
|
||||
start_unit() {
|
||||
local name=$1 kind=$2 org=$3 out="$TEST_AREA/$1-start.json"
|
||||
rm -f "$out"
|
||||
stack run "$name" -- "$TEST_AREA/unit.sh" "$kind" "$org" "$out"
|
||||
for _ in $(seq 1 150); do [ -s "$out" ] && jq -e '.pid > 1' "$out" >/dev/null 2>&1 && return; sleep 0.2; done
|
||||
stack logs "$name"; fail "$name did not start"
|
||||
}
|
||||
|
||||
org_setup alpha
|
||||
|
||||
step "alpha bus"
|
||||
check "$(jq -r .root "$TEST_AREA/alpha-show.json")/notify/alpha/notify.json" '.notifyVersion == 1 and .binding == null' "alpha records notifications: none"
|
||||
"${RUN_ENV[@]}" "$REPO/scripts/mosaic" bus start alpha --dev-local-owner >"$TEST_AREA/bus-alpha.out" 2>&1 &
|
||||
BUS_PID=$!
|
||||
netwatch "$BUS_PID" "$TEST_AREA/netwatch" &
|
||||
WATCH_PID=$!
|
||||
up=0
|
||||
for _ in $(seq 1 100); do
|
||||
kill -0 "$BUS_PID" 2>/dev/null || { cat "$TEST_AREA/bus-alpha.out"; fail "bus host exited"; }
|
||||
if /usr/bin/grep -q '^bus host up' "$TEST_AREA/bus-alpha.out"; then up=1; break; fi
|
||||
sleep 0.2
|
||||
done
|
||||
cat "$TEST_AREA/bus-alpha.out"
|
||||
[ $up = 1 ] || fail "bus host not up after 20s"
|
||||
show bus-status-alpha.json mosaic bus status --json
|
||||
check "$TEST_AREA/bus-status-alpha.json" ".host.business == \"alpha\" and .host.live == true and .socket != null and .socket != false" "bus status: alpha host live, socket present"
|
||||
if /usr/bin/grep -q "$(jq -r .endpoint "$TEST_AREA/fixture-init.json")" "$TEST_AREA/bus-alpha.out"; then echo "note: bus output names the fixture endpoint"; fi
|
||||
skip "bus start --detach, bus status alpha" "S1 is not in this candidate; the bus runs in the background of this script and bus status takes no organization"
|
||||
|
||||
step "alpha worker"
|
||||
show project.out mosaic project create work --repo work --business alpha "${TESTER[@]}"
|
||||
show thread.out mosaic thread create work trial --business alpha "${TESTER[@]}"
|
||||
show talk.out mosaic talk --thread work/trial 'Reply with fixture word' --wait 60 --business alpha "${TESTER[@]}"
|
||||
show thread-read.json mosaic thread read work/trial --business alpha --json "${TESTER[@]}"
|
||||
/usr/bin/grep -q PINEAPPLE "$TEST_AREA/thread-read.json" || fail "thread read has no fixture reply"
|
||||
echo "ok: thread read carries the fixture reply PINEAPPLE"
|
||||
show evidence.json mosaic run evidence alpha --thread work/trial --verify --json "${TESTER[@]}"
|
||||
check "$TEST_AREA/evidence.json" '.verified == true and (.runs | length) >= 1' "run evidence verified"
|
||||
check "$TEST_AREA/evidence.json" '.org.id == "alpha" and (.runs[0].run | test("^r[0-9a-f]{12}$")) and (.runs[0].launch.sha256 | length) == 64' "evidence names org, run and launch line"
|
||||
check "$TEST_AREA/evidence.json" '[.runs[0].stderr[] | .sha256 | length == 64] | all' "stderr paths hashed"
|
||||
check "$TEST_AREA/evidence.json" '.runs[0].launch.model == "fixture-deterministic" and .runs[0].launch.harness == "pi"' "the launch used pi and the fixture model"
|
||||
check "$TEST_AREA/evidence.json" '(.runs[0].snapshot.files | length) >= 1 and ([.runs[0].snapshot.files[].ok] | all)' "bundle digests recomputed and equal"
|
||||
show fixture-status-after.json mosaic dev fixture status --json
|
||||
check "$TEST_AREA/fixture-status-after.json" '.health.counts.messages >= 1' "the fixture endpoint answered at least one model call"
|
||||
|
||||
step "alpha bus stop and evidence after"
|
||||
kill -TERM "$BUS_PID"
|
||||
for _ in $(seq 1 100); do kill -0 "$BUS_PID" 2>/dev/null || break; sleep 0.2; done
|
||||
kill -0 "$BUS_PID" 2>/dev/null && fail "bus host pid $BUS_PID still running 20s after SIGTERM"
|
||||
wait "$BUS_PID" || true
|
||||
BUS_PID=""
|
||||
wait "$WATCH_PID" || true
|
||||
WATCH_PID=""
|
||||
tail -1 "$TEST_AREA/bus-alpha.out"
|
||||
echo "netwatch: $(cat "$TEST_AREA/netwatch.polls") samples, $(wc -l <"$TEST_AREA/netwatch.pids") pids seen in the bus host tree, $(sort -u "$TEST_AREA/netwatch.loopback" | wc -l) distinct loopback socket lines, $(sort -u "$TEST_AREA/netwatch.remote" | wc -l) non-loopback"
|
||||
/usr/bin/grep -v ' ENOENT$' "$TEST_AREA/netwatch.fderr" && fail "the watcher could not read the fds of a process in the bus host tree"
|
||||
[ -s "$TEST_AREA/netwatch.remote" ] && { sort -u "$TEST_AREA/netwatch.remote"; fail "the bus host tree held a socket with a peer outside loopback"; }
|
||||
echo "ok: no socket of the bus host tree had a peer outside loopback while sampled"
|
||||
FIXTURE_PORT="$(jq -r .port "$TEST_AREA/fixture-init.json")"
|
||||
/usr/bin/grep -E -- "-> 127\.0\.0\.1:$FIXTURE_PORT\$" "$TEST_AREA/netwatch.loopback" || fail "positive control: the watcher saw no socket from the bus host tree to the fixture port $FIXTURE_PORT, so its non-loopback result proves nothing"
|
||||
echo "ok: positive control, the watcher saw the worker's socket to the fixture port $FIXTURE_PORT"
|
||||
THREAD_ID="$(jq -r .thread.id "$TEST_AREA/evidence.json")"
|
||||
show evidence-after.json mosaic run evidence alpha --thread "$THREAD_ID" --verify --json "${TESTER[@]}"
|
||||
check "$TEST_AREA/evidence-after.json" '.verified == true and (.runs[0].end | length) >= 1' "with the bus stopped, evidence by thread id verifies and has the run's end record"
|
||||
check "$TEST_AREA/evidence-after.json" ".runs[0].launch.sha256 == \"$(jq -r '.runs[0].launch.sha256' "$TEST_AREA/evidence.json")\"" "the launch line is unchanged"
|
||||
|
||||
org_setup beta
|
||||
step "beta distinct from alpha"
|
||||
check "$TEST_AREA/beta-show.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-show.json")\"" "beta root differs from alpha root"
|
||||
check "$TEST_AREA/beta-repo.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-repo.json")\"" "beta work repo root differs from alpha's"
|
||||
check "$TEST_AREA/beta-queue.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-queue.json")\"" "beta queue root differs from alpha's"
|
||||
step "distinct organization roots"
|
||||
check "$TEST_AREA/beta-show.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-show.json")\"" "beta root differs from alpha"
|
||||
check "$TEST_AREA/beta-repo.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-repo.json")\"" "beta project root differs from alpha"
|
||||
start_unit alpha bus alpha
|
||||
start_unit beta bus beta
|
||||
for id in alpha beta; do
|
||||
show "$id-status.json" mosaic bus status "$id" --json
|
||||
check "$TEST_AREA/$id-status.json" ".org == \"$id\" and .instance == \"$NAME\" and .healthy == true and .pid > 1" "$id bus identity and health"
|
||||
pid=$(jq -r .pid "$TEST_AREA/$id-status.json")
|
||||
netwatch "$pid" "$TEST_AREA/netwatch-$id" & WATCH_PIDS+=("$!")
|
||||
done
|
||||
check "$TEST_AREA/beta-status.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-status.json")\" and .socket != \"$(jq -r .socket "$TEST_AREA/alpha-status.json")\"" "both buses have distinct roots and sockets"
|
||||
start_unit console console console
|
||||
check "$TEST_AREA/console-start.json" '.pid > 1 and (.url | test("^http://127\\.0\\.0\\.1:[0-9]+"))' "Console started on loopback"
|
||||
|
||||
skip "Both buses" "S1 is not in this candidate: the socket is <dataRoot>/bus/broker.sock, one per instance, so beta's bus cannot run beside alpha's"
|
||||
skip "dev proxy key-check" "dropped from this E2E by the S3 brief; S2's tests cover key survival after a second bus starts"
|
||||
skip "Console" "mosaic console start --detach --port auto is S1 work and is not in this candidate"
|
||||
worker() {
|
||||
local id=$1
|
||||
step "$id worker"
|
||||
show "$id-project.out" mosaic project create work --repo work --org "$id"
|
||||
show "$id-thread.out" mosaic thread create work trial --org "$id"
|
||||
show "$id-talk.out" mosaic talk --thread work/trial 'Reply with fixture word' --wait 60 --org "$id"
|
||||
show "$id-thread-read.json" mosaic thread read work/trial --org "$id" --json
|
||||
/usr/bin/grep -q PINEAPPLE "$TEST_AREA/$id-thread-read.json" || fail "$id thread has no fixture reply"
|
||||
show "$id-evidence.json" mosaic run evidence "$id" --thread work/trial --verify --json
|
||||
check "$TEST_AREA/$id-evidence.json" ".verified == true and .org.id == \"$id\" and (.runs | length) >= 1" "$id evidence verified"
|
||||
check "$TEST_AREA/$id-evidence.json" '.runs[0].launch.model == "fixture-deterministic" and .runs[0].launch.harness == "pi" and ([.runs[0].snapshot.files[].ok] | all)' "$id fixture launch and bundle hashes"
|
||||
}
|
||||
worker alpha
|
||||
worker beta
|
||||
show fixture-status-after.json fixture status --json
|
||||
check "$TEST_AREA/fixture-status-after.json" '.health.counts.messages >= 2' "fixture answered both workers"
|
||||
|
||||
step "clean stop and restart alpha"
|
||||
show alpha-stop.json mosaic bus stop alpha
|
||||
check "$TEST_AREA/alpha-stop.json" '.stopped == true' "alpha stopped cleanly"
|
||||
wait "${WATCH_PIDS[0]}" || true
|
||||
start_unit alpha bus alpha
|
||||
show alpha-restarted.json mosaic bus status alpha --json
|
||||
check "$TEST_AREA/alpha-restarted.json" ".healthy == true and .pid != $(jq -r .pid "$TEST_AREA/alpha-status.json")" "alpha restarted under a new host"
|
||||
show beta-still.json mosaic bus status beta --json
|
||||
check "$TEST_AREA/beta-still.json" ".healthy == true and .pid == $(jq -r .pid "$TEST_AREA/beta-status.json")" "beta stayed healthy"
|
||||
for id in alpha beta; do
|
||||
[ "$id" = beta ] && { kill -TERM "${WATCH_PIDS[1]}"; wait "${WATCH_PIDS[1]}" || true; }
|
||||
watch="$TEST_AREA/netwatch-$id"
|
||||
[ -f "$watch.fderr" ] || fail "$id sampler did not finish"
|
||||
/usr/bin/grep -v ' ENOENT$' "$watch.fderr" && fail "$id sampler could not read process fds"
|
||||
[ ! -s "$watch.remote" ] || fail "$id sampler saw non-loopback peer"
|
||||
port=$(jq -r .port "$TEST_AREA/fixture-init.json")
|
||||
/usr/bin/grep -Eq -- "-> 127\\.0\\.0\\.1:$port\$" "$watch.loopback" || fail "$id sampler missed fixture positive control"
|
||||
echo "ok: $id socket sampler saw fixture and no remote peers"
|
||||
done
|
||||
WATCH_PIDS=()
|
||||
|
||||
step "shared fake proxy"
|
||||
# The model fixture and shared proxy are intentionally separate phases.
|
||||
for id in alpha beta; do mosaic bus stop "$id" >/dev/null; done
|
||||
mosaic console stop >/dev/null
|
||||
stack stop
|
||||
fixture stop --json >"$TEST_AREA/fixture-stop.json"
|
||||
FIXTURE_UP=0
|
||||
start_proxy() {
|
||||
rm -f "$TEST_AREA/proxy-ready.json"
|
||||
"${RUN_ENV[@]}" node "$REPO/scripts/e2e-orgs-proxy.mjs" "$TEST_AREA" "${1:-0}" >"$TEST_AREA/proxy.log" 2>&1 & PROXY_PID=$!
|
||||
for _ in $(seq 1 100); do [ -s "$TEST_AREA/proxy-ready.json" ] && return; kill -0 "$PROXY_PID" 2>/dev/null || { cat "$TEST_AREA/proxy.log"; fail "fake proxy stopped"; }; sleep 0.1; done
|
||||
fail "fake proxy did not start"
|
||||
}
|
||||
start_proxy
|
||||
PROXY_PORT=$(jq -r .port "$TEST_AREA/proxy-ready.json")
|
||||
mkdir -m 700 "$DATA_ROOT/proxy"
|
||||
mkdir -p -m 700 "$(dirname "$CONFIG")/secrets"
|
||||
printf '%s' fixture-management >"$(dirname "$CONFIG")/secrets/proxy-management.key"
|
||||
chmod 600 "$(dirname "$CONFIG")/secrets/proxy-management.key"
|
||||
PROXY_VERSION=$(jq -r .version "$REPO/packages/model-proxy/PIN")
|
||||
jq -n --argjson port "$PROXY_PORT" --arg binary "$DATA_ROOT/opt/cliproxyapi/$PROXY_VERSION/cli-proxy-api" '{port:$port,binary:$binary}' >"$(dirname "$CONFIG")/proxy.json"
|
||||
chmod 600 "$(dirname "$CONFIG")/proxy.json"
|
||||
start_unit alpha bus alpha
|
||||
show alpha-proxy-status.json mosaic bus status alpha --json
|
||||
check "$TEST_AREA/alpha-proxy-status.json" '.healthy == true' 'alpha bus with shared proxy is healthy'
|
||||
show alpha-proxy-thread.out mosaic thread create work proxytrial --org alpha
|
||||
show alpha-proxy-talk.out mosaic talk --thread work/proxytrial 'Reply with fixture word' --wait 60 --org alpha
|
||||
show alpha-proxy-read.json mosaic thread read work/proxytrial --org alpha --json
|
||||
/usr/bin/grep -q PINEAPPLE "$TEST_AREA/alpha-proxy-read.json" || fail "alpha proxy worker did not answer"
|
||||
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-before.json"
|
||||
check "$TEST_AREA/key-before.json" '.accepted == true and .alphaKeys == 1' 'alpha current proxy key is accepted'
|
||||
start_unit beta bus beta
|
||||
show beta-proxy-status.json mosaic bus status beta --json
|
||||
check "$TEST_AREA/beta-proxy-status.json" '.healthy == true' 'beta bus with shared proxy is healthy'
|
||||
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-after.json"
|
||||
check "$TEST_AREA/key-after.json" ".accepted == true and .betaSentinels == 1 and .keyId == \"$(jq -r .keyId "$TEST_AREA/key-before.json")\"" 'alpha current key survives beta registration'
|
||||
org_setup gamma
|
||||
: >"$TEST_AREA/delay-put"
|
||||
start_unit gamma bus gamma
|
||||
show fence-set.json mosaic proxy fence status --json
|
||||
check "$TEST_AREA/fence-set.json" '.uncertain.org == "gamma" and .uncertain.op == "sentinel.register"' 'timed-out PUT leaves durable fence'
|
||||
show alpha-fenced-thread.out mosaic thread create work fenced --org alpha
|
||||
show alpha-fenced-talk.out mosaic talk --thread work/fenced 'This launch must be fenced' --wait 0 --org alpha
|
||||
alpha_log="$(jq -r .root "$TEST_AREA/alpha-show.json")/launches/alpha.jsonl"
|
||||
for _ in $(seq 1 100); do /usr/bin/grep -q '"code":"proxy-uncertain"' "$alpha_log" && break; sleep 0.1; done
|
||||
/usr/bin/grep -q '"code":"proxy-uncertain"' "$alpha_log" || fail 'alpha launch did not refuse the proxy fence'
|
||||
echo 'ok: alpha proxy-dependent launch refused while fenced'
|
||||
if mosaic proxy fence clear >"$TEST_AREA/fence-refusal.out" 2>&1; then fail 'fence cleared without restart statement'; fi
|
||||
/usr/bin/grep -q 'proxy-restart-required' "$TEST_AREA/fence-refusal.out" || fail 'expected restart-required refusal'
|
||||
kill -TERM "$PROXY_PID"; wait "$PROXY_PID"; PROXY_PID=''
|
||||
rm "$TEST_AREA/delay-put"
|
||||
start_proxy "$PROXY_PORT"
|
||||
show fence-clear.json mosaic proxy fence clear --proxy-restarted
|
||||
check "$TEST_AREA/fence-clear.json" '.cleared == true' 'fence cleared after proxy restart'
|
||||
show fence-final.json mosaic proxy fence status --json
|
||||
check "$TEST_AREA/fence-final.json" '.uncertain == null' 'fence status clear'
|
||||
mosaic bus stop gamma >/dev/null
|
||||
start_unit gamma bus gamma
|
||||
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-after-recovery.json"
|
||||
check "$TEST_AREA/key-after-recovery.json" ".accepted == true and .betaSentinels == 1 and .gammaSentinels == 1 and .keyId == \"$(jq -r .keyId "$TEST_AREA/key-before.json")\"" 'recovery registers gamma and preserves alpha key and beta sentinel'
|
||||
@@ -0,0 +1,49 @@
|
||||
// Loopback-only synthetic proxy for the organizations integration E2E.
|
||||
import http from 'node:http';
|
||||
import fs from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
const [area, portArg = '0'] = process.argv.slice(2);
|
||||
const forbidden = new Set([7332, 7340, 7341, 33223, 17450, 17443, 44307, 8317]);
|
||||
const file = `${area}/proxy-keys.json`, delay = `${area}/delay-put`;
|
||||
let keys = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : ['fixture-foreign'];
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const reply = (status, data) => { res.writeHead(status, {'content-type':'application/json'}); res.end(JSON.stringify(data)); };
|
||||
if (req.url === '/test/alpha-key') {
|
||||
const alpha = keys.filter(k => /-alpha-key-/.test(k));
|
||||
if (alpha.length !== 1) return reply(409, { error: 'alpha key missing' });
|
||||
const key = alpha[0];
|
||||
const probe = await fetch(`http://127.0.0.1:${server.address().port}/v1/models`, { headers: { authorization: `Bearer ${key}` } });
|
||||
return reply(200, { keyId: createHash('sha256').update(key).digest('hex').slice(0,12), accepted: probe.ok, alphaKeys: alpha.length,
|
||||
betaSentinels: keys.filter(k => /-beta-sentinel-/.test(k)).length, gammaSentinels: keys.filter(k => /-gamma-sentinel-/.test(k)).length });
|
||||
}
|
||||
if (req.url === '/v1/models') return reply(keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401, {});
|
||||
if (req.url === '/v8/management/config/access/api-keys') {
|
||||
if (req.headers.authorization !== 'Bearer fixture-management') return reply(401, {});
|
||||
if (req.method === 'GET') return reply(200, keys);
|
||||
if (req.method !== 'PUT') return reply(405, {});
|
||||
const chunks = []; for await (const c of req) chunks.push(c);
|
||||
const next = JSON.parse(Buffer.concat(chunks));
|
||||
if (fs.existsSync(delay)) await new Promise(resolve => setTimeout(resolve, 12000));
|
||||
keys = next; fs.writeFileSync(file, JSON.stringify(keys), {mode:0o600}); return reply(200, {status:'ok'});
|
||||
}
|
||||
if (req.method === 'POST' && /^\/v1\/messages(?:\?|$)/.test(req.url ?? '')) {
|
||||
if (!keys.includes(req.headers['x-api-key'])) return reply(401, {});
|
||||
const chunks = []; for await (const c of req) chunks.push(c);
|
||||
const body = JSON.parse(Buffer.concat(chunks));
|
||||
const msg = {id:'msg_proxy_fixture',type:'message',role:'assistant',model:body.model,stop_sequence:null};
|
||||
if (!body.stream) return reply(200, {...msg,content:[{type:'text',text:'PINEAPPLE'}],stop_reason:'end_turn',usage:{input_tokens:1,output_tokens:1}});
|
||||
res.writeHead(200, {'content-type':'text/event-stream','cache-control':'no-cache'});
|
||||
const event = (type,data) => res.write(`event: ${type}\ndata: ${JSON.stringify({type,...data})}\n\n`);
|
||||
event('message_start',{message:{...msg,content:[],stop_reason:null,usage:{input_tokens:1,output_tokens:0}}});
|
||||
event('content_block_start',{index:0,content_block:{type:'text',text:''}});
|
||||
event('content_block_delta',{index:0,delta:{type:'text_delta',text:'PINEAPPLE'}});
|
||||
event('content_block_stop',{index:0}); event('message_delta',{delta:{stop_reason:'end_turn',stop_sequence:null},usage:{output_tokens:1}}); event('message_stop',{}); return res.end();
|
||||
}
|
||||
return reply(404, {});
|
||||
});
|
||||
async function listen(port) { await new Promise((resolve,reject)=>{ server.once('error',reject); server.listen(port,'127.0.0.1',resolve); }); }
|
||||
let port = Number(portArg);
|
||||
for (;;) { await listen(port); port = server.address().port; if (!forbidden.has(port)) break; await new Promise(r => server.close(r)); if (portArg !== '0') throw Error('forbidden port'); port = 0; }
|
||||
fs.writeFileSync(`${area}/proxy-ready.json`, JSON.stringify({port,pid:process.pid}), {mode:0o600});
|
||||
const stop = () => { server.closeAllConnections(); server.close(() => process.exit(0)); };
|
||||
process.on('SIGTERM',stop); process.on('SIGINT',stop);
|
||||
Reference in new issue
Block a user