#!/usr/bin/env bash
# The organizations CLI E2E (business-design r4 section 7), Bootstrap to
# Console, on a fresh instance `orge2e` in a scratch HOME. A real worker turn
# runs through the real launcher and sandboxed runner against the
# deterministic fixture model on loopback. A separate fake-proxy phase checks
# shared keys and fencing; all endpoints are loopback and credentials are synthetic.
#
# Usage: scripts/e2e-orgs-cli.sh
#
# Bus, Console and human verbs run through the Lead-owned devstack launcher.
# Scratch HOME and test artifacts are retained for inspection. The trap stops
# only this run's processes and checks that its devstack units are gone.
# A /proc sampler checks worker sockets, with the fixture as a positive control.
set -euo pipefail

REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
NAME=orge2e
H="$(mktemp -d /mnt/storage/scratch/tmp/XXXXXX)"
TEST_AREA="$H/area"
mkdir -m 700 "$TEST_AREA"
CONFIG="$H/.config/mosaic-$NAME/config.json"
WATCH_PIDS=()
SETUP_PID=''
FIXTURE_UP=0
PROXY_PID=''
DEVSTACK=/mnt/storage/scratch/mosaic-lead/uidev/org-release/devstack.sh
export HOME="$H" MOSAIC_CONFIG="$CONFIG" TMPDIR="$H"
RUN_ENV=(env -i "PATH=$REPO/scripts:$PATH" "HOME=$H" "MOSAIC_CONFIG=$CONFIG" "TMPDIR=$H" "LANG=C.UTF-8")
stack() { "$DEVSTACK" "$@"; }
mosaic() { stack exec -- "$REPO/scripts/mosaic" "$@"; }
fixture() { "${RUN_ENV[@]}" "$REPO/scripts/mosaic" dev fixture "$@"; }
cleanup() {
  local rc=$?
  trap - EXIT
  set +e
  if [ -n "$SETUP_PID" ]; then touch "$TEST_AREA/git-release"; kill -TERM "$SETUP_PID" 2>/dev/null; wait "$SETUP_PID" 2>/dev/null; fi
  for pid in "${WATCH_PIDS[@]}"; do kill -TERM "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; done
  for name in alpha beta gamma; do mosaic bus stop "$name" >/dev/null 2>&1; done
  mosaic console stop >/dev/null 2>&1
  stack stop >/dev/null 2>&1
  if [ -n "$PROXY_PID" ]; then kill -TERM "$PROXY_PID" 2>/dev/null || rc=1; wait "$PROXY_PID" 2>/dev/null || rc=1; fi
  if [ "$FIXTURE_UP" = 1 ]; then fixture stop --json >"$TEST_AREA/fixture-stop.json" 2>&1 || rc=1; fi
  stack list >"$TEST_AREA/devstack-final.txt" 2>&1 || rc=1
  if [ -s "$TEST_AREA/devstack-final.txt" ]; then cat "$TEST_AREA/devstack-final.txt"; rc=1; fi
  echo "scratch HOME kept: $H"
  if [ "$rc" -eq 0 ]; then echo "E2E PASS"; else echo "E2E FAIL (exit $rc)"; fi
  exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT TERM

STEP=""
step() { STEP="$1"; printf '\n== %s\n' "$1"; }
fail() { echo "FAIL [$STEP]: $*" >&2; exit 1; }
# check FILE JQ-FILTER DESCRIPTION: the filter must yield true.
check() { jq -e "$2" "$1" >/dev/null || fail "$3 ($2 on $1)"; echo "ok: $3"; }
# show NAME CMD...: run, keep stdout in $TEST_AREA/NAME, print it.
show() { local out="$TEST_AREA/$1"; shift; "$@" >"$out" || fail "exit $? from: $*"; cat "$out"; }

# netwatch ROOT OUT: until ROOT exits, every 5 ms read /proc for ROOT and its
# descendants, map their socket fds to /proc/net/{tcp,tcp6,udp,udp6} by inode,
# and write each connected socket once, as "pid comm proto local -> peer", to
# OUT.loopback (peer on loopback) or OUT.remote (any other peer). OUT.polls
# counts samples, OUT.pids lists every pid seen in the tree, OUT.sockets every
# socket inode seen, OUT.fderr fd directories that could not be read (ENOENT
# is a process that exited mid-sample; EACCES would be a blind spot). Listening and
# unconnected sockets are skipped. A connection opened and closed between two
# samples is missed: this is a check with a positive control (the worker's
# socket to the fixture port must show up), not a proof.
NETWATCH_JS='
import fs from "node:fs";
const [root, out] = process.argv.slice(1);
const seen = new Set(), pids = new Set(), fderr = new Map(), socks = new Set();
let polls = 0;
const v4 = (h) => [3, 2, 1, 0].map((i) => parseInt(h.slice(i * 2, i * 2 + 2), 16)).join(".");
const addr = (hex) => {
  const [h, port] = hex.split(":");
  const p = parseInt(port, 16);
  if (h.length === 8) return { ip: v4(h), p };
  const words = [0, 1, 2, 3].map((w) => h.slice(w * 8, w * 8 + 8));
  if (words[0] === "00000000" && words[1] === "00000000" && words[2] === "FFFF0000") return { ip: v4(words[3]), p };
  if (h === "00000000000000000000000001000000") return { ip: "::1", p };
  if (/^0+$/.test(h)) return { ip: "::", p };
  return { ip: "[" + words.map((w) => w.match(/../g).reverse().join("")).join("") + "]", p };
};
const loop = (ip) => ip.startsWith("127.") || ip === "::1";
const unset = (ip) => ip === "0.0.0.0" || ip === "::";
const read = (f) => { try { return fs.readFileSync(f, "utf8"); } catch { return ""; } };
const tick = () => {
  polls++;
  // Walk down from ROOT through the children list of every thread: cheaper than
  // reading the stat of every process on the host each sample.
  const tree = [], comm = new Map(), queue = [String(root)];
  while (queue.length) {
    const pid = queue.shift();
    let tasks = [];
    try { tasks = fs.readdirSync(`/proc/${pid}/task`); } catch { continue; }
    tree.push(pid);
    const st = read(`/proc/${pid}/stat`);
    comm.set(pid, st.slice(st.indexOf("(") + 1, st.lastIndexOf(")")));
    for (const t of tasks) for (const c of read(`/proc/${pid}/task/${t}/children`).split(" ")) if (c.trim()) queue.push(c.trim());
  }
  const owner = new Map();
  for (const pid of tree) {
    pids.add(pid);
    let fds = [];
    try { fds = fs.readdirSync(`/proc/${pid}/fd`); } catch (e) { fderr.set(`${pid} ${comm.get(pid)}`, e.code); continue; }
    for (const fd of fds) {
      let l = "";
      try { l = fs.readlinkSync(`/proc/${pid}/fd/${fd}`); } catch { continue; }
      const m = /^socket:\[(\d+)\]$/.exec(l);
      if (m) { owner.set(m[1], pid); socks.add(`${pid} ${comm.get(pid)} ${m[1]}`); }
    }
  }
  if (owner.size === 0) return;
  for (const proto of ["tcp", "tcp6", "udp", "udp6"]) {
    for (const line of read(`/proc/net/${proto}`).split("\n").slice(1)) {
      const f = line.trim().split(/\s+/);
      if (f.length < 10 || !owner.has(f[9])) continue;
      const l = addr(f[1]), r = addr(f[2]);
      if (unset(r.ip) && r.p === 0) continue;
      const pid = owner.get(f[9]);
      const text = `${pid} ${comm.get(pid)} ${proto} ${l.ip}:${l.p} -> ${r.ip}:${r.p}`;
      if (seen.has(text)) continue;
      seen.add(text);
      fs.appendFileSync(`${out}.${loop(r.ip) ? "loopback" : "remote"}`, text + "\n");
    }
  }
};

fs.writeFileSync(`${out}.loopback`, ""); fs.writeFileSync(`${out}.remote`, "");
const done = () => {
  fs.writeFileSync(`${out}.polls`, String(polls));
  fs.writeFileSync(`${out}.pids`, [...pids].join("\n") + "\n");
  fs.writeFileSync(`${out}.fderr`, [...fderr].map(([k, v]) => `${k} ${v}`).join("\n") + (fderr.size ? "\n" : ""));
  fs.writeFileSync(`${out}.sockets`, [...socks].join("\n") + (socks.size ? "\n" : ""));
};
process.on("SIGTERM", () => { done(); process.exit(0); });
while (fs.existsSync(`/proc/${root}`)) { tick(); await new Promise(resolve => setTimeout(resolve, 5)); }
done();
'
netwatch() { exec "${RUN_ENV[@]}" node --input-type=module -e "$NETWATCH_JS" "$1" "$2"; }

echo "scratch HOME: $H"
echo "instance: $NAME"
echo "env passed to every command: ${RUN_ENV[*]:2}" | sed -E 's/(PATH=)[^ ]*/\1.../'
held="$(env | cut -d= -f1 | /usr/bin/grep -E '^(ANTHROPIC_|OPENAI_|OPENROUTER_|ZAI_|GEMINI_|GOOGLE_API|CLAUDE_CODE_OAUTH)' | tr '\n' ' ' || true)"
[ -n "$held" ] && echo "caller provider variables not passed (names only): $held"

step "Bootstrap"
[ -e "$H/.config/mosaic-$NAME" ] && fail "instance $NAME already exists"
"${RUN_ENV[@]}" "$REPO/scripts/bootstrap.sh" --instance "$NAME" | tee "$TEST_AREA/bootstrap.out"
[ -f "$CONFIG" ] || fail "no config at $CONFIG"
DATA_ROOT="$(jq -r .dataRoot "$CONFIG")"
echo "config: $CONFIG, dataRoot: $DATA_ROOT"
[ "$DATA_ROOT" = "$H/.mosaic-$NAME" ] || fail "data root $DATA_ROOT is not $H/.mosaic-$NAME"
check "$DATA_ROOT/instance.json" ".instance == \"$NAME\" or .name == \"$NAME\"" "marker names instance $NAME"
echo "ok: instance identity read from config and marker"

step "Fixture"
show fixture-init.json fixture init --instance "$NAME" --model deterministic --json
FIXTURE_UP=1
check "$TEST_AREA/fixture-init.json" '.receipt == "dev-fixture-init" and .loopback == true and .external == false and .upstream == "none"' "fixture receipt: loopback, no external upstream"
check "$TEST_AREA/fixture-init.json" '.endpoint | test("^http://127\\.0\\.0\\.1:[0-9]+$")' "endpoint on 127.0.0.1, OS-assigned port"
check "$TEST_AREA/fixture-init.json" '.principal == "local-dev:local-dev-owner"' "test principal is local-dev"
show fixture-status.json fixture status --json
check "$TEST_AREA/fixture-status.json" '.ready == true and .upstream == "none" and .external == false and .health.upstream == "none"' "status ready, upstream none"

org_setup() {
  local id="$1" doc="$TEST_AREA/$1.json"
  step "$id document"
  mosaic org template render "$id" --fixture local --output "$doc" | tee "$TEST_AREA/$id-render.out"
  if [ "$id" = alpha ]; then
    jq '.projects.concurrent = (.projects.work | .root = "/__mosaic_org__/alpha/projects/concurrent")' "$doc" >"$doc.new"
    mv "$doc.new" "$doc"
  fi
  check "$doc" ".id == \"$id\" and (.projects.work | type) == \"object\"" "$id document names project work"
  if /usr/bin/grep -Eiq '(api[_-]?key|token|secret|password)"[[:space:]]*:' "$doc"; then fail "$doc carries a secret-looking key"; fi
  echo "ok: no secret-looking keys in $doc"
  mosaic org validate "$doc" | tee "$TEST_AREA/$id-validate.out"

  step "$id create"
  mosaic org create "$id" --from "$doc" | tee "$TEST_AREA/$id-create.out"
  show "$id-show.json" mosaic org show "$id" --json
  check "$TEST_AREA/$id-show.json" ".id == \"$id\"" "org show reads back $id"

  check "$TEST_AREA/$id-create.out" '.notifications == "none"' "$id receipt records notifications: none"
  check "$(jq -r .root "$TEST_AREA/$id-show.json")/notify/$id/notify.json" '.notifyVersion == 1 and .binding == null' "$id notifier records explicit null"

  step "$id Git"
  mosaic org queue init "$id" --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-queue-init.out"
  mosaic org project repo init "$id" work --local-only --accept-local-loss-risk | tee "$TEST_AREA/$id-repo-init.out"
  show "$id-queue.json" mosaic org queue show "$id" --json
  show "$id-repo.json" mosaic org project repo show "$id" work --json
  local root; root="$(jq -r .root "$TEST_AREA/$id-show.json")"
  check "$TEST_AREA/$id-queue.json" ".org == \"$id\" and .mode == \"local-only\" and .riskAccepted == true and .root == \"$root/checkout\"" "$id queue: local-only, risk accepted, root under the org root"
  check "$TEST_AREA/$id-repo.json" ".org == \"$id\" and .project == \"work\" and .mode == \"local-only\" and .riskAccepted == true and .root == \"$root/projects/work\"" "$id work repo: local-only, risk accepted, root derived from $id and work"
  local kind f dir
  for kind in queue repo; do
    f="$TEST_AREA/$id-$kind.json"; dir="$(jq -r .root "$f")"
    [ "$(git -C "$dir" rev-parse HEAD)" = "$(jq -r .commit "$f")" ] || fail "$dir HEAD is not the recorded commit"
    [ -z "$(git -C "$dir" remote)" ] || fail "$dir has a remote"
    echo "ok: $dir is a Git checkout at the recorded commit with no remote"
  done
}

# A unit owns each detached child until its matching CLI stop completes.
cat >"$TEST_AREA/unit.sh" <<'UNIT'
#!/usr/bin/env bash
set -euo pipefail
kind=$1; name=$2; out=$3
if [ "$kind" = bus ]; then
  "$REPO_ROOT/scripts/mosaic" bus start "$name" --dev-local-owner --detach >"$out"
else
  "$REPO_ROOT/scripts/mosaic" console start --dev-local-owner --detach --port auto >"$out"
fi
pid=$(jq -r .pid "$out")
while kill -0 "$pid" 2>/dev/null; do sleep 0.2; done
UNIT
chmod 700 "$TEST_AREA/unit.sh"
# devstack passes its own small environment, so put the checkout path in the
# unit script itself rather than depending on a caller's environment.
sed -i "s|\$REPO_ROOT|$REPO|g" "$TEST_AREA/unit.sh"
start_unit() {
  local name=$1 kind=$2 org=$3 out="$TEST_AREA/$1-start.json"
  rm -f "$out"
  stack run "$name" -- "$TEST_AREA/unit.sh" "$kind" "$org" "$out"
  for _ in $(seq 1 150); do [ -s "$out" ] && jq -e '.pid > 1' "$out" >/dev/null 2>&1 && return; sleep 0.2; done
  stack logs "$name"; fail "$name did not start"
}

org_setup alpha
org_setup beta
step "distinct organization roots"
check "$TEST_AREA/beta-show.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-show.json")\"" "beta root differs from alpha"
check "$TEST_AREA/beta-repo.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-repo.json")\"" "beta project root differs from alpha"
start_unit alpha bus alpha
start_unit beta bus beta
for id in alpha beta; do
  show "$id-status.json" mosaic bus status "$id" --json
  check "$TEST_AREA/$id-status.json" ".org == \"$id\" and .instance == \"$NAME\" and .healthy == true and .pid > 1" "$id bus identity and health"
  pid=$(jq -r .pid "$TEST_AREA/$id-status.json")
  netwatch "$pid" "$TEST_AREA/netwatch-$id" & WATCH_PIDS+=("$!")
done
check "$TEST_AREA/beta-status.json" ".root != \"$(jq -r .root "$TEST_AREA/alpha-status.json")\" and .socket != \"$(jq -r .socket "$TEST_AREA/alpha-status.json")\"" "both buses have distinct roots and sockets"
start_unit console console console
check "$TEST_AREA/console-start.json" '.pid > 1 and (.url | test("^http://127\\.0\\.0\\.1:[0-9]+"))' "Console started on loopback"
CONSOLE_URL=$(jq -r .url "$TEST_AREA/console-start.json")
curl --noproxy '*' -fsS "${CONSOLE_URL%/}/api/orgs" >"$TEST_AREA/console-orgs.json"
check "$TEST_AREA/console-orgs.json" '([.[] | select(.id == "alpha" and .busHealthy == true)] | length) == 1 and ([.[] | select(.id == "beta" and .busHealthy == true)] | length) == 1' 'Console lists both healthy organizations'
for id in alpha beta; do
  curl --noproxy '*' -fsS "${CONSOLE_URL%/}/api/orgs/$id/business" >"$TEST_AREA/console-$id-business.json"
  check "$TEST_AREA/console-$id-business.json" ".business.id == \"$id\"" "Console scoped business read returns $id"
done

step 'broker document write remains visible to the organization service and Console'
mkdir -m 700 "$TEST_AREA/git-bin"
cat >"$TEST_AREA/git-bin/git" <<WRAPPER
#!/usr/bin/env bash
for arg in "\$@"; do
  if [ "\$arg" = init ]; then
    touch "$TEST_AREA/git-ready"
    while [ ! -e "$TEST_AREA/git-release" ]; do sleep 0.1; done
    break
  fi
done
exec /usr/bin/git "\$@"
WRAPPER
chmod 700 "$TEST_AREA/git-bin/git"
stack exec -- env "PATH=$TEST_AREA/git-bin:$PATH" "$REPO/scripts/mosaic" org project repo init alpha concurrent --local-only --accept-local-loss-risk >"$TEST_AREA/concurrent-setup.out" 2>&1 & SETUP_PID=$!
for _ in $(seq 1 150); do [ -e "$TEST_AREA/git-ready" ] && break; kill -0 "$SETUP_PID" 2>/dev/null || fail 'concurrent setup ended before git init'; sleep 0.1; done
[ -e "$TEST_AREA/git-ready" ] || fail 'concurrent setup did not reach git init'
show alpha-plugins-before.json mosaic plugins --org alpha --json
jq '{revision, plugins: {channel: ["cli"]}}' "$TEST_AREA/alpha-plugins-before.json" >"$TEST_AREA/alpha-plugins-patch.json"
show alpha-plugins-set.json mosaic plugins set --file "$TEST_AREA/alpha-plugins-patch.json" --org alpha
touch "$TEST_AREA/git-release"
if wait "$SETUP_PID"; then fail 'concurrent setup published a stale document'; fi
SETUP_PID=''
/usr/bin/grep -q 'revision-conflict' "$TEST_AREA/concurrent-setup.out" || fail 'concurrent setup did not report a revision conflict'
echo 'ok: setup refused its stale document after the broker write'
show alpha-after-write.json mosaic org show alpha --json
check "$TEST_AREA/alpha-after-write.json" '.doc.plugins.channel == ["cli"]' 'organization service reads broker change'
curl --noproxy '*' -fsS "${CONSOLE_URL%/}/api/orgs/alpha/business" >"$TEST_AREA/console-alpha-after-write.json"
check "$TEST_AREA/console-alpha-after-write.json" '.business.id == "alpha"' 'Console resolves alpha after broker change'

worker() {
  local id=$1
  step "$id worker"
  show "$id-project.out" mosaic project create work --repo work --org "$id"
  show "$id-thread.out" mosaic thread create work trial --org "$id"
  show "$id-talk.out" mosaic talk --thread work/trial 'Reply with fixture word' --wait 60 --org "$id"
  show "$id-thread-read.json" mosaic thread read work/trial --org "$id" --json
  /usr/bin/grep -q PINEAPPLE "$TEST_AREA/$id-thread-read.json" || fail "$id thread has no fixture reply"
  show "$id-evidence.json" mosaic run evidence "$id" --thread work/trial --verify --json
  check "$TEST_AREA/$id-evidence.json" ".verified == true and .org.id == \"$id\" and (.runs | length) >= 1" "$id evidence verified"
  check "$TEST_AREA/$id-evidence.json" '.runs[0].launch.model == "fixture-deterministic" and .runs[0].launch.harness == "pi" and ([.runs[0].snapshot.files[].ok] | all)' "$id fixture launch and bundle hashes"
}
worker alpha
worker beta
show fixture-status-after.json fixture status --json
check "$TEST_AREA/fixture-status-after.json" '.health.counts.messages >= 2' "fixture answered both workers"

step "clean stop and restart alpha"
show alpha-stop.json mosaic bus stop alpha
check "$TEST_AREA/alpha-stop.json" '.stopped == true' "alpha stopped cleanly"
wait "${WATCH_PIDS[0]}" || true
for _ in $(seq 1 150); do
  if ! stack list | /usr/bin/grep -q 'orgdev-orge2e-alpha\.service'; then break; fi
  sleep 0.2
done
if stack list | /usr/bin/grep -q 'orgdev-orge2e-alpha\.service'; then fail 'alpha devstack unit did not clear after stop'; fi
start_unit alpha bus alpha
show alpha-restarted.json mosaic bus status alpha --json
check "$TEST_AREA/alpha-restarted.json" ".healthy == true and .pid != $(jq -r .pid "$TEST_AREA/alpha-status.json")" "alpha restarted under a new host"
show beta-still.json mosaic bus status beta --json
check "$TEST_AREA/beta-still.json" ".healthy == true and .pid == $(jq -r .pid "$TEST_AREA/beta-status.json")" "beta stayed healthy"
for id in alpha beta; do
  [ "$id" = beta ] && { kill -TERM "${WATCH_PIDS[1]}"; wait "${WATCH_PIDS[1]}" || true; }
  watch="$TEST_AREA/netwatch-$id"
  [ -f "$watch.fderr" ] || fail "$id sampler did not finish"
  /usr/bin/grep -v ' ENOENT$' "$watch.fderr" && fail "$id sampler could not read process fds"
  [ ! -s "$watch.remote" ] || fail "$id sampler saw non-loopback peer"
  port=$(jq -r .port "$TEST_AREA/fixture-init.json")
  /usr/bin/grep -Eq -- "-> 127\\.0\\.0\\.1:$port\$" "$watch.loopback" || fail "$id sampler missed fixture positive control"
  echo "ok: $id socket sampler saw fixture and no remote peers"
done
WATCH_PIDS=()

step "shared fake proxy"
# The model fixture and shared proxy are intentionally separate phases.
for id in alpha beta; do mosaic bus stop "$id" >/dev/null; done
for id in alpha beta; do
  thread_id=$(jq -r .thread.id "$TEST_AREA/$id-evidence.json")
  show "$id-evidence-stopped.json" mosaic run evidence "$id" --thread "$thread_id" --verify --json
  check "$TEST_AREA/$id-evidence-stopped.json" ".verified == true and .org.id == \"$id\" and (.runs | length) >= 1" "$id evidence remains verified after bus stop"
done
mosaic console stop >/dev/null
stack stop
fixture stop --json >"$TEST_AREA/fixture-stop.json"
FIXTURE_UP=0
start_proxy() {
  rm -f "$TEST_AREA/proxy-ready.json"
  "${RUN_ENV[@]}" node "$REPO/scripts/e2e-orgs-proxy.mjs" "$TEST_AREA" "${1:-0}" >"$TEST_AREA/proxy.log" 2>&1 & PROXY_PID=$!
  for _ in $(seq 1 100); do [ -s "$TEST_AREA/proxy-ready.json" ] && return; kill -0 "$PROXY_PID" 2>/dev/null || { cat "$TEST_AREA/proxy.log"; fail "fake proxy stopped"; }; sleep 0.1; done
  fail "fake proxy did not start"
}
start_proxy
PROXY_PORT=$(jq -r .port "$TEST_AREA/proxy-ready.json")
mkdir -m 700 "$DATA_ROOT/proxy"
mkdir -p -m 700 "$(dirname "$CONFIG")/secrets"
printf '%s' fixture-management >"$(dirname "$CONFIG")/secrets/proxy-management.key"
chmod 600 "$(dirname "$CONFIG")/secrets/proxy-management.key"
PROXY_VERSION=$(jq -r .version "$REPO/packages/model-proxy/PIN")
jq -n --argjson port "$PROXY_PORT" --arg binary "$DATA_ROOT/opt/cliproxyapi/$PROXY_VERSION/cli-proxy-api" '{port:$port,binary:$binary}' >"$(dirname "$CONFIG")/proxy.json"
chmod 600 "$(dirname "$CONFIG")/proxy.json"
start_unit alpha bus alpha
show alpha-proxy-status.json mosaic bus status alpha --json
check "$TEST_AREA/alpha-proxy-status.json" '.healthy == true' 'alpha bus with shared proxy is healthy'
show alpha-proxy-thread.out mosaic thread create work proxytrial --org alpha
show alpha-proxy-talk.out mosaic talk --thread work/proxytrial 'Reply with fixture word' --wait 60 --org alpha
show alpha-proxy-read.json mosaic thread read work/proxytrial --org alpha --json
/usr/bin/grep -q PINEAPPLE "$TEST_AREA/alpha-proxy-read.json" || fail "alpha proxy worker did not answer"
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-before.json"
check "$TEST_AREA/key-before.json" '.accepted == true and .alphaKeys == 1' 'alpha current proxy key is accepted'
start_unit beta bus beta
show beta-proxy-status.json mosaic bus status beta --json
check "$TEST_AREA/beta-proxy-status.json" '.healthy == true' 'beta bus with shared proxy is healthy'
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-after.json"
check "$TEST_AREA/key-after.json" ".accepted == true and .betaSentinels == 1 and .keyId == \"$(jq -r .keyId "$TEST_AREA/key-before.json")\"" 'alpha current key survives beta registration'
org_setup gamma
: >"$TEST_AREA/delay-put"
start_unit gamma bus gamma
show fence-set.json mosaic proxy fence status --json
check "$TEST_AREA/fence-set.json" '.uncertain.org == "gamma" and .uncertain.op == "sentinel.register"' 'timed-out PUT leaves durable fence'
show alpha-fenced-thread.out mosaic thread create work fenced --org alpha
show alpha-fenced-talk.out mosaic talk --thread work/fenced 'This launch must be fenced' --wait 0 --org alpha
alpha_log="$(jq -r .root "$TEST_AREA/alpha-show.json")/launches/alpha.jsonl"
for _ in $(seq 1 100); do /usr/bin/grep -q '"code":"proxy-uncertain"' "$alpha_log" && break; sleep 0.1; done
/usr/bin/grep -q '"code":"proxy-uncertain"' "$alpha_log" || fail 'alpha launch did not refuse the proxy fence'
echo 'ok: alpha proxy-dependent launch refused while fenced'
if mosaic proxy fence clear >"$TEST_AREA/fence-refusal.out" 2>&1; then fail 'fence cleared without restart statement'; fi
/usr/bin/grep -q 'proxy-restart-required' "$TEST_AREA/fence-refusal.out" || fail 'expected restart-required refusal'
kill -TERM "$PROXY_PID"; wait "$PROXY_PID"; PROXY_PID=''
rm "$TEST_AREA/delay-put"
start_proxy "$PROXY_PORT"
show fence-clear.json mosaic proxy fence clear --proxy-restarted
check "$TEST_AREA/fence-clear.json" '.cleared == true' 'fence cleared after proxy restart'
show fence-final.json mosaic proxy fence status --json
check "$TEST_AREA/fence-final.json" '.uncertain == null' 'fence status clear'
mosaic bus stop gamma >/dev/null
start_unit gamma bus gamma
curl --noproxy "*" -fsS "http://127.0.0.1:$PROXY_PORT/test/alpha-key" >"$TEST_AREA/key-after-recovery.json"
check "$TEST_AREA/key-after-recovery.json" ".accepted == true and .betaSentinels == 1 and .gammaSentinels == 1 and .keyId == \"$(jq -r .keyId "$TEST_AREA/key-before.json")\"" 'recovery registers gamma and preserves alpha key and beta sentinel'