Compare commits

..
Author SHA1 Message Date
jason.woltje 46784c8dd4 feat(ri-050): bind next publication to exact-commit terminal verification (#1275)
ci/woodpecker/pr/ci Pipeline was successful
2026-08-17 20:12:20 -05:00
14 changed files with 545 additions and 937 deletions
+24 -1
View File
@@ -30,6 +30,19 @@ steps:
# the baked pnpm store.
- pnpm install --frozen-lockfile --prefer-offline
# ---------------------------------------------------------------------------
# The steps below (sanitization, upgrade-guard, typecheck, lint, format,
# test) are the COMPLETE mandatory verification set. SDLC-D-034 mirrors them
# one-for-one in the canonical terminal verification command — root
# `pnpm verify:release` (scripts/verify-release.mjs) — which the publish
# pipeline (.woodpecker/publish.yml `verify` step) runs before ANY publish
# effect. These lines stay direct (not routed through the runner) because the
# #1017 test-enumeration guard audits framework tool paths through THIS
# surface; scripts/verify-release.test.mjs enforces that the runner's stage
# table keeps matching these commands exactly, so the two cannot drift.
# ---------------------------------------------------------------------------
# Canonical verify:release stage `sanitization`.
# Blocking gate: public framework package must contain no operator-specific
# personal data or private $HOME defaults. Runs early (no node_modules needed).
sanitization:
@@ -47,6 +60,7 @@ steps:
# with everything it guards; this direct line keeps one instrument running.
- bash packages/mosaic/framework/tools/quality/scripts/check-test-enumeration.sh
# Canonical verify:release stage `upgrade-guard`.
# Blocking gate (#791): a framework upgrade must never write or delete an
# operator-owned path. The HARD GATE proves an unanticipated operator sentinel
# survives a keep-mode reseed byte-identical (with rsync present AND absent —
@@ -68,6 +82,8 @@ steps:
- bash packages/mosaic/framework/tools/quality/scripts/test-upgrade-durable-snapshot.sh
- bash packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh
# Canonical verify:release stage `typecheck` — the same `pnpm typecheck`
# invocation (which runs the checkout preflight first, then turbo).
typecheck:
image: *node_image
commands:
@@ -78,7 +94,8 @@ steps:
- sanitization
- upgrade-guard
# lint, format, and test are independent — run in parallel after typecheck
# lint, format, and test are independent — run in parallel after typecheck.
# Each runs exactly its canonical verify:release stage command.
lint:
image: *node_image
commands:
@@ -95,6 +112,12 @@ steps:
depends_on:
- typecheck
# Canonical verify:release stage `test` — the `pnpm test` line below is the
# shared command; everything else in this step is PIPELINE-LEVEL
# prerequisite the canonical command expects its caller to provide (SDLC-D-034):
# the ci-postgres service + pg_isready wait + db:migrate (postgres path),
# `apk add openssl`, and the pinned pi install. None of those can move into
# the runner (it must also work locally on the PGlite path with no database).
test:
image: *node_image
environment:
+60
View File
@@ -1,5 +1,19 @@
# Build, publish npm packages, and push Docker images
# Runs on main for stable publishes and on next for integration-line prereleases/images
#
# SDLC-D-034 publish gate: every publish effect (publish-npm, publish-next-npm,
# and every image build/push step) depends DIRECTLY on the `verify` step below.
# `verify` (a) asserts the provider's commit identity matches the actual
# checkout (CI_COMMIT_SHA == git rev-parse HEAD, fail closed on mismatch or
# emptiness) and (b) runs the canonical terminal verification command
# (`pnpm verify:release`), which mirrors the PR CI pipeline's complete
# mandatory set (sanitization, upgrade-guard, preflight+typecheck, lint,
# format:check, test, build) — see scripts/verify-release.mjs. A missing,
# failed, skipped, cancelled, or inconclusive verification therefore skips the
# dependent publish effects (fail closed). Path-filtered short-circuits may
# skip publish EFFECTS (e.g. docs-only merges) but never bypass `verify` for a
# publish that does run: `verify` itself carries no path filter.
# scripts/verify-release.test.mjs enforces this DAG invariant at checkout time.
variables:
# Pre-baked CI base (see .woodpecker/ci-image.yml): node:24-alpine +
@@ -48,6 +62,45 @@ steps:
# Resolve from the baked pnpm store instead of a cold network fetch.
- pnpm install --frozen-lockfile --prefer-offline
# SDLC-D-034 exact-commit publish gate. No `when`/path filter on purpose: it
# runs for every event this pipeline serves so no publish effect can ever
# start without it. Fails closed on commit-identity mismatch (or either SHA
# being empty) and on any incomplete verification.
verify:
image: *node_image
commands:
- *enable_pnpm
# (a) Commit identity: the provider's claimed SHA must equal the actual
# checkout HEAD — verification of anything else must never authorize a
# publish of this commit.
- |
if [ -z "$CI_COMMIT_SHA" ]; then
echo "[verify] FATAL: CI_COMMIT_SHA is empty — cannot certify commit identity" >&2
exit 1
fi
CHECKOUT_SHA="$(git rev-parse HEAD 2>/dev/null || true)"
if [ -z "$CHECKOUT_SHA" ]; then
echo "[verify] FATAL: git rev-parse HEAD returned nothing — cannot certify commit identity" >&2
exit 1
fi
if [ "$CI_COMMIT_SHA" != "$CHECKOUT_SHA" ]; then
echo "[verify] FATAL: provider commit ($CI_COMMIT_SHA) != checkout HEAD ($CHECKOUT_SHA)" >&2
exit 1
fi
echo "[verify] commit identity confirmed: $CHECKOUT_SHA"
# (b) Canonical terminal verification. Caller-provided prerequisites the
# runner expects (see .woodpecker/ci.yml comments): bash/rsync for the
# guard stages, openssl + the pinned pi binary for the test stage. git is
# baked into ci-base but re-asserted here so the identity check above can
# never silently depend on a stale baked image. DATABASE_URL is
# deliberately NOT set: the canonical command must hold on the PGlite
# path too and never sets or requires a database itself.
- apk add --no-cache bash rsync openssl git
- npm install -g @earendil-works/[email protected]
- pnpm verify:release
depends_on:
- install
build:
image: *node_image
commands:
@@ -55,6 +108,7 @@ steps:
- pnpm build
depends_on:
- install
- verify
publish-npm:
image: *node_image
@@ -114,6 +168,7 @@ steps:
exit 1
depends_on:
- build
- verify
publish-next-npm:
image: *node_image
@@ -192,6 +247,7 @@ steps:
echo "[publish-next] @mosaicstack/mosaic@next resolves to $RESOLVED_VERSION"
depends_on:
- build
- verify
# TODO: Uncomment when ready to publish to npmjs.org
# publish-npmjs:
@@ -205,6 +261,7 @@ steps:
# - bash scripts/publish-npmjs.sh
# depends_on:
# - build
# - verify
# when:
# - event: [tag]
@@ -242,6 +299,7 @@ steps:
/kaniko/executor --context . --dockerfile docker/gateway.Dockerfile $DESTINATIONS
depends_on:
- build
- verify
build-appservice:
image: gcr.io/kaniko-project/executor:debug
@@ -268,6 +326,7 @@ steps:
/kaniko/executor --context . --dockerfile docker/appservice.Dockerfile $DESTINATIONS
depends_on:
- build
- verify
build-web:
image: gcr.io/kaniko-project/executor:debug
@@ -294,3 +353,4 @@ steps:
/kaniko/executor --context . --dockerfile docker/web.Dockerfile $DESTINATIONS
depends_on:
- build
- verify
+1
View File
@@ -9,6 +9,7 @@
"preflight": "node scripts/preflight.mjs",
"clean:generated": "node scripts/clean-generated.mjs",
"typecheck": "pnpm preflight && turbo run typecheck",
"verify:release": "node scripts/verify-release.mjs",
"test:checkout": "node --test scripts/*.test.mjs",
"test": "pnpm test:checkout && turbo run test && pnpm run test:installer",
"test:installer": "bash tools/install-next-lane.test.sh",
@@ -233,36 +233,8 @@ assert_owned_tmux_server() {
fail "tmux server ownership or environment validation failed"
}
# Lease-broker socket preflight (#1292). The gated runtime (`mosaic yolo …` →
# launch-runtime.py) registers with the broker or dies ~4 seconds in, with the
# diagnostic invisible because tmux destroys the dead pane. This check runs
# BEFORE any tmux effect — including the ownership probe below — so a host
# without a broker produces a named, surviving refusal instead of a doomed
# pane. Exit 75 (EX_TEMPFAIL), distinct from 64 (bad projection) and 69 (host
# not ready for other reasons); the agent@ unit is Type=oneshot with no
# Restart=, so the failed unit keeps its message instead of looping. Socket
# resolution matches launch.ts's defaultLeaseBrokerSocket precedence exactly.
# This preflight DETECTS and REFUSES — it never starts the broker (activation
# belongs to the fleet control plane; a component that both detects and fixes
# cannot be used to measure whether the fix worked).
broker_socket_path() {
if [ -n "${MOSAIC_LEASE_BROKER_SOCKET:-}" ]; then
printf '%s\n' "$MOSAIC_LEASE_BROKER_SOCKET"
return 0
fi
local runtime_dir="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
printf '%s\n' "${runtime_dir}/mosaic-lease/broker.sock"
}
if [ "$MODE" = "launch" ]; then
_broker_socket=$(broker_socket_path)
if [ ! -S "$_broker_socket" ]; then
echo "[fleet] FAIL_LAUNCH broker-absent: lease broker socket ${_broker_socket} missing; runtime launch denied (#1292)." >&2
echo "[fleet] remedy: systemctl --user enable --now mosaic-lease-broker.service (or reinstall via: mosaic fleet install)" >&2
exit 75
fi
fi
# Validate exact server ownership before querying, cleaning, or creating any
# managed session. An unmanaged or contaminated named socket is never repaired.
assert_owned_tmux_server
if [ "$MODE" = interaction ]; then
@@ -1,216 +0,0 @@
#!/usr/bin/env bash
# CI-fit regression suite for the #1292 lease-broker socket preflight in
# start-agent-session.sh.
#
# WHY THIS SUITE IS CI-FIT WHERE test-start-agent-session.sh IS NOT (#1017/#1270
# context): that older suite's precondition is "the host does not have the pi
# binary", which a CI image that ships pi violates — its guard correctly
# refuses to report a pass there, so it is excluded from the chain. THIS suite
# controls its own preconditions instead of inheriting them from the host: a
# fake tmux on PATH, a fake mosaic on PATH, a real unix socket created in a
# tmpdir, a hermetic env (env -i, fake HOME, GIT_CONFIG_GLOBAL severed). It
# never depends on what the host has installed, so a green here means the same
# thing on every host. Anyone adding cases: keep that property — no case may
# depend on host state.
#
# The failure this suite is written down to catch (#1292): a seat launched on a
# host with no lease broker dies ~4 seconds in at registration, with the
# diagnostic invisible because tmux destroys the dead pane. The preflight runs
# BEFORE any tmux effect and refuses with a NAMED code (exit 75, EX_TEMPFAIL)
# so the message survives. The agent@ unit is Type=oneshot with no Restart=,
# so a failed unit keeps its output instead of looping.
#
# Cases:
# 1. absent socket -> exit 75, message names broker-absent + socket path +
# remedy, and NO tmux session was ever created (the doomed-pane half).
# 2. present socket (real unix socket in tmpdir) -> proceeds PAST the
# preflight (the suite then stops at the next precondition, proving the
# preflight was not the refusal).
# 3. explicit MOSAIC_LEASE_BROKER_SOCKET wins over XDG_RUNTIME_DIR default.
# 4. --stop mode does NOT require the broker (teardown must not be fenced on
# a component whose absence is exactly what teardown may follow).
#
# Sabotage control, run by the developer (not in-suite): remove the preflight
# block from start-agent-session.sh, re-run — case 1 fails (a tmux session is
# created / exit is not 75), cases 2-4 still pass; restore byte-identically.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/agent-session-broker-preflight}"
FAKE_HOME="$WORK_DIR/home"
BIN_DIR="$WORK_DIR/bin"
ENV_DIR="$WORK_DIR/env"
SOCK_DIR="$WORK_DIR/sockets"
LOG_FILE="$WORK_DIR/tmux-calls.log"
rm -rf "$WORK_DIR"
# The script asserts a managed directory tree under MOSAIC_HOME: mosaic/,
# mosaic/fleet/, mosaic/fleet/agents/ — private (0700/0750-style) modes, no
# symlinks — plus a per-agent env projection. Build the full tree the launcher
# expects so the suite reaches the BROKER preflight rather than dying at
# environment validation.
mkdir -p "$FAKE_HOME/.config/mosaic/fleet/agents" "$BIN_DIR" "$SOCK_DIR"
chmod 700 "$FAKE_HOME/.config/mosaic" "$FAKE_HOME/.config/mosaic/fleet/agents"
chmod 750 "$FAKE_HOME/.config/mosaic/fleet"
cat > "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated" <<'ENVEOF'
MOSAIC_AGENT_NAME=preflight-test
MOSAIC_AGENT_CLASS=worker
MOSAIC_AGENT_RUNTIME=pi
MOSAIC_AGENT_MODEL=
MOSAIC_AGENT_REASONING=
MOSAIC_AGENT_TOOL_POLICY=code
MOSAIC_AGENT_WORKDIR=/tmp
MOSAIC_TMUX_SOCKET=mosaic-fleet
ENVEOF
chmod 600 "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated"
# ─── Fake tmux: records every invocation; new-session marks the marker. ────
: > "$LOG_FILE"
cat > "$BIN_DIR/tmux" <<SH
#!/usr/bin/env bash
printf 'tmux %s\n' "\$*" >> "$LOG_FILE"
if [[ "\$*" == *new-session* ]]; then
echo "TMUX-NEW-SESSION-INVOKED" >> "$LOG_FILE"
fi
exit 0
SH
chmod +x "$BIN_DIR/tmux"
# ─── Fake mosaic/pi binaries so the script proceeds past its own lookups. ───
for bin in mosaic pi claude; do
printf '#!/usr/bin/env bash\nexit 0\n' > "$BIN_DIR/$bin"
chmod +x "$BIN_DIR/$bin"
done
# ─── Minimal launch environment the script expects. ────────────────────────
# (Enough for the preflight to be reached; later stages will still fail in
# case 2 — that is expected and asserted.)
run_session_script() {
local mode="$1"; shift
(
cd "$WORK_DIR"
env -i HOME="$FAKE_HOME" PATH="$BIN_DIR:/usr/bin:/bin" \
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
MOSAIC_HOME="$FAKE_HOME/.config/mosaic" \
AGENT_NAME=preflight-test \
"$@" \
bash "$SCRIPT_DIR/start-agent-session.sh" $mode preflight-test
)
}
fail=0
assert() {
local desc="$1" expected="$2" actual="$3"
if [[ "$expected" != "$actual" ]]; then
echo "FAIL: $desc — expected '$expected', got '$actual'" >&2
fail=1
fi
}
assert_contains() {
local desc="$1" haystack="$2" needle="$3"
[[ "$haystack" == *"$needle"* ]] || { echo "FAIL: $desc — missing '$needle' in: $haystack" >&2; fail=1; }
}
assert_not_contains() {
local desc="$1" haystack="$2" needle="$3"
if [[ "$haystack" == *"$needle"* ]]; then
echo "FAIL: $desc — must not contain '$needle'" >&2
fail=1
fi
return 0
}
# ─── 1. Absent socket → named refusal, NO tmux session. ────────────────────
: > "$LOG_FILE"
stderr_file="$WORK_DIR/stderr-1.tmp"
set +e
out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent.sock" 2>"$stderr_file")
rc=$?
set -e
assert "absent socket exit code" "75" "$rc"
err=$(cat "$stderr_file")
assert_contains "absent socket names the failure" "$err" "FAIL_LAUNCH broker-absent"
assert_contains "absent socket names the socket path" "$err" "$SOCK_DIR/absent.sock"
assert_contains "absent socket names a remedy" "$err" "mosaic fleet install"
log1=$(cat "$LOG_FILE")
assert_not_contains "absent socket must not create a tmux session" "$log1" "TMUX-NEW-SESSION-INVOKED"
# ─── 2. Present socket → passes the preflight. ─────────────────────────────
# Expected: ownership/env checks AFTER the preflight may refuse (fixture is
# minimal by design); the assertion is only that the refusal is NOT
# broker-absent and the exit is NOT 75.
# Create a REAL unix socket: a detached python holder binds it and stays alive
# for the duration (bash cannot create sockets; a foreground python would
# close the socket on exit and -S on a closed-but-unlinked path fails). Written
# as a script file + setsid nohup so no job-control/heredoc interaction with
# set -e can silently kill the suite.
# AF_UNIX binds cap at 108 path bytes; the suite's workdir exceeds that, so
# the live socket lives at a SHORT path under /tmp (unique per run, cleaned
# with the suite). The preflight takes its socket path explicitly, so this
# stays fully controlled.
LIVE_SOCK=$(mktemp -u /tmp/mosaic-preflight-XXXXXX.sock)
trap 'rm -f "$LIVE_SOCK"' EXIT
rm -f "$SOCK_DIR/live.sock" "$LIVE_SOCK"
cat > "$SOCK_DIR/holder.py" <<'PY'
import socket, sys, time
path = sys.argv[1]
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.bind(path)
s.listen(1)
time.sleep(120)
PY
python3 "$SOCK_DIR/holder.py" "$LIVE_SOCK" >/dev/null 2>"$SOCK_DIR/holder.err" &
HOLDER_PID=$!
# Wait for the socket object to exist (bind is near-instant, but do not race it).
for _ in $(seq 1 50); do
[ -S "$LIVE_SOCK" ] && break
sleep 0.1
done
if [ ! -S "$LIVE_SOCK" ]; then
echo "FAIL: could not create live socket fixture (holder pid $HOLDER_PID)" >&2
ps -p "$HOLDER_PID" -o pid,stat,cmd --no-headers >&2 || echo "(holder exited)" >&2
cat "$SOCK_DIR/holder.err" >&2 || true
exit 1
fi
: > "$LOG_FILE"
set +e
out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$LIVE_SOCK" 2>"$WORK_DIR/stderr-2.tmp")
rc=$?
set -e
# The preflight PASSED if the failure (whatever later stage refused) is NOT
# the broker refusal, and tmux was reached or a later precondition named
# something else.
err2=$(cat "$WORK_DIR/stderr-2.tmp")
assert_not_contains "live socket must not refuse broker-absent" "$err2" "broker-absent"
if [[ "$rc" == "75" ]]; then
echo "FAIL: live socket — preflight still refused (exit 75) with a live socket" >&2
fail=1
fi
# ─── 3. Explicit socket env wins over XDG default. ─────────────────────────
set +e
out=$(run_session_script "" XDG_RUNTIME_DIR="$SOCK_DIR/no-runtime-here" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent2.sock" 2>"$WORK_DIR/stderr-3.tmp")
rc=$?
set -e
assert "explicit env wins (exit 75)" "75" "$rc"
assert_contains "explicit env path named" "$(cat "$WORK_DIR/stderr-3.tmp")" "$SOCK_DIR/absent2.sock"
# ─── 4. --stop is not fenced on the broker. ────────────────────────────────
: > "$LOG_FILE"
set +e
out=$(run_session_script "--stop" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent3.sock" 2>"$WORK_DIR/stderr-4.tmp")
rc=$?
set -e
err4=$(cat "$WORK_DIR/stderr-4.tmp")
assert_not_contains "--stop must not refuse broker-absent" "$err4" "broker-absent"
if [[ "$rc" == "75" ]]; then
echo "FAIL: --stop — exit 75 means teardown was fenced on the broker" >&2
fail=1
fi
kill "$HOLDER_PID" 2>/dev/null || true
if [[ "$fail" -eq 0 ]]; then
echo "start-agent-session lease-broker preflight regression passed"
fi
exit "$fail"
@@ -1,177 +0,0 @@
import { lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { placeUnitFile, resolveLeaseBrokerSocketForPreflight } from './fleet.js';
/**
* Unit-placement regression harness for #1292.
*
* The two measured defects this suite pins:
* 1. `systemctl enable <name>` does NOT rewrite an existing by-path
* wants-symlink — so placement must remove stale residue explicitly, and
* acceptance asserts on the RESULTING SYMLINK TARGET, never on the enable
* call's argument (asserting the call cannot see where the link ended up).
* 2. Node's copyFile FOLLOWS a by-path symlink at the destination and
* overwrites the SEED template. Acceptance asserts on the SEED's bytes
* AND mtime — unchanged — which is the only check that can redden for
* finding 2. The symlink-target assertion catches finding 1; these are
* different defects with different failure modes.
*
* Fixtures are entirely inside tmpdirs (source template, active systemd dir,
* wants dir) — no real host paths are touched by this suite.
*/
describe('placeUnitFile (#1292 unit placement)', () => {
const cleanup: string[] = [];
afterEach(async () => {
while (cleanup.length > 0) {
await rm(cleanup.pop()!, { recursive: true, force: true });
}
});
async function fixture() {
const root = await mkdtemp(join(tmpdir(), 'place-unit-'));
cleanup.push(root);
const seedDir = join(root, 'seed');
const activeDir = join(root, 'active');
await mkdir(seedDir, { recursive: true });
await mkdir(activeDir, { recursive: true });
const seedTemplate = join(seedDir, 'unit-under-test.service');
await writeFile(
seedTemplate,
'[Unit]\nDescription=seed template\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n',
);
const activeSource = join(root, 'active-source.service');
await writeFile(
activeSource,
'[Unit]\nDescription=active copy v2\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n',
);
return { root, seedDir, activeDir, seedTemplate, activeSource };
}
it('places a regular file on a clean host (negative control: no residue anywhere)', async () => {
const f = await fixture();
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
expect(result.unlinkedDestinationSymlink).toBe(false);
expect(result.removedStaleWantsSymlink).toBe(false);
const info = await lstat(join(f.activeDir, 'unit-under-test.service'));
expect(info.isSymbolicLink()).toBe(false);
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain(
'active copy v2',
);
// Seed untouched by construction — but assert it, so the clean-host case
// cannot silently regress into seed-mutation.
expect(await readFile(f.seedTemplate, 'utf8')).toContain('seed template');
});
it('by-path residue: unlinks destination symlink, places the file, seed bytes AND mtime unchanged (finding 2)', async () => {
const f = await fixture();
const seedBefore = await readFile(f.seedTemplate, 'utf8');
const mtimeBefore = (await lstat(f.seedTemplate)).mtimeMs;
// The fomo-lin convention: by-path enable left a symlink AT the unit name
// pointing at the seed template, plus a wants-symlink doing the same.
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
const wantsDir = join(f.activeDir, 'default.target.wants');
await mkdir(wantsDir, { recursive: true });
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
expect(result.unlinkedDestinationSymlink).toBe(true);
expect(result.removedStaleWantsSymlink).toBe(true);
// FINDING 2's check: the seed is byte-identical and its mtime did not move.
expect(await readFile(f.seedTemplate, 'utf8')).toBe(seedBefore);
expect((await lstat(f.seedTemplate)).mtimeMs).toBe(mtimeBefore);
// The destination is now a regular file carrying the ACTIVE content.
const destInfo = await lstat(join(f.activeDir, 'unit-under-test.service'));
expect(destInfo.isSymbolicLink()).toBe(false);
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain(
'active copy v2',
);
});
it('by-path residue: no wants-symlink remains pointing at the seed (finding 1 residue cleared)', async () => {
const f = await fixture();
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
const wantsDir = join(f.activeDir, 'default.target.wants');
await mkdir(wantsDir, { recursive: true });
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
// After placement the stale wants link is GONE (enable-by-name recreates
// it correctly). A link still present must not point at the seed.
try {
const link = await lstat(join(wantsDir, 'unit-under-test.service'));
if (link.isSymbolicLink()) {
const target = await readFile(join(wantsDir, 'unit-under-test.service'), 'utf8').catch(
async () => '',
);
expect(target).not.toContain('seed template');
}
} catch {
// absent wants link — the expected post-placement state
}
});
it('idempotence: second placement on a reconciled host is a no-op producing the identical final state', async () => {
const f = await fixture();
// Reconciled starting state: regular file at the name, wants link to the active copy.
await writeFile(
join(f.activeDir, 'unit-under-test.service'),
await readFile(f.activeSource, 'utf8'),
);
const wantsDir = join(f.activeDir, 'default.target.wants');
await mkdir(wantsDir, { recursive: true });
await symlink(
join(f.activeDir, 'unit-under-test.service'),
join(wantsDir, 'unit-under-test.service'),
);
const before = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
// No destructive step fired: no unlink, no wants removal.
expect(result.unlinkedDestinationSymlink).toBe(false);
expect(result.removedStaleWantsSymlink).toBe(false);
// Identical final state.
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toBe(before);
const link = await lstat(join(wantsDir, 'unit-under-test.service'));
expect(link.isSymbolicLink()).toBe(true);
});
it('double install on by-path residue converges to the identical reconciled state', async () => {
const f = await fixture();
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
const wantsDir = join(f.activeDir, 'default.target.wants');
await mkdir(wantsDir, { recursive: true });
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
const first = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
const secondRun = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
const second = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
expect(secondRun.unlinkedDestinationSymlink).toBe(false);
expect(second).toBe(first);
});
});
describe('resolveLeaseBrokerSocketForPreflight (#1292 preflight resolution)', () => {
it('explicit MOSAIC_LEASE_BROKER_SOCKET wins', () => {
expect(
resolveLeaseBrokerSocketForPreflight({ MOSAIC_LEASE_BROKER_SOCKET: '/custom/sock' }, 1000),
).toBe('/custom/sock');
});
it('XDG_RUNTIME_DIR next', () => {
expect(resolveLeaseBrokerSocketForPreflight({ XDG_RUNTIME_DIR: '/run/user/1001' }, 1000)).toBe(
'/run/user/1001/mosaic-lease/broker.sock',
);
});
it('falls back to /run/user/<uid>', () => {
expect(resolveLeaseBrokerSocketForPreflight({}, 1002)).toBe(
'/run/user/1002/mosaic-lease/broker.sock',
);
});
});
+1 -110
View File
@@ -835,25 +835,13 @@ describe('fleet command construction', () => {
};
const program = new Command();
program.exitOverride();
// #1292: inject a present broker socket so the preflight passes and this
// spec keeps testing its ORIGINAL property (holder-before-agent ordering).
// The preflight's own refusal behavior has dedicated specs below.
registerFleetCommand(program, {
runner,
mosaicHome: home,
checkBrokerSocket: async () => true,
});
registerFleetCommand(program, { runner, mosaicHome: home });
try {
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
await program.parseAsync(['node', 'mosaic', 'fleet', 'stop']);
expect(calls).toEqual([
// #1292: fleet start enables + starts the broker FIRST (enable is
// idempotent; the unit exists after install), re-checking the socket
// before any holder/agent lifecycle effect.
['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
['systemctl', '--user', 'start', 'mosaic-tmux-holder.service'],
['systemctl', '--user', 'start', '[email protected]'],
['systemctl', '--user', 'stop', '[email protected]'],
@@ -864,92 +852,6 @@ describe('fleet command construction', () => {
}
});
it('fleet start refuses with a named error when the broker socket does not appear (#1292)', async () => {
const home = await tempDir();
const rosterPath = join(home, 'fleet', 'roster.yaml');
await mkdir(join(home, 'fleet'), { recursive: true });
await writeFile(
rosterPath,
['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join(
'\n',
),
);
const calls: string[][] = [];
const runner: CommandRunner = async (command, args) => {
calls.push([command, ...args]);
return { stdout: '', stderr: '', exitCode: 0 };
};
const program = new Command();
program.exitOverride();
const errors: string[] = [];
const origError = console.error;
console.error = (...args: unknown[]) => {
errors.push(args.join(' '));
};
registerFleetCommand(program, {
runner,
mosaicHome: home,
checkBrokerSocket: async () => false,
});
try {
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
// Refused: no holder/agent starts were issued after the broker attempt.
expect(calls).toEqual([
['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
]);
expect(errors.join('\n')).toContain('broker-absent');
expect(errors.join('\n')).toContain('mosaic fleet install');
} finally {
console.error = origError;
await rm(home, { recursive: true, force: true });
}
});
it('fleet start re-probes the broker on the SECOND invocation — no ActiveState trust (#1292 sticky half)', async () => {
const home = await tempDir();
const rosterPath = join(home, 'fleet', 'roster.yaml');
await mkdir(join(home, 'fleet'), { recursive: true });
await writeFile(
rosterPath,
['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join(
'\n',
),
);
const calls: string[][] = [];
const runner: CommandRunner = async (command, args) => {
calls.push([command, ...args]);
return { stdout: '', stderr: '', exitCode: 0 };
};
const program = new Command();
program.exitOverride();
// Broker socket NEVER appears — the second start must refuse exactly like
// the first; RemainAfterExit-style stale unit state changes nothing
// because the check is the socket, not systemctl.
registerFleetCommand(program, {
runner,
mosaicHome: home,
checkBrokerSocket: async () => false,
});
const errors: string[] = [];
const origError = console.error;
console.error = (...args: unknown[]) => {
errors.push(args.join(' '));
};
try {
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
// Two invocations, each refusing after its own broker attempt:
expect(
calls.filter((c) => c.join(' ') === 'systemctl --user start [email protected]'),
).toHaveLength(0);
expect(errors.filter((e) => e.includes('broker-absent')).length).toBeGreaterThanOrEqual(2);
} finally {
console.error = origError;
await rm(home, { recursive: true, force: true });
}
});
it('waits for an in-flight restart to clear before relaunching (re-entry guard)', async () => {
const home = await tempDir();
const rosterPath = join(home, 'fleet', 'roster.yaml');
@@ -2163,19 +2065,8 @@ describe('fleet install — auto-enable units for boot-survival', () => {
await enableFleetUnits(runner, minimalRoster, {});
expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-lease-broker.service']);
expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-tmux-holder.service']);
expect(calls).toContainEqual(['systemctl', '--user', 'enable', '[email protected]']);
// The broker must be enabled BEFORE the holder and agents: a start of any
// gated runtime without the broker is exactly the #1292 4-second death.
const brokerIndex = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service',
);
const holderIndex = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user enable mosaic-tmux-holder.service',
);
expect(brokerIndex).toBeGreaterThanOrEqual(0);
expect(brokerIndex).toBeLessThan(holderIndex);
});
it('install still succeeds when systemctl enable returns non-zero (non-fatal)', async () => {
+13 -214
View File
@@ -3,11 +3,9 @@ import {
access,
chmod,
copyFile,
lstat,
mkdir,
open,
readFile,
readlink,
stat,
unlink,
writeFile,
@@ -91,8 +89,6 @@ export type SleepFn = (ms: number) => Promise<void>;
export interface FleetCommandDeps {
runner?: CommandRunner;
/** Test seam for the #1292 fleet-start broker preflight (socket presence). */
checkBrokerSocket?: (path: string) => Promise<boolean> | boolean;
/** Injectable interactive runner for commands needing inherited TTY (e.g., `tmux attach`). */
interactiveRunner?: InteractiveRunner;
/**
@@ -801,96 +797,6 @@ export function buildSystemdEnableCommand(unit: string): string[] {
return ['systemctl', '--user', 'enable', unit];
}
/**
* Place a unit file into the ACTIVE systemd user directory, never through a
* symlink (#1292, measured 2026-08-17).
*
* ⚠ SET-INDEPENDENCE (fomo-lin, 2026-08-17): the set of unit names carrying
* by-path residue and the set of unit names this install copies are
* INDEPENDENT. Until 0.0.50 they were disjoint only by accident of which
* units the install happened to name — fomo-lin survived copy-through solely
* because its one by-path symlink (the broker) was the one unit the install
* did NOT copy. Adding the broker to the copy set made the intersection
* non-empty on the first run. Whoever adds a fifth unit to the placement
* list inherits this helper and its unlink step; do not place units with a
* bare copyFile.
*
* A host provisioned by the enable-by-path convention carries a symlink AT
* the unit-name path in ~/.config/systemd/user/ pointing at the shipped
* template under ~/.config/mosaic/systemd/user/. Node's copyFile FOLLOWS
* that link and overwrites the SEED template instead of placing the active
* unit (verified with fs.copyFile on a throwaway systemd user instance) —
* silent, rc=0, and it mutates the directory every later reseed reads from.
* The same measurement showed `systemctl enable <name>` does NOT rewrite an
* existing by-path wants-symlink, so reconciliation must be explicit.
*
* Placement therefore: if the destination is a symlink, unlink it first
* (unlink → copy — copy-then-unlink would mutate the seed and then destroy
* the evidence that it did); then copy. Also removes a stale
* `default.target.wants/<name>` symlink that points outside the active
* directory (readlink — NOT readFile, which follows the link and returns the
* target's CONTENT), so the subsequent enable-by-name recreates it against
* the active copy. Idempotent: on a clean or already-reconciled destination
* every step is a no-op (the copy rewrites identical bytes).
*
* Returns what was done, for assertions and install reporting.
*/
export interface PlaceUnitResult {
readonly unit: string;
readonly destination: string;
/** A symlink at the unit-name path was unlinked (by-path residue). */
readonly unlinkedDestinationSymlink: boolean;
/** A stale wants-symlink pointing outside the active dir was removed. */
readonly removedStaleWantsSymlink: boolean;
}
export async function placeUnitFile(
source: string,
systemdUserDir: string,
unit: string,
): Promise<PlaceUnitResult> {
const destination = join(systemdUserDir, unit);
let unlinkedDestinationSymlink = false;
try {
const destInfo = await lstat(destination);
if (destInfo.isSymbolicLink()) {
await unlink(destination);
unlinkedDestinationSymlink = true;
}
} catch {
// absent destination — nothing to unlink
}
await copyFile(source, destination);
let removedStaleWantsSymlink = false;
const wantsLink = join(systemdUserDir, 'default.target.wants', unit);
try {
const wantsInfo = await lstat(wantsLink);
if (wantsInfo.isSymbolicLink()) {
// readlink — NOT readFile: readFile FOLLOWS the link and returns the
// target file's CONTENT, which is not the question being asked.
let target: string | undefined;
try {
target = await readlink(wantsLink);
} catch {
target = undefined;
}
// Normalize (systemctl writes absolute targets; a relative one resolves
// against the wants dir). A wants-symlink pointing anywhere other than
// the active copy (the by-path convention points at the seed template)
// survives enable-by-name unchanged — remove it so enable recreates it.
if (target !== undefined && resolve(dirname(wantsLink), target) !== destination) {
await unlink(wantsLink);
removedStaleWantsSymlink = true;
}
}
} catch {
// absent wants link — nothing to reconcile
}
return { unit, destination, unlinkedDestinationSymlink, removedStaleWantsSymlink };
}
/**
* Returns the systemctl --user disable command for a given unit.
* Used by `fleet remove` so a removed agent's enabled unit cannot resurrect on
@@ -925,22 +831,6 @@ export async function enableFleetUnits(
let succeeded = 0;
let failed = 0;
// The lease broker ships with the fleet and every gated runtime needs it
// (#1292): seats die at lease registration without it, and no documented
// path ever enabled it. Enabled first — alongside the holder — and the
// unit must have been placed by installFleet's placeUnitFile step.
const brokerResult = await runner(
...splitCommand(buildSystemdEnableCommand('mosaic-lease-broker.service')),
);
if (brokerResult.exitCode === 0) {
succeeded++;
} else {
failed++;
process.stderr.write(
`Warning: could not enable mosaic-lease-broker.service: ${brokerResult.stderr || brokerResult.stdout || 'non-zero exit'}\n`,
);
}
const holderResult = await runner(
...splitCommand(buildSystemdEnableCommand('mosaic-tmux-holder.service')),
);
@@ -1637,7 +1527,7 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
.description('Install local fleet tools and user systemd units')
.option('--no-enable', 'Skip enabling units for boot-survival')
.action(async (opts: { enable?: boolean }) => {
await installFleet(cmd, frameworkRoot, runner);
await installFleet(cmd, frameworkRoot);
// Unit enablement needs agent names only, so it reads either version.
const roster = await loadRosterReadModel(cmd);
await enableFleetUnits(runner, roster, opts);
@@ -1648,7 +1538,7 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
.description('Install local fleet tools and user systemd units')
.option('--no-enable', 'Skip enabling units for boot-survival')
.action(async (opts: { enable?: boolean }) => {
await installFleet(cmd, frameworkRoot, runner);
await installFleet(cmd, frameworkRoot);
// Unit enablement needs agent names only, so it reads either version.
const roster = await loadRosterReadModel(cmd);
await enableFleetUnits(runner, roster, opts);
@@ -1701,37 +1591,6 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
);
return;
}
if (action === 'start') {
// Broker preflight (#1292), re-probed on EVERY invocation: a
// gated runtime started without a live lease broker dies ~4s in
// while the unit reports active (RemainAfterExit) — enabling +
// starting here and then RE-CHECKING the socket refuses loudly
// instead of reporting rc0 over a doomed start. This is the
// second-start check as much as the first: it never trusts unit
// ActiveState.
await runChecked(runner, [
'systemctl',
'--user',
'enable',
'mosaic-lease-broker.service',
]);
await runChecked(runner, [
'systemctl',
'--user',
'start',
'mosaic-lease-broker.service',
]);
if (!(await brokerSocketPresent(deps))) {
console.error(
'[fleet] broker-absent: lease broker socket did not appear after enable+start (#1292).',
);
console.error(
'[fleet] remedy: mosaic fleet install (it reconciles either enable convention)',
);
process.exitCode = 1;
return;
}
}
if (action === 'restart') {
// Serialize the holder+agents teardown/relaunch behind the restart lock
// so a re-entrant restart waits for clean shutdown before relaunching,
@@ -2490,11 +2349,7 @@ export function registerFleetAgentCommands(
});
}
async function installFleet(
cmd: Command,
frameworkRoot: string,
runner: CommandRunner,
): Promise<void> {
async function installFleet(cmd: Command, frameworkRoot: string): Promise<void> {
const activePaths = resolveFleetPaths(cmd.opts<{ mosaicHome: string }>().mosaicHome);
assertDefaultMosaicHomeForSystemd(activePaths.mosaicHome);
// Read model first: every file this function places is roster-independent, and
@@ -2546,40 +2401,18 @@ async function installFleet(
for (const toolPath of executableToolPaths) {
await chmod(toolPath, 0o755);
}
// Unit placement (#1292): every unit goes through placeUnitFile — never a
// bare copyFile — so a by-path-enable symlink at the destination is
// unlinked rather than written through (copy-through would silently
// overwrite the SEED template, measured 2026-08-17). The lease broker unit
// is placed here too: previously the install named three units and omitted
// the broker entirely, which is why no documented path ever enabled it.
const placedUnits = await Promise.all(
[
'mosaic-tmux-holder.service',
'[email protected]',
'[email protected]',
'mosaic-lease-broker.service',
].map((unit) =>
placeUnitFile(join(frameworkRoot, 'systemd', 'user', unit), activePaths.systemdUserDir, unit),
),
await copyFile(
join(frameworkRoot, 'systemd', 'user', 'mosaic-tmux-holder.service'),
join(activePaths.systemdUserDir, 'mosaic-tmux-holder.service'),
);
const reconciled = placedUnits.filter(
(result) => result.unlinkedDestinationSymlink || result.removedStaleWantsSymlink,
await copyFile(
join(frameworkRoot, 'systemd', 'user', '[email protected]'),
join(activePaths.systemdUserDir, '[email protected]'),
);
await copyFile(
join(frameworkRoot, 'systemd', 'user', '[email protected]'),
join(activePaths.systemdUserDir, '[email protected]'),
);
if (reconciled.length > 0) {
console.log(
`Reconciled ${reconciled.length} unit placement(s) from by-path enable residue: ${reconciled.map((r) => r.unit).join(', ')}`,
);
}
// systemd will not see a replaced unit file without a reload; do it once
// after all placements, before any enable call below. runCommand never
// rejects (it resolves exitCode 127 on spawn error), so a plain await with
// an exitCode check matches the rest of this file's systemctl handling.
const reloadResult = await runner(...splitCommand(['systemctl', '--user', 'daemon-reload']));
if (reloadResult.exitCode !== 0) {
process.stderr.write(
`Warning: systemctl --user daemon-reload after unit placement failed (non-systemd host?): ${reloadResult.stderr || reloadResult.stdout || 'non-zero exit'}\n`,
);
}
// On roster v2 the reconciler owns the generated env: `apply` writes it and
// `regen` rebuilds it, both from projectRosterV2AgentGeneratedEnv. Writing it
@@ -2776,40 +2609,6 @@ function splitCommand(command: string[]): [string, string[]] {
return [bin, args];
}
/**
* Lease-broker socket presence for the fleet-start preflight (#1292).
* Resolution precedence matches launch.ts's defaultLeaseBrokerSocket and
* start-agent-session.sh's broker_socket_path: explicit
* MOSAIC_LEASE_BROKER_SOCKET, else $XDG_RUNTIME_DIR/mosaic-lease/broker.sock,
* else /run/user/<uid>/mosaic-lease/broker.sock. Pure filesystem check — this
* deliberately does NOT consult systemd state: a unit can be active
* (RemainAfterExit) with no live socket, and the socket is the thing the
* gated runtime connects to. Injectable via deps for tests.
*/
export function resolveLeaseBrokerSocketForPreflight(
env: NodeJS.ProcessEnv = process.env,
uid: number = typeof process.getuid === 'function' ? process.getuid() : 0,
): string {
if (env['MOSAIC_LEASE_BROKER_SOCKET']) return env['MOSAIC_LEASE_BROKER_SOCKET'];
const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`;
return join(runtimeDir, 'mosaic-lease', 'broker.sock');
}
async function brokerSocketPresent(
deps: FleetCommandDeps,
env: NodeJS.ProcessEnv = process.env,
): Promise<boolean> {
const check = deps.checkBrokerSocket;
if (check) return check(resolveLeaseBrokerSocketForPreflight(env));
try {
const socketPath = resolveLeaseBrokerSocketForPreflight(env);
await access(socketPath, constants.S_IFSOCK);
return true;
} catch {
return false;
}
}
/** All supported fleet profile names. */
export type FleetProfile =
| 'general'
@@ -205,12 +205,6 @@ export async function runLeaseEnforcementDoctorCheck(
message:
`Lease-enforcement hooks (${matchedMarkers.join(', ')}) are wired in ~/.claude/settings.json, but ${reasons.join(' and ')}. ` +
'Every gated tool call will fail closed and BRICK this agent (see #869). ' +
// #1292: one remedy, correct under BOTH enable conventions (by-path on
// the seed template, and copy-then-enable in the active dir). Written
// from the 2026-08-17 symlink measurement: `systemctl enable` by name
// does NOT rewrite an existing by-path wants-symlink, so teaching a
// manual systemctl line here could leave a host with two competing
// wants links. fleet install reconciles either shape.
'Remedy: run `mosaic fleet install` (it reconciles either enable convention), or remove the enforcement hooks from ~/.claude/settings.json.',
'Remediate by activating the lease-broker supervisor (systemd unit + socket) or by removing the enforcement hooks from ~/.claude/settings.json.',
};
}
@@ -459,7 +459,6 @@ describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => {
plan: {
generation: 7,
holder: 'owned',
broker: { unitInstalled: false, socketPresent: false },
agents: [
{
name: 'coder0',
@@ -92,112 +92,6 @@ async function run(command: FleetReconcileCommand, overrides: Partial<FleetRecon
}
describe('fleet roster-owned reconciler', (): void => {
// ── #1292: broker as first-class plan member + broker-first start ordering ──
it('reports broker unit and socket state in the plan (socket is the signal, not unit state)', async (): Promise<void> => {
const result = await run('status', {
statPath: async () => true,
checkBrokerSocket: async () => true,
});
expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: true });
});
it('reports a dead broker as socketPresent=false even when the unit is installed (enabled-but-dead is the #1292 shape)', async (): Promise<void> => {
const result = await run('status', {
statPath: async () => true,
checkBrokerSocket: async () => false,
});
expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: false });
});
it('reports broker-absent when neither seam is present (defaults false, never guesses healthy)', async (): Promise<void> => {
const result = await run('status');
expect(result.plan.broker).toEqual({ unitInstalled: false, socketPresent: false });
});
it('command start enables and starts the broker BEFORE the holder and any agent unit', async (): Promise<void> => {
const calls: string[][] = [];
const result = await run('start', {
runner: async (command, args) => {
calls.push([command, ...args]);
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
},
});
expect(result.lifecycle).toBe('complete');
const brokerEnable = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service',
);
const brokerStart = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service',
);
const holderStart = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user start mosaic-tmux-holder.service',
);
const agentStart = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user start [email protected]',
);
expect(brokerEnable).toBeGreaterThanOrEqual(0);
expect(brokerStart).toBeGreaterThan(brokerEnable);
// Holder start may be absent (holder 'owned' in this fixture); if present it must follow the broker.
if (holderStart >= 0) expect(holderStart).toBeGreaterThan(brokerStart);
expect(agentStart).toBeGreaterThan(brokerStart);
});
it('apply with a running desired agent also enables and starts the broker first', async (): Promise<void> => {
const calls: string[][] = [];
const runningRoster: FleetRosterV2 = {
...roster,
agents: roster.agents.map((agent) =>
agent.name === 'coder0'
? { ...agent, lifecycle: { enabled: true, desiredState: 'running' as const } }
: agent,
),
};
const result = await executeFleetReconcile({
roster: runningRoster,
command: 'apply',
expectedGeneration: 7,
deps: deps({
readRoster: async () => runningRoster,
runner: async (command, args) => {
calls.push([command, ...args]);
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
},
}),
});
expect(result.applied).toBe(true);
const brokerStart = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service',
);
const agentStart = calls.findIndex(
(c) => c.join(' ') === 'systemctl --user start [email protected]',
);
expect(brokerStart).toBeGreaterThanOrEqual(0);
expect(agentStart).toBeGreaterThan(brokerStart);
});
it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise<void> => {
const home = await lockHome();
const fleet = join(home, 'fleet');
@@ -43,10 +43,6 @@ export interface FleetReconcileDeps {
readonly overrideDir?: string;
readonly homeDirectory?: string;
readonly readHolderIdentity?: () => Promise<string>;
/** Test/observation seams for the lease-broker plan member (#1292). */
readonly statPath?: (path: string) => Promise<boolean> | boolean;
readonly checkBrokerSocket?: (path: string) => Promise<boolean> | boolean;
readonly brokerSocketEnv?: NodeJS.ProcessEnv;
readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
@@ -78,17 +74,6 @@ export interface FleetReconcileObservedAgent {
export interface FleetReconcilePlan {
readonly generation: number;
readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
/**
* Lease broker observation (#1292): every gated runtime registers with the
* broker or dies ~4s in a broker not in the plan cannot be reported as
* drifted, which made "broker died an hour ago" and "broker fine"
* produce identical output. `unitInstalled` = unit file present in the
* active dir; `socketPresent` = live broker at the resolved socket path.
*/
readonly broker: {
readonly unitInstalled: boolean;
readonly socketPresent: boolean;
};
readonly agents: readonly FleetReconcileObservedAgent[];
readonly unmanagedSessions: readonly string[];
}
@@ -329,39 +314,6 @@ function isObservational(command: FleetReconcileCommand): boolean {
return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
}
/**
* Observe the lease broker for the plan (#1292). Unit presence via systemctl
* is-system-running is NOT the signal a unit can be enabled-but-dead. The
* authoritative signal is the socket the gated runtimes connect to, matching
* broker-supervisor.ts's `checkBrokerSupervisorHealth` (healthy ===
* socketPresent). Injectable so tests drive every branch without a broker.
*/
async function observeBroker(deps: FleetReconcileDeps): Promise<FleetReconcilePlan['broker']> {
const homeDirectory = deps.homeDirectory ?? homedir();
const env = (deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv;
const uid = typeof process.getuid === 'function' ? process.getuid() : 0;
const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`;
const socketPath =
env['MOSAIC_LEASE_BROKER_SOCKET'] ?? join(runtimeDir, 'mosaic-lease', 'broker.sock');
const configHome = env['XDG_CONFIG_HOME'] ?? join(homeDirectory, '.config');
const unitPath = join(configHome, 'systemd', 'user', 'mosaic-lease-broker.service');
const statPath = deps.statPath;
const checkBrokerSocket = deps.checkBrokerSocket;
let unitInstalled = false;
let socketPresent = false;
try {
unitInstalled = statPath ? await statPath(unitPath) : false;
} catch {
unitInstalled = false;
}
try {
socketPresent = checkBrokerSocket ? await checkBrokerSocket(socketPath) : false;
} catch {
socketPresent = false;
}
return { unitInstalled, socketPresent };
}
async function observeFleet(
roster: FleetRosterV2,
deps: FleetReconcileDeps,
@@ -372,12 +324,10 @@ async function observeFleet(
'-F',
'#{session_name}',
]);
const broker = await observeBroker(deps);
if (sessionsResult.exitCode !== 0) {
return {
generation: roster.generation,
holder: 'missing',
broker,
agents: await observeAgents(roster, deps, new Set<string>()),
unmanagedSessions: [],
};
@@ -400,7 +350,6 @@ async function observeFleet(
return {
generation: roster.generation,
holder,
broker,
agents: await observeAgents(roster, deps, sessions),
unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
};
@@ -568,24 +517,6 @@ async function executeExplicitLifecycle(
}
}
try {
// Broker FIRST (#1292): a gated runtime started without a running lease
// broker dies ~4 seconds in at registration — enable the unit (install
// places it) and start it before any holder/agent lifecycle effect. The
// socket re-check after start is the same probe observeBroker uses, so a
// unit that starts but never produces a socket is caught here, not four
// seconds later inside a doomed seat.
if (request.command === 'start') {
await runChecked(request.deps, 'systemctl', [
'--user',
'enable',
'mosaic-lease-broker.service',
]);
await runChecked(request.deps, 'systemctl', [
'--user',
'start',
'mosaic-lease-broker.service',
]);
}
if (request.command === 'start' && plan.holder === 'missing') {
await runChecked(request.deps, 'systemctl', [
'--user',
@@ -631,12 +562,6 @@ async function applyDesiredLifecycle(
(agent: FleetRosterV2Agent): boolean =>
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
);
// Broker before any running agent, same ordering and reason as the
// command-driven path above (#1292).
if (needsRunningAgent) {
await runChecked(deps, 'systemctl', ['--user', 'enable', 'mosaic-lease-broker.service']);
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-lease-broker.service']);
}
if (needsRunningAgent && plan.holder === 'missing') {
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
}
+166
View File
@@ -0,0 +1,166 @@
#!/usr/bin/env node
// verify-release.mjs — the ONE canonical terminal verification command
// (SDLC-D-034, `pnpm verify:release`).
//
// Publication (.woodpecker/publish.yml `verify` step) is bound to terminal
// verification of the exact commit through this command, which is composed
// from the SAME commands the PR CI pipeline (.woodpecker/ci.yml) runs — CI and
// publish share one semantic checklist:
//
// stage | mirrors ci.yml step | commands
// --------------|---------------------|------------------------------------------
// sanitization | sanitization | verify-sanitized.sh, check-resident-
// | | budget.sh (--self-test + run),
// | | check-test-enumeration.sh
// upgrade-guard | upgrade-guard | test-upgrade-manifest-guard.sh,
// | | test-upgrade-rollback.sh,
// | | test-upgrade-durable-snapshot.sh,
// | | test-install-migration.sh
// typecheck | typecheck | pnpm typecheck (runs the checkout
// | | preflight, then turbo typecheck)
// lint | lint | pnpm lint
// format | format | pnpm format:check
// test | test | pnpm test
// build | publish.yml build | pnpm build
//
// Caller-provided prerequisites (kept at the pipeline level — see the comments
// in .woodpecker/ci.yml): `bash` + `rsync` for the guard stages, `openssl` and
// the pinned @earendil-works/pi-coding-agent for the test stage, and — on the
// postgres path only — the ci-postgres service plus
// `pnpm --filter @mosaicstack/db run db:migrate` before the test stage.
//
// This command works with DATABASE_URL set (CI postgres path) or unset (local
// PGlite path); it never sets, exports, or requires a database itself.
//
// scripts/verify-release.test.mjs enforces that this stage table keeps
// matching .woodpecker/ci.yml step-for-step, so the two surfaces cannot drift
// apart silently.
import { spawnSync } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
export const STAGES = [
{
name: 'sanitization',
// Mirror of the .woodpecker/ci.yml `sanitization` step (minus its
// `apk add` environment prep). Kept as direct command strings here: the
// #1017 test-enumeration guard audits these paths through the ci.yml
// surface, so indirection from ci.yml into this file is not possible.
commands: [
'bash packages/mosaic/framework/tools/quality/scripts/verify-sanitized.sh',
'bash packages/mosaic/framework/tools/quality/scripts/check-resident-budget.sh --self-test',
'bash packages/mosaic/framework/tools/quality/scripts/check-resident-budget.sh',
'bash packages/mosaic/framework/tools/quality/scripts/check-test-enumeration.sh',
],
},
{
name: 'upgrade-guard',
// Mirror of the .woodpecker/ci.yml `upgrade-guard` step (minus its
// `apk add` environment prep).
commands: [
'bash packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh',
'bash packages/mosaic/framework/tools/quality/scripts/test-upgrade-rollback.sh',
'bash packages/mosaic/framework/tools/quality/scripts/test-upgrade-durable-snapshot.sh',
'bash packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh',
],
},
{
// `pnpm typecheck` is `pnpm preflight && turbo run typecheck`, so the
// checkout preflight (scripts/preflight.mjs) is part of this stage exactly
// as it is part of the ci.yml `typecheck` step.
name: 'typecheck',
commands: ['pnpm typecheck'],
},
{
name: 'lint',
commands: ['pnpm lint'],
},
{
name: 'format',
commands: ['pnpm format:check'],
},
{
// Requires `openssl` and the pinned `pi` binary on the pipeline path; see
// the caller-provided prerequisites above.
name: 'test',
commands: ['pnpm test'],
},
{
name: 'build',
commands: ['pnpm build'],
},
];
export function stageByName(name) {
return STAGES.find((stage) => stage.name === name);
}
function missingBinaries(bins) {
return bins.filter(
(bin) => spawnSync('sh', ['-c', `command -v ${bin} >/dev/null 2>&1`]).status !== 0,
);
}
function runCommand(command) {
const result = spawnSync(command, { shell: true, stdio: 'inherit' });
if (result.error) {
console.error(`[verify:release] failed to launch '${command}': ${result.error.message}`);
return false;
}
if (result.status !== 0) {
const reason = result.signal ? `terminated by ${result.signal}` : `exited ${result.status}`;
console.error(`[verify:release] command '${command}' ${reason}`);
return false;
}
return true;
}
// Runs the complete mandatory verification set (or, with --stage <name>, the
// single named stage — used for wiring/smoke-testing, not for gating: only a
// run of every stage is a terminal verification). Fails fast: the first
// failing command aborts with a non-zero exit code. Returns the exit code.
export function verifyRelease({ stages = STAGES } = {}) {
const missing = missingBinaries(['bash', 'rsync']);
if (missing.length > 0) {
console.error(
`[verify:release] FATAL: required binaries missing from PATH: ${missing.join(', ')}. ` +
'The caller provides them (ci-base bakes bash; pipelines apk add rsync).',
);
return 1;
}
for (const stage of stages) {
console.log(`\n[verify:release] === stage: ${stage.name} ===`);
for (const command of stage.commands) {
console.log(`[verify:release] $ ${command}`);
if (!runCommand(command)) {
console.error(
`[verify:release] FATAL: stage '${stage.name}' failed — verification inconclusive`,
);
return 1;
}
}
}
console.log(`\n[verify:release] all ${stages.length} stage(s) passed`);
return 0;
}
function main(argv) {
const stageFlagIndex = argv.indexOf('--stage');
if (stageFlagIndex !== -1) {
const name = argv[stageFlagIndex + 1];
const stage = stageByName(name);
if (!stage) {
console.error(
`[verify:release] unknown stage '${name ?? ''}' — expected one of: ${STAGES.map((entry) => entry.name).join(', ')}`,
);
process.exit(2);
}
process.exit(verifyRelease({ stages: [stage] }));
}
process.exit(verifyRelease());
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main(process.argv.slice(2));
}
+277
View File
@@ -0,0 +1,277 @@
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { createRequire } from 'node:module';
import path from 'node:path';
import test from 'node:test';
import { STAGES } from './verify-release.mjs';
// SDLC-D-034 checkout invariant: publication in .woodpecker/publish.yml is
// bound to exact-commit terminal verification. This suite parses the real
// pipeline files and fails red when the gate is bypassed, weakened, or drifts
// out of sync with the canonical `pnpm verify:release` command.
// Reuse the monorepo's existing YAML parser (@mosaicstack/mosaic's direct
// dependency) instead of adding a root dependency or vendoring a parser.
const mosaicRequire = createRequire(
path.resolve(process.cwd(), 'packages', 'mosaic', 'package.json'),
);
const { parse: parseYaml } = mosaicRequire('yaml');
const publishYmlPath = path.join(process.cwd(), '.woodpecker', 'publish.yml');
const ciYmlPath = path.join(process.cwd(), '.woodpecker', 'ci.yml');
async function readPublishPipeline() {
return parseYaml(await readFile(publishYmlPath, 'utf8'));
}
// A step has an external publication effect when its name starts with
// `publish` or when any command pushes an image to a registry.
function pushesImage(step) {
return (step.commands ?? []).some((command) =>
/(^|\s)(\/kaniko\/executor|docker push)\b|--destination/.test(command),
);
}
function publishEffectSteps(pipeline) {
return Object.entries(pipeline.steps ?? {})
.filter(([name, step]) => name.startsWith('publish') || pushesImage(step))
.map(([name]) => name);
}
// Transitive closure of a step's depends_on graph.
function dependencyClosure(pipeline, stepName, seen = new Set()) {
const dependencies = pipeline.steps?.[stepName]?.depends_on ?? [];
for (const dependency of dependencies) {
if (seen.has(dependency)) continue;
seen.add(dependency);
dependencyClosure(pipeline, dependency, seen);
}
return seen;
}
function verifyCommands(pipeline) {
const verify = pipeline.steps?.verify;
assert.ok(verify, 'publish pipeline must define a `verify` step');
assert.ok(Array.isArray(verify.commands), '`verify` step must have commands');
return verify.commands;
}
function assertCommitIdentityAssertion(commands) {
const text = commands.join('\n');
assert.match(
text,
/CI_COMMIT_SHA/,
'`verify` must compare the provider commit identity (CI_COMMIT_SHA)',
);
assert.match(text, /git rev-parse HEAD/, '`verify` must compare against git rev-parse HEAD');
assert.match(
text,
/exit 1/,
'`verify` must fail closed (exit 1) on identity mismatch or emptiness',
);
}
function assertCanonicalCommand(commands) {
assert.ok(
commands.some((command) => /^pnpm verify:release\b/.test(command.trim())),
'`verify` must run the canonical terminal verification command `pnpm verify:release`',
);
}
function assertPublishGate(pipeline) {
assert.ok(pipeline.steps, 'publish pipeline must define steps');
const commands = verifyCommands(pipeline);
assertCommitIdentityAssertion(commands);
assertCanonicalCommand(commands);
const effects = publishEffectSteps(pipeline);
assert.ok(effects.length > 0, 'publish pipeline must contain publish effect steps to guard');
for (const stepName of effects) {
const step = pipeline.steps[stepName];
assert.ok(
Array.isArray(step.depends_on) && step.depends_on.includes('verify'),
`publish effect '${stepName}' must depend DIRECTLY on the verify step (SDLC-D-034: transitively through build is not enough)`,
);
assert.ok(
dependencyClosure(pipeline, stepName).has('verify'),
`publish effect '${stepName}' must depend on a chain that includes verify`,
);
}
return effects;
}
test('the publish pipeline gates every publish effect behind exact-commit verification', async () => {
const pipeline = await readPublishPipeline();
const effects = assertPublishGate(pipeline);
assert.deepEqual(effects.sort(), [
'build-appservice',
'build-gateway',
'build-web',
'publish-next-npm',
'publish-npm',
]);
});
test('the verify step carries no path/event short-circuit of its own', async () => {
const pipeline = await readPublishPipeline();
// A `when` filter on `verify` would let a publish effect fire on an event
// class that skipped verification — the gate must be unconditional.
assert.equal(pipeline.steps.verify.when, undefined);
});
test('a publish step that bypasses verify fails the gate checker', () => {
// Negative fixture: a plausible publish pipeline where `publish-npm` hangs
// off `build` only and `build` never chains to `verify` — the exact bypass
// class SDLC-D-034 closes. The checker must go red on it.
const bypassingPipeline = `
steps:
install:
image: node:24-alpine
commands:
- pnpm install --frozen-lockfile
verify:
image: node:24-alpine
commands:
- |
if [ -z "$CI_COMMIT_SHA" ] || [ "$CI_COMMIT_SHA" != "$(git rev-parse HEAD)" ]; then
echo "identity mismatch" >&2
exit 1
fi
- pnpm verify:release
depends_on:
- install
build:
image: node:24-alpine
commands:
- pnpm build
depends_on:
- install
publish-npm:
image: node:24-alpine
commands:
- pnpm publish
depends_on:
- build
`;
assert.throws(
() => assertPublishGate(parseYaml(bypassingPipeline)),
/publish-npm.*DIRECTLY.*verify/s,
);
});
test('a publish step chained to verify only transitively fails the gate checker', () => {
// Negative fixture: `build` depends on verify but `publish-npm` does not
// carry the direct edge — weaker than SDLC-D-034 requires of the real DAG.
const transitiveOnlyPipeline = `
steps:
install:
image: node:24-alpine
commands:
- pnpm install --frozen-lockfile
verify:
image: node:24-alpine
commands:
- |
if [ -z "$CI_COMMIT_SHA" ] || [ "$CI_COMMIT_SHA" != "$(git rev-parse HEAD)" ]; then
echo "identity mismatch" >&2
exit 1
fi
- pnpm verify:release
depends_on:
- install
build:
image: node:24-alpine
commands:
- pnpm build
depends_on:
- install
- verify
publish-npm:
image: node:24-alpine
commands:
- pnpm publish
depends_on:
- build
`;
assert.throws(
() => assertPublishGate(parseYaml(transitiveOnlyPipeline)),
/publish-npm.*DIRECTLY.*verify/s,
);
});
test('a verify step without the commit-identity assertion fails the gate checker', () => {
const noIdentityPipeline = `
steps:
verify:
image: node:24-alpine
commands:
- pnpm verify:release
publish-npm:
image: node:24-alpine
commands:
- pnpm publish
depends_on:
- verify
`;
assert.throws(() => assertPublishGate(parseYaml(noIdentityPipeline)), /CI_COMMIT_SHA/);
});
test('the canonical verify:release stages mirror the PR CI pipeline one-for-one', async () => {
const ci = parseYaml(await readFile(ciYmlPath, 'utf8'));
const canonical = Object.fromEntries(STAGES.map((stage) => [stage.name, stage.commands]));
// The complete mandatory set, in gate order.
assert.deepEqual(
STAGES.map((stage) => stage.name),
['sanitization', 'upgrade-guard', 'typecheck', 'lint', 'format', 'test', 'build'],
);
// Guard stages: ci.yml commands minus its `apk add` environment prep must be
// exactly the canonical stage commands (order included).
for (const stageName of ['sanitization', 'upgrade-guard']) {
assert.deepEqual(
ci.steps[stageName].commands.filter((command) => !command.startsWith('apk add')),
canonical[stageName],
`canonical '${stageName}' stage must match the ci.yml step`,
);
}
// pnpm stages: ci.yml commands minus `corepack enable` must be exactly the
// canonical stage commands.
for (const stepName of ['typecheck', 'lint', 'format']) {
assert.deepEqual(
ci.steps[stepName].commands.filter((command) => command !== 'corepack enable'),
canonical[stepName],
`canonical '${stepName}' stage must match the ci.yml step`,
);
}
// The test stage is shared, but ci.yml wraps it in pipeline-level
// prerequisites the canonical command expects its caller to provide
// (SDLC-D-034): the postgres service + readiness wait + db:migrate, openssl,
// and the pinned pi runtime. None of those may be dropped silently.
for (const command of canonical.test) {
assert.ok(
ci.steps.test.commands.includes(command),
`ci.yml test step must run the canonical test stage command '${command}'`,
);
}
for (const fragment of [
'pg_isready -h ci-postgres',
'pnpm --filter @mosaicstack/db run db:migrate',
'npm install -g @earendil-works/[email protected]',
]) {
assert.ok(
ci.steps.test.commands.some((command) => command.includes(fragment)),
`ci.yml test step must keep its pipeline-level prerequisite '${fragment}'`,
);
}
});
test('the root package.json exposes verify:release as the canonical command', async () => {
const packageJson = JSON.parse(await readFile(path.join(process.cwd(), 'package.json'), 'utf8'));
assert.match(packageJson.scripts['verify:release'], /scripts\/verify-release\.mjs/);
});