feat(lease-broker): C3 broker supervisor artifacts + health predicate (#869)

#828 shipped fail-closed enforcement hooks (mutator-gate.py,
receipt-observer-client.py) with nothing supervising daemon.py or
guaranteeing its socket exists before a gated runtime starts. This adds
the activation-side supervisor mechanism:

- a systemd --user unit (mosaic-lease-broker.service) mirroring the
  existing tmux-fleet unit convention, with RuntimeDirectoryMode=0700
  so it satisfies daemon.py's secure_parent() fail-closed check
- a wrapper script (start-lease-broker.sh) that resolves the broker
  socket with the same precedence as defaultLeaseBrokerSocket() in
  commands/launch.ts, and colocates the state file next to it
- broker-supervisor.ts: deterministic path resolution, an idempotent
  apply() that materializes the unit/wrapper/daemon sources (reseed-safe,
  never runs systemctl or starts the daemon), and a health predicate
  (isBrokerSupervisorHealthy / checkBrokerSupervisorHealth) other cards
  (e.g. the C1 activation probe) can call

All tests use temp dirs/fakes; nothing here installs, enables, or
starts anything on this host.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

packages/mosaic/framework/systemd/user/mosaic-lease-broker.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Mosaic lease broker daemon (framework tools/lease-broker/daemon.py)
+Documentation=https://git.mosaicstack.dev/mosaicstack/stack
+After=default.target
+
+[Service]
+Type=simple
+# The broker socket lives under the runtime directory so it disappears with
+# the user session instead of surviving as stale state across logins.
+# daemon.py's secure_parent() fails closed unless this directory is exactly
+# 0700, so RuntimeDirectoryMode is not cosmetic.
+RuntimeDirectory=mosaic-lease
+RuntimeDirectoryMode=0700
+# Remove loader and noninteractive-shell controls before ExecStart loads env,
+# matching the tmux fleet units in this same directory.
+UnsetEnvironment=LD_PRELOAD BASH_ENV ENV
+ExecStart=/usr/bin/env -i HOME=%h PATH=/usr/bin:/bin XDG_RUNTIME_DIR=%t /bin/bash --noprofile --norc %h/.config/mosaic/tools/lease-broker/start-lease-broker.sh
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target

packages/mosaic/framework/tools/lease-broker/start-lease-broker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Supervisor entry point for the Mosaic lease broker daemon (issue #869, C3).
+#
+# Resolves the broker socket path with the SAME precedence as
+# `defaultLeaseBrokerSocket` in `packages/mosaic/src/commands/launch.ts`, so a
+# gated runtime launched through that client always finds the socket this
+# supervisor creates:
+#   1. an explicit MOSAIC_LEASE_BROKER_SOCKET
+#   2. "$XDG_RUNTIME_DIR/mosaic-lease/broker.sock"
+#   3. "/run/user/<uid>/mosaic-lease/broker.sock"
+#
+# The state file is colocated next to the socket (same directory,
+# "state.json"), mirroring how the broker already colocates its per-session
+# generation files beside the socket.
+#
+# This script never installs, enables, or starts the systemd unit that calls
+# it; it is only ever invoked BY that unit (or by a human/test harness that
+# passes its own HOME/XDG_RUNTIME_DIR).
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd)
+
+if [ -n "${MOSAIC_LEASE_BROKER_SOCKET:-}" ]; then
+  SOCKET="$MOSAIC_LEASE_BROKER_SOCKET"
+else
+  RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
+  SOCKET="$RUNTIME_DIR/mosaic-lease/broker.sock"
+fi
+STATE="$(dirname -- "$SOCKET")/state.json"
+
+exec python3 "$SCRIPT_DIR/daemon.py" --socket "$SOCKET" --state "$STATE"

packages/mosaic/src/lease-broker/broker-supervisor.spec.ts

@@ -0,0 +1,237 @@
+import { createServer, type Server } from 'node:net';
+import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import {
+  applyBrokerSupervisor,
+  checkBrokerSupervisorHealth,
+  isBrokerSupervisorHealthy,
+  resolveBrokerSupervisorPaths,
+  resolveLeaseBrokerSocketPath,
+  type BrokerSupervisorPaths,
+} from './broker-supervisor.js';
+
+const REAL_FRAMEWORK_ROOT = new URL('../../framework/', import.meta.url).pathname;
+
+const cleanupDirs: string[] = [];
+const cleanupServers: Server[] = [];
+
+async function tempDir(prefix: string): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), prefix));
+  cleanupDirs.push(dir);
+  return dir;
+}
+
+afterEach(async () => {
+  for (const server of cleanupServers.splice(0)) {
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+  }
+  for (const dir of cleanupDirs.splice(0)) {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+describe('resolveLeaseBrokerSocketPath', () => {
+  it('honors an explicit MOSAIC_LEASE_BROKER_SOCKET override', () => {
+    expect(resolveLeaseBrokerSocketPath({ MOSAIC_LEASE_BROKER_SOCKET: '/tmp/explicit.sock' })).toBe(
+      '/tmp/explicit.sock',
+    );
+  });
+
+  it('falls back to $XDG_RUNTIME_DIR/mosaic-lease/broker.sock', () => {
+    expect(resolveLeaseBrokerSocketPath({ XDG_RUNTIME_DIR: '/run/user/1000' })).toBe(
+      join('/run/user/1000', 'mosaic-lease', 'broker.sock'),
+    );
+  });
+
+  it('falls back to /run/user/<uid>/mosaic-lease/broker.sock as a last resort', () => {
+    expect(resolveLeaseBrokerSocketPath({}, 4242)).toBe(
+      join('/run/user', '4242', 'mosaic-lease', 'broker.sock'),
+    );
+  });
+
+  it('prefers the explicit override over XDG_RUNTIME_DIR', () => {
+    expect(
+      resolveLeaseBrokerSocketPath({
+        MOSAIC_LEASE_BROKER_SOCKET: '/explicit.sock',
+        XDG_RUNTIME_DIR: '/run/user/1000',
+      }),
+    ).toBe('/explicit.sock');
+  });
+});
+
+describe('resolveBrokerSupervisorPaths', () => {
+  it('colocates the state file next to the resolved socket', () => {
+    const paths = resolveBrokerSupervisorPaths({
+      mosaicHome: '/home/x/.config/mosaic',
+      frameworkRoot: '/repo/framework',
+      env: { XDG_RUNTIME_DIR: '/run/user/1000' },
+    });
+    expect(paths.socketPath).toBe(join('/run/user/1000', 'mosaic-lease', 'broker.sock'));
+    expect(paths.statePath).toBe(join('/run/user/1000', 'mosaic-lease', 'state.json'));
+  });
+
+  it('targets the systemd --user dir under the given home, not mosaicHome', () => {
+    const paths = resolveBrokerSupervisorPaths({
+      mosaicHome: '/somewhere-else/.config/mosaic',
+      frameworkRoot: '/repo/framework',
+      homeDir: '/home/canary',
+      env: {},
+      uid: 0,
+    });
+    expect(paths.systemdUserDir).toBe(join('/home/canary', '.config', 'systemd', 'user'));
+    expect(paths.unitTargetPath).toBe(
+      join('/home/canary', '.config', 'systemd', 'user', 'mosaic-lease-broker.service'),
+    );
+  });
+
+  it('is a pure function: identical options resolve to identical paths', () => {
+    const options = {
+      mosaicHome: '/h/.config/mosaic',
+      frameworkRoot: '/repo/framework',
+      env: { XDG_RUNTIME_DIR: '/run/user/1000' },
+    };
+    expect(resolveBrokerSupervisorPaths(options)).toEqual(resolveBrokerSupervisorPaths(options));
+  });
+});
+
+describe('applyBrokerSupervisor', () => {
+  async function fakePaths(): Promise<BrokerSupervisorPaths> {
+    const home = await tempDir('mosaic-broker-supervisor-home-');
+    const mosaicHome = join(home, '.config', 'mosaic');
+    const runtimeDir = await tempDir('mosaic-broker-supervisor-runtime-');
+    return resolveBrokerSupervisorPaths({
+      mosaicHome,
+      frameworkRoot: REAL_FRAMEWORK_ROOT,
+      homeDir: home,
+      env: { XDG_RUNTIME_DIR: runtimeDir },
+    });
+  }
+
+  it('renders a unit that references the installed wrapper script and hardens the runtime dir', async () => {
+    const paths = await fakePaths();
+    const unitSource = await readFile(paths.unitSourcePath, 'utf8');
+    expect(unitSource).toContain('ExecStart=');
+    expect(unitSource).toContain('%h/.config/mosaic/tools/lease-broker/start-lease-broker.sh');
+    expect(unitSource).toContain('RuntimeDirectory=mosaic-lease');
+    expect(unitSource).toContain('RuntimeDirectoryMode=0700');
+    expect(unitSource).toContain('Restart=on-failure');
+    expect(unitSource).toContain('WantedBy=default.target');
+    // No ambient environment file preload, matching the other fleet units'
+    // strict-parsing convention.
+    expect(unitSource).not.toMatch(/^Environment(File)?=/m);
+  });
+
+  it('materializes the unit, wrapper script, and daemon sources on first apply', async () => {
+    const paths = await fakePaths();
+
+    const result = await applyBrokerSupervisor(paths);
+
+    expect(result.installedFiles).toContain(paths.unitTargetPath);
+    expect(result.installedFiles).toContain(paths.wrapperTargetPath);
+    for (const target of paths.daemonTargetPaths) {
+      expect(result.installedFiles).toContain(target);
+    }
+
+    const unitTargetContent = await readFile(paths.unitTargetPath, 'utf8');
+    const unitSourceContent = await readFile(paths.unitSourcePath, 'utf8');
+    expect(unitTargetContent).toBe(unitSourceContent);
+
+    const wrapperMode = (await stat(paths.wrapperTargetPath)).mode & 0o777;
+    expect(wrapperMode).toBe(0o755);
+
+    for (const target of paths.daemonTargetPaths) {
+      await expect(stat(target)).resolves.toBeDefined();
+    }
+  });
+
+  it('is idempotent: applying twice reproduces identical files with no error', async () => {
+    const paths = await fakePaths();
+
+    await applyBrokerSupervisor(paths);
+    const firstUnit = await readFile(paths.unitTargetPath, 'utf8');
+    const firstWrapper = await readFile(paths.wrapperTargetPath, 'utf8');
+    const firstWrapperMode = (await stat(paths.wrapperTargetPath)).mode & 0o777;
+
+    await expect(applyBrokerSupervisor(paths)).resolves.toBeDefined();
+
+    const secondUnit = await readFile(paths.unitTargetPath, 'utf8');
+    const secondWrapper = await readFile(paths.wrapperTargetPath, 'utf8');
+    const secondWrapperMode = (await stat(paths.wrapperTargetPath)).mode & 0o777;
+
+    expect(secondUnit).toBe(firstUnit);
+    expect(secondWrapper).toBe(firstWrapper);
+    expect(secondWrapperMode).toBe(firstWrapperMode);
+  });
+
+  it('never touches the real host: only writes under the supplied temp dirs', async () => {
+    const paths = await fakePaths();
+    await applyBrokerSupervisor(paths);
+    expect(paths.systemdUserDir.startsWith(tmpdir())).toBe(true);
+    expect(paths.mosaicHome.startsWith(tmpdir())).toBe(true);
+  });
+});
+
+describe('checkBrokerSupervisorHealth / isBrokerSupervisorHealthy', () => {
+  async function fakeHealthPaths(): Promise<
+    Pick<BrokerSupervisorPaths, 'unitTargetPath' | 'socketPath'>
+  > {
+    const runtimeDir = await tempDir('mosaic-broker-supervisor-health-');
+    await mkdir(join(runtimeDir, 'systemd-user'), { recursive: true });
+    return {
+      unitTargetPath: join(runtimeDir, 'systemd-user', 'mosaic-lease-broker.service'),
+      socketPath: join(runtimeDir, 'broker.sock'),
+    };
+  }
+
+  it('reports unhealthy when neither the unit nor the socket exist', async () => {
+    const paths = await fakeHealthPaths();
+
+    const health = await checkBrokerSupervisorHealth(paths);
+
+    expect(health).toEqual({ unitInstalled: false, socketPresent: false, healthy: false });
+    expect(await isBrokerSupervisorHealthy(paths)).toBe(false);
+  });
+
+  it('reports unhealthy when the unit is installed but no socket is listening', async () => {
+    const paths = await fakeHealthPaths();
+    await writeFile(paths.unitTargetPath, '[Unit]\n');
+
+    const health = await checkBrokerSupervisorHealth(paths);
+
+    expect(health.unitInstalled).toBe(true);
+    expect(health.socketPresent).toBe(false);
+    expect(health.healthy).toBe(false);
+  });
+
+  it('reports healthy=true once a real Unix socket exists at the resolved path, and false again once removed', async () => {
+    const paths = await fakeHealthPaths();
+
+    const server = createServer();
+    cleanupServers.push(server);
+    await new Promise<void>((resolve, reject) => {
+      server.once('error', reject);
+      server.listen(paths.socketPath, resolve);
+    });
+
+    expect(await isBrokerSupervisorHealthy(paths)).toBe(true);
+    const health = await checkBrokerSupervisorHealth(paths);
+    expect(health.socketPresent).toBe(true);
+    expect(health.healthy).toBe(true);
+
+    await new Promise<void>((resolve) => server.close(() => resolve()));
+    await rm(paths.socketPath, { force: true });
+
+    expect(await isBrokerSupervisorHealthy(paths)).toBe(false);
+  });
+
+  it('does not confuse a stale regular file at the socket path with a live socket', async () => {
+    const paths = await fakeHealthPaths();
+    await writeFile(paths.socketPath, 'not actually a socket');
+
+    expect(await isBrokerSupervisorHealthy(paths)).toBe(false);
+  });
+});

packages/mosaic/src/lease-broker/broker-supervisor.ts

@@ -0,0 +1,223 @@
+/**
+ * Activation-side supervisor for the Mosaic lease broker (issue #869, Point-1
+ * C3). #828 shipped fail-closed enforcement hooks (`mutator-gate.py`,
+ * `receipt-observer-client.py`) with nothing that guaranteed `daemon.py` was
+ * running or that its socket existed before a gated runtime started. This
+ * module:
+ *
+ *  - resolves the broker socket/state paths and the on-disk locations of the
+ *    supervisor artifacts, deterministically and consistently with
+ *    `defaultLeaseBrokerSocket` in `../commands/launch.ts`;
+ *  - idempotently applies (materializes) a systemd `--user` unit plus the
+ *    wrapper script and daemon sources it execs, mirroring the tmux fleet
+ *    unit convention in `framework/systemd/user/`;
+ *  - exposes a health predicate other cards (e.g. the C1 activation probe)
+ *    can call to learn whether a broker supervisor is present and healthy.
+ *
+ * `applyBrokerSupervisor` only writes files under the paths it is given. It
+ * never runs `systemctl`, never starts `daemon.py`, and never touches a real
+ * host's `~/.config` unless the caller explicitly resolves paths there.
+ * Enabling/starting the unit is a separate, later, out-of-scope step.
+ */
+import { chmod, copyFile, mkdir, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+const UNIT_NAME = 'mosaic-lease-broker.service';
+const WRAPPER_SCRIPT_NAME = 'start-lease-broker.sh';
+
+/** Co-located modules `daemon.py` imports at runtime; kept alongside it. */
+const DAEMON_SOURCE_FILE_NAMES = [
+  'daemon.py',
+  'lease_generation.py',
+  'normative_fragments.py',
+  'receipt_challenge.py',
+  'receipt_observer.py',
+] as const;
+
+export interface ResolveBrokerSupervisorPathsOptions {
+  /** `~/.config/mosaic` (or an override) — where installed tool copies live. */
+  mosaicHome: string;
+  /** Root of the checked-out `framework/` directory (canonical file source). */
+  frameworkRoot: string;
+  /** Defaults to `process.env`; pass a fake for tests. */
+  env?: NodeJS.ProcessEnv;
+  /** Defaults to `os.homedir()`; pass a temp dir in tests. */
+  homeDir?: string;
+  /** Defaults to `process.getuid()` (or 0); pass a fake for tests. */
+  uid?: number;
+}
+
+export interface BrokerSupervisorPaths {
+  readonly mosaicHome: string;
+  readonly frameworkRoot: string;
+  readonly systemdUserDir: string;
+  readonly leaseBrokerToolsDir: string;
+  readonly unitSourcePath: string;
+  readonly unitTargetPath: string;
+  readonly wrapperSourcePath: string;
+  readonly wrapperTargetPath: string;
+  readonly daemonSourcePaths: readonly string[];
+  readonly daemonTargetPaths: readonly string[];
+  /**
+   * Resolved with the same precedence as `defaultLeaseBrokerSocket` in
+   * `../commands/launch.ts`: an explicit `MOSAIC_LEASE_BROKER_SOCKET`, else
+   * `$XDG_RUNTIME_DIR/mosaic-lease/broker.sock`, else
+   * `/run/user/<uid>/mosaic-lease/broker.sock`.
+   */
+  readonly socketPath: string;
+  /** Colocated next to the socket, matching the broker's own generation-file convention. */
+  readonly statePath: string;
+}
+
+/**
+ * Resolve the lease broker socket path alone, with the same precedence as
+ * `defaultLeaseBrokerSocket` in `../commands/launch.ts`. Exported so callers
+ * (and tests) can assert the two stay in agreement without importing the CLI
+ * command module.
+ */
+export function resolveLeaseBrokerSocketPath(
+  env: NodeJS.ProcessEnv = process.env,
+  uid: number = typeof process.getuid === 'function' ? process.getuid() : 0,
+): string {
+  const explicit = env['MOSAIC_LEASE_BROKER_SOCKET'];
+  if (explicit) return explicit;
+  const runtimeDir = env['XDG_RUNTIME_DIR'];
+  if (runtimeDir) return join(runtimeDir, 'mosaic-lease', 'broker.sock');
+  return join('/run/user', String(uid), 'mosaic-lease', 'broker.sock');
+}
+
+/** Resolve every path the supervisor apply/health functions need, deterministically. */
+export function resolveBrokerSupervisorPaths(
+  options: ResolveBrokerSupervisorPathsOptions,
+): BrokerSupervisorPaths {
+  const { mosaicHome, frameworkRoot } = options;
+  const env = options.env ?? process.env;
+  const homeDir = options.homeDir ?? homedir();
+  const systemdUserDir = join(homeDir, '.config', 'systemd', 'user');
+  const leaseBrokerToolsDir = join(mosaicHome, 'tools', 'lease-broker');
+  const frameworkLeaseBrokerDir = join(frameworkRoot, 'tools', 'lease-broker');
+  const socketPath = resolveLeaseBrokerSocketPath(env, options.uid);
+  const statePath = join(dirname(socketPath), 'state.json');
+
+  return {
+    mosaicHome,
+    frameworkRoot,
+    systemdUserDir,
+    leaseBrokerToolsDir,
+    unitSourcePath: join(frameworkRoot, 'systemd', 'user', UNIT_NAME),
+    unitTargetPath: join(systemdUserDir, UNIT_NAME),
+    wrapperSourcePath: join(frameworkLeaseBrokerDir, WRAPPER_SCRIPT_NAME),
+    wrapperTargetPath: join(leaseBrokerToolsDir, WRAPPER_SCRIPT_NAME),
+    daemonSourcePaths: DAEMON_SOURCE_FILE_NAMES.map((name) => join(frameworkLeaseBrokerDir, name)),
+    daemonTargetPaths: DAEMON_SOURCE_FILE_NAMES.map((name) => join(leaseBrokerToolsDir, name)),
+    socketPath,
+    statePath,
+  };
+}
+
+export interface ApplyBrokerSupervisorResult {
+  readonly installedFiles: readonly string[];
+}
+
+/**
+ * Idempotently materialize the supervisor unit, its wrapper script, and the
+ * daemon sources it execs. Safe to call on every reseed: every write is a
+ * deterministic overwrite of the same target path from the same source, so a
+ * second call reproduces identical bytes/modes and never errors.
+ *
+ * Never runs `systemctl`; the caller decides separately whether/when to
+ * `daemon-reload`/`enable`/`start` the installed unit.
+ */
+export async function applyBrokerSupervisor(
+  paths: BrokerSupervisorPaths,
+): Promise<ApplyBrokerSupervisorResult> {
+  await mkdir(paths.leaseBrokerToolsDir, { recursive: true });
+  await mkdir(paths.systemdUserDir, { recursive: true });
+
+  const installedFiles: string[] = [];
+
+  for (let index = 0; index < paths.daemonSourcePaths.length; index += 1) {
+    const source = paths.daemonSourcePaths[index];
+    const target = paths.daemonTargetPaths[index];
+    if (source === undefined || target === undefined) continue;
+    await copyFile(source, target);
+    await chmod(target, 0o644);
+    installedFiles.push(target);
+  }
+
+  await copyFile(paths.wrapperSourcePath, paths.wrapperTargetPath);
+  await chmod(paths.wrapperTargetPath, 0o755);
+  installedFiles.push(paths.wrapperTargetPath);
+
+  await copyFile(paths.unitSourcePath, paths.unitTargetPath);
+  await chmod(paths.unitTargetPath, 0o644);
+  installedFiles.push(paths.unitTargetPath);
+
+  return { installedFiles };
+}
+
+export interface BrokerSupervisorHealth {
+  /** Whether the systemd unit file has been materialized at its target path. */
+  readonly unitInstalled: boolean;
+  /** Whether a Unix domain socket currently exists at the resolved socket path. */
+  readonly socketPresent: boolean;
+  /**
+   * The signal other cards (e.g. C1's activation probe) should treat as
+   * "a broker supervisor is present and healthy". Presence of a live socket
+   * is the authoritative signal: a gated runtime can only ever succeed by
+   * connecting to it, so this is what fail-closed callers must check.
+   */
+  readonly healthy: boolean;
+}
+
+/**
+ * Report the supervisor's on-disk/health signals. Never throws for an
+ * absent unit or socket — both simply report `false`; unexpected filesystem
+ * errors (permission issues, etc.) still propagate.
+ */
+export async function checkBrokerSupervisorHealth(
+  paths: Pick<BrokerSupervisorPaths, 'unitTargetPath' | 'socketPath'>,
+): Promise<BrokerSupervisorHealth> {
+  const [unitInstalled, socketPresent] = await Promise.all([
+    pathExists(paths.unitTargetPath),
+    isUnixSocket(paths.socketPath),
+  ]);
+  return { unitInstalled, socketPresent, healthy: socketPresent };
+}
+
+/** Convenience boolean form of {@link checkBrokerSupervisorHealth} for simple call sites. */
+export async function isBrokerSupervisorHealthy(
+  paths: Pick<BrokerSupervisorPaths, 'unitTargetPath' | 'socketPath'>,
+): Promise<boolean> {
+  return (await checkBrokerSupervisorHealth(paths)).healthy;
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    if (isEnoent(error)) return false;
+    throw error;
+  }
+}
+
+async function isUnixSocket(path: string): Promise<boolean> {
+  try {
+    const info = await stat(path);
+    return info.isSocket();
+  } catch (error) {
+    if (isEnoent(error)) return false;
+    throw error;
+  }
+}
+
+function isEnoent(error: unknown): boolean {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    'code' in error &&
+    (error as NodeJS.ErrnoException).code === 'ENOENT'
+  );
+}