feat(fleet): mosaic fleet regen — regenerate roster-derived projections (PR3 of #791) (#813)

packages/mosaic/src/fleet/fleet-reconciler.ts

@@ -584,6 +584,29 @@ function defaultValidateRoster(
   };
 }
 
+/**
+ * The single roster-v2 → generated-projection mapping. Both the reconciler's
+ * apply path ({@link defaultPrepareProjections}) and the recovery-framed
+ * `mosaic fleet regen` command derive their generated env from THIS one
+ * function, so the two paths can never drift (the #791 single-SSOT invariant).
+ * Pure: no IO, no clock — a deterministic function of the roster alone.
+ */
+export function projectRosterV2AgentGeneratedEnv(
+  roster: FleetRosterV2,
+  agent: FleetRosterV2Agent,
+): Readonly<Record<string, string>> {
+  return {
+    MOSAIC_AGENT_NAME: agent.name,
+    MOSAIC_AGENT_CLASS: agent.className,
+    MOSAIC_AGENT_RUNTIME: agent.runtime,
+    MOSAIC_AGENT_MODEL: agent.model,
+    MOSAIC_AGENT_REASONING: agent.reasoning,
+    MOSAIC_AGENT_TOOL_POLICY: agent.toolPolicy,
+    MOSAIC_AGENT_WORKDIR: agent.workingDirectory,
+    MOSAIC_TMUX_SOCKET: roster.tmux.socketName,
+  };
+}
+
 function defaultPrepareProjections(
   request: FleetReconcileRequest,
 ): (roster: FleetRosterV2) => Promise<readonly PreparedAgentEnvironmentProjection[]> {
@@ -596,16 +619,7 @@ function defaultPrepareProjections(
             mosaicHome,
             agentEnvDir: join(mosaicHome, 'fleet', 'agents'),
             agentName: agent.name,
-            generated: {
-              MOSAIC_AGENT_NAME: agent.name,
-              MOSAIC_AGENT_CLASS: agent.className,
-              MOSAIC_AGENT_RUNTIME: agent.runtime,
-              MOSAIC_AGENT_MODEL: agent.model,
-              MOSAIC_AGENT_REASONING: agent.reasoning,
-              MOSAIC_AGENT_TOOL_POLICY: agent.toolPolicy,
-              MOSAIC_AGENT_WORKDIR: agent.workingDirectory,
-              MOSAIC_TMUX_SOCKET: roster.tmux.socketName,
-            },
+            generated: projectRosterV2AgentGeneratedEnv(roster, agent),
           }),
       ),
     );
@@ -620,62 +634,193 @@ function mosaicHomeFor(deps: FleetReconcileDeps): string {
   return deps.mosaicHome ?? join(homedir(), '.config', 'mosaic');
 }
 
-/** Acquires a private lock only after proving the canonical managed path. */
+/** Acquires the private reconcile lock only after proving the canonical managed path. */
 export function acquirePrivateReconcileLock(
   mosaicHome: string,
   openLock: typeof open = open,
+): () => Promise<() => Promise<void>> {
+  return acquirePrivateManagedRosterLock(
+    mosaicHome,
+    'roster.yaml.reconcile.lock',
+    'Another roster reconciliation is in progress.',
+    openLock,
+  );
+}
+
+/**
+ * Acquires the private roster MUTATION lock (`roster.yaml.mutation.lock`) with the
+ * exact same ownership-proof discipline as the reconcile lock. Exposed so the
+ * recovery-framed `mosaic fleet regen` can serialize its projection rewrites
+ * against agent CRUD — which holds this very lock while it rewrites the roster and
+ * the same derived projections — reusing this hardened acquirer rather than
+ * reimplementing it.
+ *
+ * The release PROVES ownership (device/inode + token) before unlinking, so a lock
+ * that was already cleared and replaced by the time release runs is never deleted
+ * out from under the new owner (it fails closed as `lock-cleanup-failed` instead),
+ * and that cleanup fault is propagated to the caller rather than silently dropped.
+ * CRUD contends on the identical path with a plain empty-file `wx` create; the two
+ * never co-own it (whoever wins `wx` owns it; the loser gets `concurrent-mutation`),
+ * so the token this writes is only ever read back by the same regen invocation that
+ * wrote it.
+ *
+ * SCOPE of the guarantee (matches the merged reconcile lock exactly — see
+ * {@link acquirePrivateManagedRosterLock}): the ownership proof closes the
+ * *reachable* window — a stale-lock reaper or operator cleared our lock and another
+ * writer took it BEFORE our release began — not the residual sub-instruction window
+ * between the final check and the path-based `unlink`. Closing that residual fully
+ * requires an fd-held advisory lock adopted by every fleet writer (CRUD, reconcile,
+ * regen), which is a cross-cutting mechanism change out of scope for this
+ * projection-only recovery command; it is unreachable within the `wx` writer
+ * protocol regardless (no Mosaic writer removes a lock it does not own, so only
+ * external interference can vacate our inode mid-release).
+ */
+export function acquirePrivateRosterMutationLock(
+  mosaicHome: string,
+  openLock: typeof open = open,
+): () => Promise<() => Promise<void>> {
+  return acquirePrivateManagedRosterLock(
+    mosaicHome,
+    'roster.yaml.mutation.lock',
+    'Another roster mutation is in progress.',
+    openLock,
+  );
+}
+
+/**
+ * Shared body for the ownership-proving managed roster locks. Acquires `lockLeaf`
+ * under `<mosaicHome>/fleet` with a private `wx` create, writes an ownership token,
+ * and returns a release that re-proves device/inode + token before unlinking so it
+ * can never remove a lock it no longer owns. Non-blocking: throws `concurrent-mutation`
+ * (with `busyMessage`) on contention rather than waiting.
+ */
+function acquirePrivateManagedRosterLock(
+  mosaicHome: string,
+  lockLeaf: string,
+  busyMessage: string,
+  openLock: typeof open,
 ): () => Promise<() => Promise<void>> {
   const fleetDir = join(mosaicHome, 'fleet');
-  const lockPath = join(fleetDir, 'roster.yaml.reconcile.lock');
+  const lockPath = join(fleetDir, lockLeaf);
+  // The message-producing helpers below serve BOTH managed locks, so every fault
+  // names the actual lock file (`fleet/<leaf>`) — an operator needs to know WHICH
+  // lock is stale, not a hardcoded "reconciliation lock".
+  const lockLabel = `fleet/${lockLeaf}`;
   return async (): Promise<() => Promise<void>> => {
     await assertPrivateManagedDirectory(mosaicHome);
     await assertPrivateManagedDirectory(fleetDir);
-    await assertSafeLockLeafIfPresent(lockPath);
+    await assertSafeLockLeafIfPresent(lockPath, lockLabel);
 
     let handle: FileHandle;
     try {
       handle = await openLock(lockPath, 'wx', 0o600);
     } catch (error: unknown) {
       if (isCode(error, 'EEXIST')) {
-        await assertSafeLockLeafIfPresent(lockPath);
-        throw new FleetReconcileError(
-          'concurrent-mutation',
-          'Another roster reconciliation is in progress.',
-        );
+        await assertSafeLockLeafIfPresent(lockPath, lockLabel);
+        throw new FleetReconcileError('concurrent-mutation', busyMessage);
       }
-      throw new FleetReconcileError('lock-io-failed', 'The reconciliation lock cannot be created.');
+      throw new FleetReconcileError('lock-io-failed', `The ${lockLabel} lock cannot be created.`);
     }
 
+    // Identity of the lock WE exclusively created (the `wx` create guaranteed it is
+    // ours). Captured up front so both the release closure and the init-failure
+    // cleanup below can prove ownership by device/inode before touching the file.
+    const created = await handle.stat().catch((): undefined => undefined);
+
     const token = randomUUID();
+    let tokenPersisted = false;
     try {
       await handle.writeFile(`${token}\n`, 'utf8');
-      const opened = await handle.stat();
+      tokenPersisted = true;
       await handle.close();
-      await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'unsafe-lock');
+      if (!created)
+        throw new FleetReconcileError(
+          'lock-io-failed',
+          `The ${lockLabel} lock cannot be initialized.`,
+        );
+      await assertLockOwnership(
+        lockPath,
+        created.dev,
+        created.ino,
+        token,
+        'unsafe-lock',
+        lockLabel,
+      );
       return async (): Promise<void> => {
         try {
-          await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
-          await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
+          await assertLockOwnership(
+            lockPath,
+            created.dev,
+            created.ino,
+            token,
+            'lock-cleanup-failed',
+            lockLabel,
+          );
           await unlink(lockPath);
         } catch (error: unknown) {
           if (error instanceof FleetReconcileError) throw error;
-          throw new FleetReconcileError(
-            'lock-cleanup-failed',
-            'The reconciliation lock cleanup failed.',
-          );
+          throw new FleetReconcileError('lock-cleanup-failed', `The ${lockLabel} cleanup failed.`);
         }
       };
     } catch (error: unknown) {
       await handle.close().catch((): void => {});
+      // Best-effort: remove the lock THIS invocation created so a transient init
+      // fault does not strand a lock that would block future regen and agent CRUD.
+      // dev/ino-guarded so it can never delete a replacement; swallowed so cleanup
+      // failure never masks the initialization error being surfaced. The token is
+      // passed only once persisted, so the fallback path (when the post-create stat
+      // failed and dev/ino is unavailable) can still prove ownership by content.
+      await removeOwnedLockLeafBestEffort(lockPath, created, tokenPersisted ? token : undefined);
       if (error instanceof FleetReconcileError) throw error;
       throw new FleetReconcileError(
         'lock-io-failed',
-        'The reconciliation lock cannot be initialized.',
+        `The ${lockLabel} lock cannot be initialized.`,
       );
     }
   };
 }
 
+/**
+ * Best-effort removal of a managed lock leaf THIS process created, used only on the
+ * initialization-failure path. Two independent ownership proofs, so a lock that was
+ * already replaced is never deleted:
+ *
+ *   - primary: the captured device/inode of the file we exclusively `wx`-created; or
+ *   - fallback: when the post-create stat itself failed (dev/ino unavailable), the
+ *     random `ownershipToken` we persisted — only OUR lock carries it, so a CRUD
+ *     (empty) or a differently-tokened replacement is never removed.
+ *
+ * Every fault is swallowed because the caller is already surfacing the initialization
+ * error and a leftover lock is recoverable via the documented runbook. If BOTH proofs
+ * are unavailable (stat failed AND the token write never landed) the lock is left in
+ * place rather than risk deleting another writer's file — a doubly-degenerate case
+ * requiring two independent fs faults on a just-created fd.
+ */
+async function removeOwnedLockLeafBestEffort(
+  lockPath: string,
+  created: { dev: number; ino: number } | undefined,
+  ownershipToken: string | undefined,
+): Promise<void> {
+  try {
+    const current = await lstat(lockPath);
+    if (!current.isFile() || current.isSymbolicLink()) return;
+    if (created) {
+      if (current.dev === created.dev && current.ino === created.ino) {
+        await unlink(lockPath);
+      }
+      return;
+    }
+    if (ownershipToken !== undefined) {
+      const contents = await readFile(lockPath, 'utf8');
+      if (contents.trim() === ownershipToken) {
+        await unlink(lockPath);
+      }
+    }
+  } catch {
+    // Swallowed: leftover lock is recoverable; do not mask the init error.
+  }
+}
+
 async function assertPrivateManagedDirectory(path: string): Promise<void> {
   try {
     const metadata = await lstat(path);
@@ -691,16 +836,16 @@ async function assertPrivateManagedDirectory(path: string): Promise<void> {
   }
 }
 
-async function assertSafeLockLeafIfPresent(lockPath: string): Promise<void> {
+async function assertSafeLockLeafIfPresent(lockPath: string, lockLabel: string): Promise<void> {
   try {
     const metadata = await lstat(lockPath);
     if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
-      throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unsafe.');
+      throw new FleetReconcileError('unsafe-lock', `The ${lockLabel} lock path is unsafe.`);
     }
   } catch (error: unknown) {
     if (isCode(error, 'ENOENT')) return;
     if (error instanceof FleetReconcileError) throw error;
-    throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unavailable.');
+    throw new FleetReconcileError('unsafe-lock', `The ${lockLabel} lock path is unavailable.`);
   }
 }
 
@@ -710,6 +855,7 @@ async function assertLockOwnership(
   inode: number,
   token: string,
   failureCode: 'unsafe-lock' | 'lock-cleanup-failed',
+  lockLabel: string,
 ): Promise<void> {
   try {
     const metadata = await lstat(lockPath);
@@ -720,24 +866,21 @@ async function assertLockOwnership(
       metadata.dev !== device ||
       metadata.ino !== inode
     ) {
-      throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
+      throw new FleetReconcileError(failureCode, `The ${lockLabel} ownership changed.`);
     }
     const handle = await open(lockPath, constants.O_RDONLY | constants.O_NOFOLLOW);
     try {
       const opened = await handle.stat();
       const contents = await handle.readFile({ encoding: 'utf8' });
       if (opened.dev !== device || opened.ino !== inode || contents !== `${token}\n`) {
-        throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
+        throw new FleetReconcileError(failureCode, `The ${lockLabel} ownership changed.`);
       }
     } finally {
       await handle.close();
     }
   } catch (error: unknown) {
     if (error instanceof FleetReconcileError) throw error;
-    throw new FleetReconcileError(
-      failureCode,
-      'The reconciliation lock ownership cannot be proven.',
-    );
+    throw new FleetReconcileError(failureCode, `The ${lockLabel} ownership cannot be proven.`);
   }
 }