fix: harden reproducible checkout races

apps/web/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.2",
   "private": true,
   "scripts": {
-    "build": "next build",
+    "build": "node ../../scripts/build-web.mjs",
     "dev": "next dev",
     "lint": "eslint src",
     "typecheck": "tsc --noEmit",

docs/scratchpads/rm-01-reproducible-checkout.md

@@ -30,6 +30,14 @@
 - AC7 DONE: install/store/worktree were on `/home`; full `pnpm -w build` exited 0; `/tmp` usage changed by 4096 bytes during the build (23,805,173,760 → 23,805,177,856 bytes), not materially.
 - AC8 DONE for the implemented path: store resolves under `$HOME`; test/quarantine/build state resolves under the worktree; no implemented component requires a writable path outside `$HOME` or the worktree.
 
+## Continuation evidence
+
+- AC6 DONE: the committed race reproducer was observed RED (`node --test --test-name-pattern='a competing successful installer is not removed by the losing process' scripts/install-hooks.test.mjs`, exit 1/ENOENT), then passed after cleanup became ownership-safe. The losing installer never removes an active hook set or restores Git configuration it did not activate. `pnpm test:checkout` passes 21/21, exit 0, including the original race and a post-rename peer-replacement regression.
+- Generated-state remediation: replaced mtime inference with a source/build-input fingerprint, written only after a serialized successful Next build with unchanged inputs. Failed/interrupted/overlapping builds leave no trusted marker. The fingerprint uses Next's own environment loader, covers resolved `NEXT_PUBLIC_*` values, inherited TypeScript configuration, lock/workspace inputs, and rejects symlink inputs.
+- Baseline: `pnpm typecheck`, `pnpm lint`, and `pnpm format:check` each exit 0. Local `pnpm test` still exits 97 only at the pre-existing Bash `BASH_LINENO` convention guard (#973/#1003), after checkout tests and package tests pass; this is not reported as a green full-suite result.
+- Automated review remediation: resolved findings for peer-hook ownership, stale/failed build markers, build-input changes, expanded environment inputs, inherited TypeScript config, symlink inputs, and overlapping build serialization. Independent PR review remains assigned to rev-974.
+- AC1 and AC4 remain pending fresh-clone/container evidence at this checkpoint.
+
 ## Handoff
 
 1. Keep the newly committed RED tests red until implementing: (a) source-fingerprint marker support for valid incremental `.next` output, and (b) ownership-safe concurrent hook activation.

scripts/build-web.mjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+
+import { sourceFingerprint } from './preflight.mjs';
+
+const scriptRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+function run(command, args, options) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, options);
+    child.once('error', reject);
+    child.once('exit', (code, signal) => {
+      if (code === 0) resolve();
+      else
+        reject(
+          new Error(signal ? `next build terminated by ${signal}` : `next build exited ${code}`),
+        );
+    });
+  });
+}
+
+const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+async function acquireBuildLock(root) {
+  const workRoot = path.join(root, '.mosaic-test-work');
+  const lock = path.join(workRoot, 'web-build.lock');
+  const nonce = randomUUID();
+  const owner = JSON.stringify({ pid: process.pid, nonce });
+  const deadline = Date.now() + 120_000;
+  await mkdir(workRoot, { recursive: true });
+
+  while (Date.now() < deadline) {
+    try {
+      await mkdir(lock);
+      await writeFile(path.join(lock, 'owner.json'), owner, { mode: 0o600 });
+      return async () => {
+        const current = await readFile(path.join(lock, 'owner.json'), 'utf8');
+        if (current !== owner) throw new Error('Web build lock ownership changed before release.');
+        const released = `${lock}.released-${nonce}`;
+        await rename(lock, released);
+        await rm(released, { recursive: true, force: true });
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      let lockOwner;
+      try {
+        lockOwner = JSON.parse(await readFile(path.join(lock, 'owner.json'), 'utf8'));
+      } catch (ownerError) {
+        if (ownerError.code === 'ENOENT') {
+          await delay(25);
+          continue;
+        }
+        throw new Error(`Web build lock is unreadable at ${lock}.`, { cause: ownerError });
+      }
+      try {
+        process.kill(lockOwner.pid, 0);
+      } catch (processError) {
+        if (processError.code !== 'ESRCH') throw processError;
+        const stale = `${lock}.stale-${nonce}`;
+        try {
+          await rename(lock, stale);
+          await rm(stale, { recursive: true, force: true });
+        } catch (renameError) {
+          if (renameError.code !== 'ENOENT') throw renameError;
+        }
+        continue;
+      }
+      await delay(25);
+    }
+  }
+  throw new Error(`Timed out waiting for the web build lock at ${lock}.`);
+}
+
+export async function buildWeb({
+  root = scriptRoot,
+  fingerprint = sourceFingerprint,
+  runBuild = async (webDir) =>
+    run(path.join(webDir, 'node_modules', '.bin', 'next'), ['build'], {
+      cwd: webDir,
+      stdio: 'inherit',
+    }),
+} = {}) {
+  const releaseLock = await acquireBuildLock(root);
+  try {
+    const webDir = path.join(root, 'apps', 'web');
+    const nextDir = path.join(webDir, '.next');
+    const marker = path.join(nextDir, '.mosaic-source-hash');
+    const temporary = `${marker}.${randomUUID()}.tmp`;
+    const before = await fingerprint(root);
+
+    await rm(marker, { force: true });
+    await runBuild(webDir);
+
+    const after = await fingerprint(root);
+    if (after !== before) {
+      throw new Error(
+        'Web build inputs changed during next build; generated output was not certified.',
+      );
+    }
+
+    await mkdir(nextDir, { recursive: true });
+    await writeFile(temporary, `${before}\n`, { mode: 0o600 });
+    await rename(temporary, marker);
+  } finally {
+    await releaseLock();
+  }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  await buildWeb();
+}

scripts/build-web.test.mjs

@@ -0,0 +1,112 @@
+import assert from 'node:assert/strict';
+import { access, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import test from 'node:test';
+
+import { buildWeb } from './build-web.mjs';
+
+const fixtureRoot = path.join(process.cwd(), '.mosaic-test-work', `build-web-${process.pid}`);
+
+async function fixture(name) {
+  const root = path.join(fixtureRoot, name);
+  await mkdir(path.join(root, 'apps', 'web', '.next'), { recursive: true });
+  return root;
+}
+
+async function exists(target) {
+  try {
+    await access(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+test.after(async () => {
+  await rm(fixtureRoot, { recursive: true, force: true });
+});
+
+test('a successful web build atomically publishes its source fingerprint', async () => {
+  const root = await fixture('success');
+  const marker = path.join(root, 'apps', 'web', '.next', '.mosaic-source-hash');
+
+  await buildWeb({ root, fingerprint: async () => 'certified', runBuild: async () => {} });
+
+  assert.equal(await readFile(marker, 'utf8'), 'certified\n');
+});
+
+test('a failed web build leaves no certification marker', async () => {
+  const root = await fixture('failure');
+  const marker = path.join(root, 'apps', 'web', '.next', '.mosaic-source-hash');
+  await writeFile(marker, 'stale\n');
+
+  await assert.rejects(
+    buildWeb({
+      root,
+      fingerprint: async () => 'before',
+      runBuild: async () => {
+        throw new Error('build failed');
+      },
+    }),
+    /build failed/,
+  );
+
+  assert.equal(await exists(marker), false);
+});
+
+test('overlapping web builds are serialized while the marker remains absent', async () => {
+  const root = await fixture('overlap');
+  const marker = path.join(root, 'apps', 'web', '.next', '.mosaic-source-hash');
+  await writeFile(marker, 'stale\n');
+  let releaseFirst;
+  let secondEntered = false;
+  const firstEntered = new Promise((resolve) => {
+    releaseFirst = resolve;
+  });
+  let markFirstEntered;
+  const firstStarted = new Promise((resolve) => {
+    markFirstEntered = resolve;
+  });
+
+  const first = buildWeb({
+    root,
+    fingerprint: async () => 'certified',
+    runBuild: async () => {
+      markFirstEntered();
+      await firstEntered;
+    },
+  });
+  await firstStarted;
+  const second = buildWeb({
+    root,
+    fingerprint: async () => 'certified',
+    runBuild: async () => {
+      secondEntered = true;
+    },
+  });
+  await new Promise((resolve) => setTimeout(resolve, 75));
+  assert.equal(secondEntered, false);
+  assert.equal(await exists(marker), false);
+
+  releaseFirst();
+  await Promise.all([first, second]);
+  assert.equal(secondEntered, true);
+  assert.equal(await readFile(marker, 'utf8'), 'certified\n');
+});
+
+test('inputs changed during a web build are not certified', async () => {
+  const root = await fixture('changed-inputs');
+  const marker = path.join(root, 'apps', 'web', '.next', '.mosaic-source-hash');
+  const fingerprints = ['before', 'after'];
+
+  await assert.rejects(
+    buildWeb({
+      root,
+      fingerprint: async () => fingerprints.shift(),
+      runBuild: async () => {},
+    }),
+    /inputs changed during next build/,
+  );
+
+  assert.equal(await exists(marker), false);
+});

scripts/install-hooks.mjs

@@ -72,28 +72,6 @@ export async function installHooks({
   activateHooks = async () => {
     await run('git', ['config', 'core.hooksPath', '.husky/_'], { cwd: root, stdio: 'inherit' });
   },
-  readHooksPath = async () => {
-    try {
-      const { stdout } = await execFileAsync('git', ['config', '--get', 'core.hooksPath'], {
-        cwd: root,
-      });
-      return stdout.trim() || null;
-    } catch (error) {
-      if (error.code === 1) return null;
-      throw error;
-    }
-  },
-  restoreHooksPath = async (previous) => {
-    if (previous === null) {
-      try {
-        await execFileAsync('git', ['config', '--unset', 'core.hooksPath'], { cwd: root });
-      } catch (error) {
-        if (error.code !== 5) throw error;
-      }
-    } else {
-      await execFileAsync('git', ['config', 'core.hooksPath', previous], { cwd: root });
-    }
-  },
 } = {}) {
   if (disabled) return;
 
@@ -103,8 +81,6 @@ export async function installHooks({
   const stagingRepo = path.join(root, '.mosaic-test-work', `husky-stage-${nonce}`);
   const stagingHooks = path.join(stagingRepo, '.husky');
   const quarantined = path.join(quarantineRoot, `${path.basename(root)}-${nonce}`);
-
-  const previousHooksPath = await readHooksPath();
   await mkdir(huskyDir, { recursive: true });
   await mkdir(quarantineRoot, { recursive: true });
 
@@ -142,13 +118,6 @@ export async function installHooks({
     }
   } catch (error) {
     const cleanupFailures = [];
-    try {
-      if ((await pathExists(active)) && !previousComplete) {
-        await rename(active, `${quarantined}-failed`);
-      }
-    } catch (cleanupError) {
-      cleanupFailures.push(`active hooks: ${cleanupError.message}`);
-    }
     try {
       if (await pathExists(stagingRepo)) {
         await rename(stagingRepo, `${quarantined}-staging`);
@@ -156,11 +125,6 @@ export async function installHooks({
     } catch (cleanupError) {
       cleanupFailures.push(`staging hooks: ${cleanupError.message}`);
     }
-    try {
-      await restoreHooksPath(previousHooksPath);
-    } catch (cleanupError) {
-      cleanupFailures.push(`core.hooksPath: ${cleanupError.message}`);
-    }
     const cleanup =
       cleanupFailures.length === 0
         ? 'No partial hook set was activated.'

scripts/install-hooks.test.mjs

@@ -54,7 +54,7 @@ test('an interrupted install quarantines the partial active hook set and fails l
   );
 
   assert.equal(await exists(path.join(root, '.husky', '_')), false);
-  assert.equal(restoredHooksPath, null);
+  assert.equal(restoredHooksPath, 'not-called');
   const quarantined = await readdir(quarantineRoot);
   assert.equal(quarantined.length, 1);
 });
@@ -138,6 +138,38 @@ test('a competing successful installer is not removed by the losing process', as
   assert.equal(restored, false);
 });
 
+test("a competing installer that replaces this installer's active set is preserved", async () => {
+  const root = await fixture('concurrent-after-rename');
+  const active = path.join(root, '.husky', '_');
+  const activeShim = path.join(active, 'h');
+  let restored = false;
+
+  await assert.rejects(
+    installHooks({
+      root,
+      quarantineRoot: path.join(fixtureRoot, 'concurrent-after-rename-quarantine'),
+      runHusky: async (stagingHooks) => {
+        await mkdir(path.join(stagingHooks, '_'), { recursive: true });
+        await writeFile(path.join(stagingHooks, '_', 'h'), 'ours');
+      },
+      activateHooks: async () => {
+        await rm(active, { recursive: true, force: true });
+        await mkdir(active, { recursive: true });
+        await writeFile(activeShim, 'peer');
+        throw new Error('our activation lost to peer');
+      },
+      readHooksPath: async () => null,
+      restoreHooksPath: async () => {
+        restored = true;
+      },
+    }),
+    /Hook installation failed/,
+  );
+
+  assert.equal(await readFile(activeShim, 'utf8'), 'peer');
+  assert.equal(restored, false);
+});
+
 test('an explicit interactive HUSKY=0 opt-out preserves existing hooks without running installer', async () => {
   const root = await fixture('disabled');
   const activeShim = path.join(root, '.husky', '_', 'h');

scripts/preflight.mjs

@@ -1,13 +1,17 @@
 #!/usr/bin/env node
 
 import { constants } from 'node:fs';
-import { access, lstat, readdir } from 'node:fs/promises';
+import { access, lstat, readFile, readdir } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { createRequire } from 'node:module';
 import { fileURLToPath } from 'node:url';
 import path from 'node:path';
 
 export const MISSING_DEPS_EXIT = 42;
 export const GENERATED_STATE_EXIT = 43;
 
+const scriptRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
 async function entries(root) {
   const result = [];
   async function walk(current) {
@@ -28,17 +32,90 @@ async function entries(root) {
   return result;
 }
 
-async function fileMtimeRange(paths) {
-  let newest = 0;
-  let oldest = Number.POSITIVE_INFINITY;
-  for (const target of paths) {
-    const stats = await lstat(target);
-    if (stats.isFile()) {
-      newest = Math.max(newest, stats.mtimeMs);
-      oldest = Math.min(oldest, stats.mtimeMs);
+const webSourceRoots = (root) => [
+  path.join(root, 'apps', 'web', 'src'),
+  path.join(root, 'apps', 'web', 'public'),
+  path.join(root, 'apps', 'web', 'next-env.d.ts'),
+  path.join(root, 'apps', 'web', 'next.config.ts'),
+  path.join(root, 'apps', 'web', 'postcss.config.mjs'),
+  path.join(root, 'apps', 'web', 'package.json'),
+  path.join(root, 'apps', 'web', 'tsconfig.json'),
+  path.join(root, 'packages', 'design-tokens', 'src'),
+  path.join(root, 'packages', 'design-tokens', 'package.json'),
+  path.join(root, 'packages', 'design-tokens', 'tsconfig.json'),
+  path.join(root, 'package.json'),
+  path.join(root, 'tsconfig.base.json'),
+  path.join(root, 'pnpm-lock.yaml'),
+  path.join(root, 'pnpm-workspace.yaml'),
+  path.join(root, 'turbo.json'),
+];
+
+// next.config.ts currently reads no server-only environment. Add any future
+// server-side build inputs here; all resolved NEXT_PUBLIC_* inputs are automatic.
+const serverBuildEnvironmentKeys = [];
+
+function publicBuildEnvironment(root) {
+  const webDir = path.join(root, 'apps', 'web');
+  const requireFromWeb = createRequire(path.join(scriptRoot, 'apps', 'web', 'package.json'));
+  const requireFromNext = createRequire(requireFromWeb.resolve('next/package.json'));
+  const { loadEnvConfig, resetEnv, updateInitialEnv } = requireFromNext('@next/env');
+  const originalEnvironment = { ...process.env };
+  updateInitialEnv(originalEnvironment);
+  try {
+    const { combinedEnv } = loadEnvConfig(webDir, false, { info() {}, error() {} }, true);
+    return Object.fromEntries(
+      Object.entries(combinedEnv).filter(
+        ([key, value]) =>
+          value !== undefined &&
+          (key.startsWith('NEXT_PUBLIC_') || serverBuildEnvironmentKeys.includes(key)),
+      ),
+    );
+  } finally {
+    resetEnv();
+  }
+}
+
+export async function sourceFingerprint(root = process.cwd()) {
+  const files = [];
+  for (const sourceRoot of webSourceRoots(root)) {
+    try {
+      const stats = await lstat(sourceRoot);
+      if (stats.isSymbolicLink()) {
+        throw new Error(
+          `Web build input must not be a symbolic link: ${path.relative(root, sourceRoot)}`,
+        );
+      }
+      if (stats.isFile()) files.push(sourceRoot);
+      if (stats.isDirectory()) {
+        for (const target of await entries(sourceRoot)) {
+          const targetStats = await lstat(target);
+          if (targetStats.isSymbolicLink()) {
+            throw new Error(
+              `Web build input must not be a symbolic link: ${path.relative(root, target)}`,
+            );
+          }
+          if (targetStats.isFile()) files.push(target);
+        }
+      }
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
     }
   }
-  return { newest, oldest: Number.isFinite(oldest) ? oldest : 0 };
+
+  const digest = createHash('sha256');
+  for (const [key, value] of Object.entries(publicBuildEnvironment(root)).sort()) {
+    digest.update(`env:${key}\0${value.length}\0${value}\0`);
+  }
+  for (const target of files.sort()) {
+    const contents = await readFile(target);
+    digest.update(path.relative(root, target).split(path.sep).join('/'));
+    digest.update('\0');
+    digest.update(String(contents.length));
+    digest.update('\0');
+    digest.update(contents);
+    digest.update('\0');
+  }
+  return digest.digest('hex');
 }
 
 export async function runPreflight({ root = process.cwd(), uid = process.getuid?.() } = {}) {
@@ -59,6 +136,17 @@ export async function runPreflight({ root = process.cwd(), uid = process.getuid?
     };
   }
 
+  const buildLock = path.join(root, '.mosaic-test-work', 'web-build.lock');
+  try {
+    await lstat(buildLock);
+    return {
+      code: GENERATED_STATE_EXIT,
+      message: `MOSAIC_PREFLIGHT_GENERATED_STATE: web build is in progress or interrupted at ${buildLock}; wait for it to finish or rerun pnpm build to recover the stale lock`,
+    };
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error;
+  }
+
   const nextDir = path.join(root, 'apps', 'web', '.next');
   let generated = [];
   try {
@@ -76,34 +164,19 @@ export async function runPreflight({ root = process.cwd(), uid = process.getuid?
       }
     }
 
-    const sourceRoots = [
-      path.join(root, 'apps', 'web', 'src'),
-      path.join(root, 'apps', 'web', 'next.config.ts'),
-      path.join(root, 'apps', 'web', 'package.json'),
-      path.join(root, 'apps', 'web', 'tsconfig.json'),
-    ];
-    const source = [];
-    for (const sourceRoot of sourceRoots) {
-      try {
-        const stats = await lstat(sourceRoot);
-        source.push(sourceRoot);
-        if (stats.isDirectory()) source.push(...(await entries(sourceRoot)));
-      } catch (error) {
-        if (error.code !== 'ENOENT') throw error;
-      }
+    let generatedFingerprint = null;
+    try {
+      generatedFingerprint = (
+        await readFile(path.join(nextDir, '.mosaic-source-hash'), 'utf8')
+      ).trim();
+    } catch (error) {
+      if (error.code !== 'ENOENT') throw error;
     }
-
-    const generatedTypes = generated.filter((target) =>
-      target.startsWith(path.join(nextDir, 'types') + path.sep),
-    );
-    const sourceMtime = await fileMtimeRange(source);
-    const generatedMtime = await fileMtimeRange(generatedTypes);
-    const stale =
-      source.length > 0 && generatedTypes.length > 0 && sourceMtime.newest > generatedMtime.oldest;
+    const stale = generatedFingerprint !== (await sourceFingerprint(root));
     if (foreign.length > 0 || stale) {
       const reasons = [
         foreign.length > 0 ? `foreign-owned paths: ${foreign.slice(0, 3).join(', ')}` : '',
-        stale ? 'generated output is older than web source/configuration' : '',
+        stale ? 'generated source fingerprint does not match web source/configuration' : '',
       ].filter(Boolean);
       return {
         code: GENERATED_STATE_EXIT,

scripts/preflight.test.mjs

@@ -98,6 +98,42 @@ test('a generated marker mismatch is identified separately from source errors',
   assert.match(result.message, /pnpm clean:generated/);
 });
 
+test('the source fingerprint includes inherited TypeScript configuration', async () => {
+  const root = await fixture('inherited-typescript-config');
+  const config = path.join(root, 'tsconfig.base.json');
+  await writeFile(config, '{"compilerOptions":{"strict":true}}\n');
+  const first = await sourceFingerprint(root);
+  await writeFile(config, '{"compilerOptions":{"strict":false}}\n');
+  const second = await sourceFingerprint(root);
+
+  assert.notEqual(first, second);
+});
+
+test('the source fingerprint rejects symbolic-link build inputs', async () => {
+  const root = await fixture('symbolic-source');
+  await writeFile(path.join(root, 'outside.ts'), 'export default 1;\n');
+  await symlink(path.join(root, 'outside.ts'), path.join(root, 'apps', 'web', 'src', 'linked.ts'));
+
+  await assert.rejects(sourceFingerprint(root), /must not be a symbolic link/);
+});
+
+test('the source fingerprint includes expanded public web build environment', async () => {
+  const root = await fixture('public-build-environment');
+  const envFile = path.join(root, 'apps', 'web', '.env.production');
+  await writeFile(
+    envFile,
+    'RM01_GATEWAY_URL=https://one.example\nNEXT_PUBLIC_RM01_URL=$RM01_GATEWAY_URL\n',
+  );
+  const first = await sourceFingerprint(root);
+  await writeFile(
+    envFile,
+    'RM01_GATEWAY_URL=https://two.example\nNEXT_PUBLIC_RM01_URL=$RM01_GATEWAY_URL\n',
+  );
+  const second = await sourceFingerprint(root);
+
+  assert.notEqual(first, second);
+});
+
 test('a matching generation marker accepts incremental output with mixed mtimes', async () => {
   const root = await fixture('incremental-next');
   await installRequiredBins(root);