fix(federation): harness CRIT bugs — admin bootstrap auth + peer FK + boot deadline (review remediation)

CRIT-1: Replace nonexistent x-admin-key header with Authorization: Bearer <token>;
add bootstrapAdmin() to call POST /api/bootstrap/setup on each pristine gateway
before any admin-guarded endpoint is used.

CRIT-2: Fix cross-gateway peer FK violation — peer keypair is now created on
Server B first (so the grant FK resolves against B's own federation_peers table),
then Server A creates its own keypair and redeems the enrollment token at B.

HIGH-3: waitForStack() now polls both gateways in parallel via Promise.all, each
with an independent deadline, so a slow gateway-a cannot starve gateway-b's budget.

MED-4: seed() throws immediately with a clear error if scenario !== 'all';
per-variant narrowing deferred to M3-11 with explicit JSDoc note.

Also: remove ADMIN_API_KEY (no such path in AdminGuard) from compose, replace
with ADMIN_BOOTSTRAP_PASSWORD; add BETTER_AUTH_URL production-code limitation
as a TODO in the README.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tools/federation-harness/seed.ts

@@ -8,13 +8,20 @@
  * What this script does:
  *  1. (Optional) Boots the compose stack if --boot flag is passed.
  *  2. Waits for both gateways to be healthy.
- *  3. Creates three grants on Server B matching the M3 acceptance test scenarios:
+ *  3. Bootstraps an admin user + token on each gateway via POST /api/bootstrap/setup.
+ *  4. Creates three grants on Server B matching the M3 acceptance test scenarios:
  *     - Scope variant A: tasks + notes, include_personal: true
  *     - Scope variant B: tasks only, include_teams: ['T1'], exclude T2
  *     - Scope variant C: tasks + credentials in resources, credentials excluded (sanity)
- *  4. For each grant, walks the enrollment flow so Server A ends up with
- *     an active peer + cert + sealed key.
- *  5. Inserts representative test tasks/notes/credentials on Server B.
+ *  5. For each grant, walks the full enrollment flow:
+ *       a. Server B creates a peer keypair (represents the requesting side).
+ *       b. Server B creates the grant referencing that peer.
+ *       c. Server B issues an enrollment token.
+ *       d. Server A creates its own peer keypair (represents its view of B).
+ *       e. Server A redeems the enrollment token at Server B's enrollment endpoint,
+ *          submitting A's CSR → receives signed cert back.
+ *       f. Server A stores the cert on its peer record → peer becomes active.
+ *  6. Inserts representative test tasks/notes/credentials on Server B.
  *
  * IMPORTANT: This script uses the real admin REST API — no direct DB writes.
  * It exercises the full enrollment flow as M3 acceptance tests will.
@@ -34,8 +41,16 @@ const COMPOSE_FILE = resolve(__dirname, 'docker-compose.two-gateways.yml');
 /** Base URLs as seen from the host machine (mapped host ports). */
 const SERVER_A_URL = process.env['GATEWAY_A_URL'] ?? 'http://localhost:14001';
 const SERVER_B_URL = process.env['GATEWAY_B_URL'] ?? 'http://localhost:14002';
-const ADMIN_KEY_A = process.env['ADMIN_KEY_A'] ?? 'harness-admin-key-a';
-const ADMIN_KEY_B = process.env['ADMIN_KEY_B'] ?? 'harness-admin-key-b';
+
+/**
+ * Bootstrap passwords used when calling POST /api/bootstrap/setup on each
+ * gateway. Each gateway starts with zero users and requires a one-time setup
+ * call before any admin-guarded endpoints can be used.
+ */
+const ADMIN_BOOTSTRAP_PASSWORD_A =
+  process.env['ADMIN_BOOTSTRAP_PASSWORD_A'] ?? 'harness-admin-password-a';
+const ADMIN_BOOTSTRAP_PASSWORD_B =
+  process.env['ADMIN_BOOTSTRAP_PASSWORD_B'] ?? 'harness-admin-password-b';
 
 const READINESS_TIMEOUT_MS = 120_000;
 const READINESS_POLL_MS = 3_000;
@@ -82,7 +97,7 @@ export const SCOPE_VARIANT_C = {
 interface AdminFetchOptions {
   method?: string;
   body?: unknown;
-  adminKey: string;
+  adminToken: string;
 }
 
 interface PeerRecord {
@@ -107,9 +122,18 @@ interface EnrollmentRedeemResult {
   certChainPem: string;
 }
 
+interface BootstrapResult {
+  adminUserId: string;
+  adminToken: string;
+}
+
 export interface SeedResult {
   serverAUrl: string;
   serverBUrl: string;
+  adminTokenA: string;
+  adminTokenB: string;
+  adminUserIdA: string;
+  adminUserIdB: string;
   grants: {
     variantA: GrantRecord;
     variantB: GrantRecord;
@@ -124,13 +148,18 @@ export interface SeedResult {
 
 // ─── HTTP helpers ─────────────────────────────────────────────────────────────
 
+/**
+ * Authenticated admin fetch. Sends `Authorization: Bearer <adminToken>` which
+ * is the only path supported by AdminGuard (DB-backed sha256 token lookup).
+ * No `x-admin-key` header path exists in the gateway.
+ */
 async function adminFetch<T>(baseUrl: string, path: string, opts: AdminFetchOptions): Promise<T> {
   const url = `${baseUrl}${path}`;
   const res = await fetch(url, {
     method: opts.method ?? 'GET',
     headers: {
       'Content-Type': 'application/json',
-      'x-admin-key': opts.adminKey,
+      Authorization: `Bearer ${opts.adminToken}`,
     },
     body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,
   });
@@ -143,6 +172,74 @@ async function adminFetch<T>(baseUrl: string, path: string, opts: AdminFetchOpti
   return res.json() as Promise<T>;
 }
 
+// ─── Admin bootstrap ──────────────────────────────────────────────────────────
+
+/**
+ * Bootstrap an admin user on a pristine gateway.
+ *
+ * Steps:
+ *  1. GET /api/bootstrap/status — confirms needsSetup === true.
+ *  2. POST /api/bootstrap/setup with { name, email, password } — returns
+ *     { user, token: { plaintext } }.
+ *
+ * The harness assumes a fresh DB. If needsSetup is false the harness fails
+ * fast with a clear error rather than proceeding with an unknown token.
+ */
+async function bootstrapAdmin(
+  baseUrl: string,
+  label: string,
+  password: string,
+): Promise<BootstrapResult> {
+  console.log(`[seed] Bootstrapping admin on ${label} (${baseUrl})...`);
+
+  // 1. Check status
+  const statusRes = await fetch(`${baseUrl}/api/bootstrap/status`);
+  if (!statusRes.ok) {
+    throw new Error(
+      `[seed] GET ${baseUrl}/api/bootstrap/status → ${statusRes.status.toString()}`,
+    );
+  }
+  const status = (await statusRes.json()) as { needsSetup: boolean };
+
+  if (!status.needsSetup) {
+    throw new Error(
+      `[seed] ${label} at ${baseUrl} already has users (needsSetup=false). ` +
+        `The harness requires a pristine database. Run 'docker compose down -v' to reset.`,
+    );
+  }
+
+  // 2. Bootstrap
+  const setupRes = await fetch(`${baseUrl}/api/bootstrap/setup`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      name: `Harness Admin (${label})`,
+      email: `harness-admin-${label.toLowerCase()}@example.invalid`,
+      password,
+    }),
+  });
+
+  if (!setupRes.ok) {
+    const body = await setupRes.text().catch(() => '(no body)');
+    throw new Error(
+      `[seed] POST ${baseUrl}/api/bootstrap/setup → ${setupRes.status.toString()}: ${body}`,
+    );
+  }
+
+  const result = (await setupRes.json()) as {
+    user: { id: string };
+    token: { plaintext: string };
+  };
+
+  console.log(`[seed]   ${label} admin user: ${result.user.id}`);
+  console.log(`[seed]   ${label} admin token: ${result.token.plaintext.slice(0, 8)}...`);
+
+  return {
+    adminUserId: result.user.id,
+    adminToken: result.token.plaintext,
+  };
+}
+
 // ─── Readiness probe ──────────────────────────────────────────────────────────
 
 async function waitForGateway(baseUrl: string, label: string): Promise<void> {
@@ -156,7 +253,7 @@ async function waitForGateway(baseUrl: string, label: string): Promise<void> {
         console.log(`[seed] ${label} is ready (${baseUrl})`);
         return;
       }
-      lastError = `HTTP ${res.status}`;
+      lastError = `HTTP ${res.status.toString()}`;
     } catch (err) {
       lastError = err instanceof Error ? err.message : String(err);
     }
@@ -164,48 +261,62 @@ async function waitForGateway(baseUrl: string, label: string): Promise<void> {
   }
 
   throw new Error(
-    `[seed] ${label} did not become ready within ${READINESS_TIMEOUT_MS}ms — last error: ${lastError}`,
+    `[seed] ${label} did not become ready within ${READINESS_TIMEOUT_MS.toString()}ms — last error: ${lastError}`,
   );
 }
 
 // ─── Enrollment flow ──────────────────────────────────────────────────────────
 
 /**
- * Walk the full enrollment flow for one grant:
- *  1. Create a peer keypair on Server A (generates CSR).
- *  2. Create a grant on Server B referencing the peer.
- *  3. Generate an enrollment token on Server B.
- *  4. Redeem the token on Server B with A's CSR → get cert back.
- *  5. Store the cert on Server A's peer record.
+ * Walk the full enrollment flow for one grant.
  *
- * Returns the activated grant record + peer info.
+ * The correct two-sided flow (matching the data model's FK semantics):
+ *
+ *  1. On Server B: POST /api/admin/federation/peers/keypair
+ *       → peerId_B (Server B's peer record representing the requesting side)
+ *  2. On Server B: POST /api/admin/federation/grants with peerId: peerId_B
+ *       → grant (FK to Server B's own federation_peers table — no violation)
+ *  3. On Server B: POST /api/admin/federation/grants/:id/tokens
+ *       → enrollmentUrl pointing back to Server B
+ *  4. On Server A: POST /api/admin/federation/peers/keypair
+ *       → peerId_A + csrPem_A (Server A's local record of Server B)
+ *  5. Server A → Server B: POST enrollmentUrl with { csrPem: csrPem_A }
+ *       → certPem signed by Server B's CA
+ *  6. On Server A: PATCH /api/admin/federation/peers/:peerId_A/cert with certPem
+ *       → Server A's peer record transitions to active
+ *
+ * Returns the activated grant (from Server B) and Server A's peer record.
  */
 async function enrollGrant(opts: {
   label: string;
   subjectUserId: string;
   scope: unknown;
+  adminTokenA: string;
+  adminTokenB: string;
+  serverAUrl: string;
+  serverBUrl: string;
 }): Promise<{ grant: GrantRecord; peer: PeerRecord & { grantId: string } }> {
-  const { label, subjectUserId, scope } = opts;
+  const { label, subjectUserId, scope, adminTokenA, adminTokenB, serverAUrl, serverBUrl } = opts;
   console.log(`\n[seed] Enrolling grant for scope variant ${label}...`);
 
-  // 1. Create peer keypair on Server A
-  const peer = await adminFetch<PeerRecord>(SERVER_A_URL, '/api/admin/federation/peers/keypair', {
+  // 1. Create peer keypair on Server B (represents the requesting peer from B's perspective)
+  const peerB = await adminFetch<PeerRecord>(serverBUrl, '/api/admin/federation/peers/keypair', {
     method: 'POST',
-    adminKey: ADMIN_KEY_A,
+    adminToken: adminTokenB,
     body: {
-      commonName: `harness-peer-${label.toLowerCase()}`,
-      displayName: `Harness Peer ${label}`,
-      endpointUrl: `${SERVER_B_URL}`,
+      commonName: `harness-peer-${label.toLowerCase()}-from-b`,
+      displayName: `Harness Peer ${label} (Server A as seen from B)`,
+      endpointUrl: serverAUrl,
     },
   });
-  console.log(`[seed]   Created peer on A: ${peer.peerId}`);
+  console.log(`[seed]   Created peer on B: ${peerB.peerId}`);
 
-  // 2. Create grant on Server B
-  const grant = await adminFetch<GrantRecord>(SERVER_B_URL, '/api/admin/federation/grants', {
+  // 2. Create grant on Server B referencing B's own peer record
+  const grant = await adminFetch<GrantRecord>(serverBUrl, '/api/admin/federation/grants', {
     method: 'POST',
-    adminKey: ADMIN_KEY_B,
+    adminToken: adminTokenB,
     body: {
-      peerId: peer.peerId,
+      peerId: peerB.peerId,
       subjectUserId,
       scope,
     },
@@ -214,46 +325,59 @@ async function enrollGrant(opts: {
 
   // 3. Generate enrollment token on Server B
   const tokenResult = await adminFetch<EnrollmentTokenResult>(
-    SERVER_B_URL,
+    serverBUrl,
     `/api/admin/federation/grants/${grant.id}/tokens`,
-    { method: 'POST', adminKey: ADMIN_KEY_B, body: { ttlSeconds: 900 } },
+    { method: 'POST', adminToken: adminTokenB, body: { ttlSeconds: 900 } },
   );
   console.log(`[seed]   Enrollment token: ${tokenResult.token.slice(0, 8)}...`);
+  console.log(`[seed]   Enrollment URL: ${tokenResult.enrollmentUrl}`);
 
-  // 4. Redeem token on Server B with A's CSR
-  //    The enrollment endpoint is not admin-guarded — it uses the one-time token.
-  const redeemUrl = `${SERVER_B_URL}/api/federation/enrollment/${tokenResult.token}`;
+  // 4. Create peer keypair on Server A (Server A's local record of Server B)
+  const peerA = await adminFetch<PeerRecord>(serverAUrl, '/api/admin/federation/peers/keypair', {
+    method: 'POST',
+    adminToken: adminTokenA,
+    body: {
+      commonName: `harness-peer-${label.toLowerCase()}-from-a`,
+      displayName: `Harness Peer ${label} (Server B as seen from A)`,
+      endpointUrl: serverBUrl,
+    },
+  });
+  console.log(`[seed]   Created peer on A: ${peerA.peerId}`);
+
+  // 5. Redeem token at Server B's enrollment endpoint with A's CSR
+  //    The enrollment endpoint is not admin-guarded — the one-time token IS the credential.
+  const redeemUrl = tokenResult.enrollmentUrl;
   const redeemRes = await fetch(redeemUrl, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ csrPem: peer.csrPem }),
+    body: JSON.stringify({ csrPem: peerA.csrPem }),
   });
 
   if (!redeemRes.ok) {
     const body = await redeemRes.text().catch(() => '(no body)');
-    throw new Error(`Enrollment redemption failed: ${redeemRes.status} — ${body}`);
+    throw new Error(`Enrollment redemption failed: ${redeemRes.status.toString()} — ${body}`);
   }
 
   const redeemResult = (await redeemRes.json()) as EnrollmentRedeemResult;
-  console.log(`[seed]   Cert issued (${redeemResult.certPem.length} bytes)`);
+  console.log(`[seed]   Cert issued (${redeemResult.certPem.length.toString()} bytes)`);
 
-  // 5. Store cert on Server A peer record
-  await adminFetch<unknown>(SERVER_A_URL, `/api/admin/federation/peers/${peer.peerId}/cert`, {
+  // 6. Store cert on Server A's peer record → transitions to active
+  await adminFetch<unknown>(serverAUrl, `/api/admin/federation/peers/${peerA.peerId}/cert`, {
     method: 'PATCH',
-    adminKey: ADMIN_KEY_A,
+    adminToken: adminTokenA,
     body: { certPem: redeemResult.certPem },
   });
-  console.log(`[seed]   Cert stored on A — peer ${peer.peerId} is now active`);
+  console.log(`[seed]   Cert stored on A — peer ${peerA.peerId} is now active`);
 
   // Verify grant flipped to active on B
   const activeGrant = await adminFetch<GrantRecord>(
-    SERVER_B_URL,
+    serverBUrl,
     `/api/admin/federation/grants/${grant.id}`,
-    { adminKey: ADMIN_KEY_B },
+    { adminToken: adminTokenB },
   );
   console.log(`[seed]   Grant status on B: ${activeGrant.status}`);
 
-  return { grant: activeGrant, peer: { ...peer, grantId: grant.id } };
+  return { grant: activeGrant, peer: { ...peerA, grantId: grant.id } };
 }
 
 // ─── Test data insertion ──────────────────────────────────────────────────────
@@ -269,7 +393,12 @@ async function enrollGrant(opts: {
  * If that endpoint does not yet exist, this function logs a warning and skips
  * without failing — M3-11 will add the session-based seeding path.
  */
-async function seedTestData(subjectUserId: string, scopeLabel: string): Promise<void> {
+async function seedTestData(
+  subjectUserId: string,
+  scopeLabel: string,
+  serverBUrl: string,
+  adminTokenB: string,
+): Promise<void> {
   console.log(`\n[seed] Seeding test data on Server B for ${scopeLabel}...`);
 
   const testTasks = [
@@ -297,9 +426,9 @@ async function seedTestData(subjectUserId: string, scopeLabel: string): Promise<
   // Attempt to insert — tolerate 404 (endpoint not yet implemented)
   for (const task of testTasks) {
     try {
-      await adminFetch<unknown>(SERVER_B_URL, '/api/admin/tasks', {
+      await adminFetch<unknown>(serverBUrl, '/api/admin/tasks', {
         method: 'POST',
-        adminKey: ADMIN_KEY_B,
+        adminToken: adminTokenB,
         body: task,
       });
       console.log(`[seed]   Inserted task: "${task.title}"`);
@@ -317,9 +446,9 @@ async function seedTestData(subjectUserId: string, scopeLabel: string): Promise<
 
   for (const note of testNotes) {
     try {
-      await adminFetch<unknown>(SERVER_B_URL, '/api/admin/notes', {
+      await adminFetch<unknown>(serverBUrl, '/api/admin/notes', {
         method: 'POST',
-        adminKey: ADMIN_KEY_B,
+        adminToken: adminTokenB,
         body: note,
       });
       console.log(`[seed]   Inserted note: "${note.title}"`);
@@ -343,45 +472,84 @@ async function seedTestData(subjectUserId: string, scopeLabel: string): Promise<
 export async function runSeed(opts?: {
   serverAUrl?: string;
   serverBUrl?: string;
-  adminKeyA?: string;
-  adminKeyB?: string;
+  adminBootstrapPasswordA?: string;
+  adminBootstrapPasswordB?: string;
   subjectUserIds?: { variantA: string; variantB: string; variantC: string };
 }): Promise<SeedResult> {
   const aUrl = opts?.serverAUrl ?? SERVER_A_URL;
   const bUrl = opts?.serverBUrl ?? SERVER_B_URL;
-  const keyA = opts?.adminKeyA ?? ADMIN_KEY_A;
-  const keyB = opts?.adminKeyB ?? ADMIN_KEY_B;
+  const passwordA = opts?.adminBootstrapPasswordA ?? ADMIN_BOOTSTRAP_PASSWORD_A;
+  const passwordB = opts?.adminBootstrapPasswordB ?? ADMIN_BOOTSTRAP_PASSWORD_B;
 
-  // Use provided or default subject user IDs
+  // Use provided or default subject user IDs.
   // In a real run these would be real user UUIDs from Server B's DB.
-  // For the harness, we use deterministic UUIDs that the seed bootstrap creates.
-  const subjectIds = opts?.subjectUserIds ?? {
-    variantA: '00000000-0000-0000-0000-000000000001',
-    variantB: '00000000-0000-0000-0000-000000000002',
-    variantC: '00000000-0000-0000-0000-000000000003',
-  };
+  // For the harness, the admin bootstrap user on Server B is used as the subject.
+  // These are overridden after bootstrap if opts.subjectUserIds is not provided.
+  const subjectIds = opts?.subjectUserIds;
 
   console.log('[seed] Waiting for gateways to be ready...');
   await Promise.all([waitForGateway(aUrl, 'Server A'), waitForGateway(bUrl, 'Server B')]);
 
-  // Enroll all three scope variants in parallel
-  console.log('\n[seed] Enrolling scope variants...');
-  const [resultA, resultB, resultC] = await Promise.all([
-    enrollGrant({ label: 'A', subjectUserId: subjectIds.variantA, scope: SCOPE_VARIANT_A }),
-    enrollGrant({ label: 'B', subjectUserId: subjectIds.variantB, scope: SCOPE_VARIANT_B }),
-    enrollGrant({ label: 'C', subjectUserId: subjectIds.variantC, scope: SCOPE_VARIANT_C }),
+  // Bootstrap admin users on both gateways (requires pristine DBs).
+  console.log('\n[seed] Bootstrapping admin accounts...');
+  const [bootstrapA, bootstrapB] = await Promise.all([
+    bootstrapAdmin(aUrl, 'Server A', passwordA),
+    bootstrapAdmin(bUrl, 'Server B', passwordB),
   ]);
 
+  // Default subject user IDs to the admin user on Server B (guaranteed to exist).
+  const resolvedSubjectIds = subjectIds ?? {
+    variantA: bootstrapB.adminUserId,
+    variantB: bootstrapB.adminUserId,
+    variantC: bootstrapB.adminUserId,
+  };
+
+  // Enroll all three scope variants sequentially to avoid race conditions on
+  // the step-ca signing queue. Parallel enrollment would work too but
+  // sequential is easier to debug when something goes wrong.
+  console.log('\n[seed] Enrolling scope variants...');
+  const resultA = await enrollGrant({
+    label: 'A',
+    subjectUserId: resolvedSubjectIds.variantA,
+    scope: SCOPE_VARIANT_A,
+    adminTokenA: bootstrapA.adminToken,
+    adminTokenB: bootstrapB.adminToken,
+    serverAUrl: aUrl,
+    serverBUrl: bUrl,
+  });
+  const resultB = await enrollGrant({
+    label: 'B',
+    subjectUserId: resolvedSubjectIds.variantB,
+    scope: SCOPE_VARIANT_B,
+    adminTokenA: bootstrapA.adminToken,
+    adminTokenB: bootstrapB.adminToken,
+    serverAUrl: aUrl,
+    serverBUrl: bUrl,
+  });
+  const resultC = await enrollGrant({
+    label: 'C',
+    subjectUserId: resolvedSubjectIds.variantC,
+    scope: SCOPE_VARIANT_C,
+    adminTokenA: bootstrapA.adminToken,
+    adminTokenB: bootstrapB.adminToken,
+    serverAUrl: aUrl,
+    serverBUrl: bUrl,
+  });
+
   // Seed test data on Server B for each scope variant
   await Promise.all([
-    seedTestData(subjectIds.variantA, 'A'),
-    seedTestData(subjectIds.variantB, 'B'),
-    seedTestData(subjectIds.variantC, 'C'),
+    seedTestData(resolvedSubjectIds.variantA, 'A', bUrl, bootstrapB.adminToken),
+    seedTestData(resolvedSubjectIds.variantB, 'B', bUrl, bootstrapB.adminToken),
+    seedTestData(resolvedSubjectIds.variantC, 'C', bUrl, bootstrapB.adminToken),
   ]);
 
   const result: SeedResult = {
     serverAUrl: aUrl,
     serverBUrl: bUrl,
+    adminTokenA: bootstrapA.adminToken,
+    adminTokenB: bootstrapB.adminToken,
+    adminUserIdA: bootstrapA.adminUserId,
+    adminUserIdB: bootstrapB.adminUserId,
     grants: {
       variantA: resultA.grant,
       variantB: resultB.grant,