feat(hierarchy): audit event + outbox machinery (M4-1b-i, contract 1 §5.2) (#1460)

packages/db/src/hierarchy-audit.witness.test.ts

@@ -0,0 +1,363 @@
+/**
+ * Hierarchy audit event + outbox schema witnesses — contract 1 §5.2 and the
+ * schema-level half of §6.4.
+ *
+ * Witnesses the guarantees the tables themselves carry: verb/target-kind/
+ * transfer CHECK constraints, idempotency uniqueness (REQ-AUD-001 duplicate
+ * suppression at the database level), monotonic append order (`seq`),
+ * deletion-safe linkage (no foreign key from the events table into any class
+ * table — events survive the deletion of their target), the causation
+ * self-FK, and the outbox's FK/uniqueness/status shape. The repository-level
+ * half of §6.4 (same-transaction atomicity, rollback, replay) is witnessed in
+ * apps/gateway/src/hierarchy/hierarchy-audit.integration.test.ts.
+ *
+ * Two legs run the same witness body:
+ *  - PGlite (WASM Postgres): always runs.
+ *  - Real PostgreSQL (§6.8): runs when DATABASE_URL is set — the binding
+ *    witness; CI migrates ci-postgres before `pnpm test`.
+ */
+import { randomUUID } from 'node:crypto';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { sql } from 'drizzle-orm';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { createDb } from './client.js';
+import { createPgliteDb } from './client-pglite.js';
+import { runPgliteMigrations } from './migrate.js';
+import { companies, hierarchyAuditEvents, hierarchyOutbox } from './schema.js';
+
+type AnyDb = {
+  db: {
+    insert: (t: unknown) => { values: (v: unknown) => Promise<unknown> };
+    execute: (q: unknown) => Promise<{ rows?: unknown[] } | unknown[]>;
+  };
+  close: () => Promise<void>;
+};
+
+/** Match a constraint failure anywhere along drizzle's cause chain. */
+async function expectViolation(p: Promise<unknown>, re: RegExp, label = ''): Promise<void> {
+  let err: unknown;
+  try {
+    await p;
+  } catch (e) {
+    err = e;
+  }
+  expect(err, label || 'expected the statement to be refused').toBeDefined();
+  const messages: string[] = [];
+  let cur: unknown = err;
+  while (cur instanceof Error) {
+    messages.push(cur.message);
+    cur = (cur as { cause?: unknown }).cause;
+  }
+  expect(messages.join(' | '), label).toMatch(re);
+}
+
+function rows(res: { rows?: unknown[] } | unknown[]): Record<string, unknown>[] {
+  return (Array.isArray(res) ? res : (res.rows ?? [])) as Record<string, unknown>[];
+}
+
+/** Unique per-run prefix so real-PG runs never collide and clean up safely. */
+const T = `hier-a-${randomUUID().slice(0, 8)}`;
+
+/** Minimal valid event row; overrides compose the negative cases. */
+type EventInsert = typeof hierarchyAuditEvents.$inferInsert;
+
+function eventRow(overrides: Partial<EventInsert> = {}): EventInsert {
+  return {
+    actorId: `${T}-actor`,
+    verb: 'create',
+    targetKind: 'company',
+    targetId: randomUUID(),
+    targetSnapshot: { id: 'x', slug: 'x', name: 'x', parentChain: [] },
+    correlationId: `${T}-corr`,
+    idempotencyKey: `${T}-${randomUUID()}`,
+    ...overrides,
+  };
+}
+
+function witnessSuite(getHandle: () => AnyDb): void {
+  const db = () => getHandle().db as unknown as ReturnType<typeof createDb>['db'];
+
+  afterAll(async () => {
+    const d = db();
+    await d.execute(sql`DELETE FROM hierarchy_outbox WHERE idempotency_key LIKE ${T + '%'}`);
+    // Caused events first: the causation self-FK is RESTRICT.
+    await d.execute(
+      sql`DELETE FROM hierarchy_audit_events WHERE idempotency_key LIKE ${T + '%'} AND causation_id IS NOT NULL`,
+    );
+    await d.execute(sql`DELETE FROM hierarchy_audit_events WHERE idempotency_key LIKE ${T + '%'}`);
+    await d.execute(sql`DELETE FROM companies WHERE slug LIKE ${T + '%'}`);
+  });
+
+  // ── CHECK constraints ──────────────────────────────────────────────────────
+
+  it('accepts every declared verb and refuses an undeclared one', async () => {
+    for (const verb of [
+      'create',
+      'rename',
+      'delete',
+      'grant_create',
+      'grant_change',
+      'grant_revoke',
+    ]) {
+      await db().insert(hierarchyAuditEvents).values(eventRow({ verb }));
+    }
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ verb: 'update' })),
+      /verb_check|violates check/i,
+      'undeclared verb must be refused',
+    );
+  });
+
+  it('refuses an undeclared target kind', async () => {
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ targetKind: 'workspace' })),
+      /target_kind_check|violates check/i,
+      'workspace is not an audited target kind (workspace mutation is SOT-side)',
+    );
+  });
+
+  it('requires transfer snapshots exactly on transfers', async () => {
+    const parent = { kind: 'company', id: randomUUID(), slug: 'p' };
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(
+        eventRow({
+          verb: 'transfer',
+          targetKind: 'estate',
+          transferFrom: parent,
+          transferTo: { ...parent, id: randomUUID() },
+        }),
+      );
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ verb: 'transfer' })),
+      /transfer_check|violates check/i,
+      'transfer without source/destination snapshots must be refused',
+    );
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ verb: 'transfer', transferFrom: parent })),
+      /transfer_check|violates check/i,
+      'transfer with only the source snapshot must be refused',
+    );
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ verb: 'create', transferFrom: parent, transferTo: parent })),
+      /transfer_check|violates check/i,
+      'non-transfer with transfer snapshots must be refused',
+    );
+  });
+
+  // ── Idempotency and ordering ───────────────────────────────────────────────
+
+  it('refuses a duplicate idempotency key', async () => {
+    const key = `${T}-dup-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ idempotencyKey: key }));
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ idempotencyKey: key })),
+      /duplicate key|unique/i,
+    );
+  });
+
+  it('assigns strictly increasing seq in insert order for one target', async () => {
+    const targetId = randomUUID();
+    const k1 = `${T}-seq-1-${randomUUID()}`;
+    const k2 = `${T}-seq-2-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ targetId, idempotencyKey: k1 }));
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ targetId, verb: 'rename', idempotencyKey: k2 }));
+    const res = rows(
+      await db().execute(
+        sql`SELECT idempotency_key, seq FROM hierarchy_audit_events WHERE target_id = ${targetId} ORDER BY seq ASC`,
+      ),
+    );
+    expect(res.map((r) => r['idempotency_key'])).toEqual([k1, k2]);
+    expect(Number(res[1]!['seq'])).toBeGreaterThan(Number(res[0]!['seq']));
+  });
+
+  // ── Deletion-safe linkage (§5.2) ───────────────────────────────────────────
+
+  it('has no foreign key into any class table, and events survive target deletion', async () => {
+    const fks = rows(
+      await db().execute(sql`
+        SELECT ccu.table_name AS referenced_table
+        FROM information_schema.table_constraints tc
+        JOIN information_schema.constraint_column_usage ccu
+          ON ccu.constraint_name = tc.constraint_name AND ccu.constraint_schema = tc.constraint_schema
+        WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name = 'hierarchy_audit_events'
+      `),
+    );
+    // The causation self-FK is the ONLY foreign key on the events table.
+    expect([...new Set(fks.map((r) => r['referenced_table']))]).toEqual(['hierarchy_audit_events']);
+
+    const companyId = randomUUID();
+    await db()
+      .insert(companies)
+      .values({ id: companyId, name: 'Doomed', slug: `${T}-doomed` });
+    const key = `${T}-survive-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(
+        eventRow({
+          verb: 'delete',
+          targetId: companyId,
+          targetSnapshot: { id: companyId, slug: `${T}-doomed`, name: 'Doomed', parentChain: [] },
+          idempotencyKey: key,
+        }),
+      );
+    await db().execute(sql`DELETE FROM companies WHERE id = ${companyId}`);
+    const after = rows(
+      await db().execute(
+        sql`SELECT target_snapshot FROM hierarchy_audit_events WHERE idempotency_key = ${key}`,
+      ),
+    );
+    expect(after).toHaveLength(1);
+    expect((after[0]!['target_snapshot'] as { id: string }).id).toBe(companyId);
+  });
+
+  it('enforces the causation self-FK and RESTRICTs deleting a cause', async () => {
+    await expectViolation(
+      db()
+        .insert(hierarchyAuditEvents)
+        .values(eventRow({ causationId: randomUUID() })),
+      /foreign key/i,
+      'causation must reference an existing event',
+    );
+    const causeKey = `${T}-cause-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ verb: 'delete', idempotencyKey: causeKey }));
+    const cause = rows(
+      await db().execute(
+        sql`SELECT id FROM hierarchy_audit_events WHERE idempotency_key = ${causeKey}`,
+      ),
+    )[0]!;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(
+        eventRow({ verb: 'grant_revoke', targetKind: 'grant', causationId: cause['id'] as string }),
+      );
+    await expectViolation(
+      db().execute(sql`DELETE FROM hierarchy_audit_events WHERE id = ${cause['id'] as string}`),
+      /foreign key/i,
+      'a cause with dependent events must not be deletable',
+    );
+  });
+
+  // ── Outbox shape ───────────────────────────────────────────────────────────
+
+  it('outbox rows require an existing event, one outbox row per event, unique idempotency', async () => {
+    await expectViolation(
+      db()
+        .insert(hierarchyOutbox)
+        .values({
+          eventId: randomUUID(),
+          idempotencyKey: `${T}-ob-${randomUUID()}`,
+          correlationId: `${T}-corr`,
+        }),
+      /foreign key/i,
+      'outbox must reference an existing event',
+    );
+    const key = `${T}-ob-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ idempotencyKey: key }));
+    const event = rows(
+      await db().execute(sql`SELECT id FROM hierarchy_audit_events WHERE idempotency_key = ${key}`),
+    )[0]!;
+    const eventId = event['id'] as string;
+    await db()
+      .insert(hierarchyOutbox)
+      .values({ eventId, idempotencyKey: key, correlationId: `${T}-corr` });
+    await expectViolation(
+      db()
+        .insert(hierarchyOutbox)
+        .values({ eventId, idempotencyKey: `${T}-ob2-${randomUUID()}`, correlationId: `${T}-c` }),
+      /duplicate key|unique/i,
+      'one outbox record per event',
+    );
+    await expectViolation(
+      db().execute(
+        sql`INSERT INTO hierarchy_outbox (event_id, idempotency_key, correlation_id, status)
+            VALUES (${eventId}, ${`${T}-ob3-${randomUUID()}`}, 'c', 'failed')`,
+      ),
+      /invalid input value for enum|22P02/i,
+      'status outside pending/processing/delivered must be refused',
+    );
+  });
+
+  it('outbox FK RESTRICTs event deletion while the outbox row exists', async () => {
+    const key = `${T}-obr-${randomUUID()}`;
+    await db()
+      .insert(hierarchyAuditEvents)
+      .values(eventRow({ idempotencyKey: key }));
+    const event = rows(
+      await db().execute(sql`SELECT id FROM hierarchy_audit_events WHERE idempotency_key = ${key}`),
+    )[0]!;
+    await db()
+      .insert(hierarchyOutbox)
+      .values({
+        eventId: event['id'] as string,
+        idempotencyKey: key,
+        correlationId: `${T}-corr`,
+      });
+    await expectViolation(
+      db().execute(sql`DELETE FROM hierarchy_audit_events WHERE id = ${event['id'] as string}`),
+      /foreign key/i,
+    );
+  });
+}
+
+// ── Leg 1: PGlite (always runs — local witness signal) ───────────────────────
+
+describe('hierarchy audit witnesses — PGlite', () => {
+  let dir: string;
+  let handle: ReturnType<typeof createPgliteDb>;
+
+  beforeAll(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'hier-audit-witness-'));
+    handle = createPgliteDb(dir);
+    await runPgliteMigrations(handle);
+  });
+
+  afterAll(async () => {
+    await handle.close();
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  witnessSuite(() => handle as unknown as AnyDb);
+});
+
+// ── Leg 2: real PostgreSQL (§6.8 — binding witness, ci-postgres in CI) ───────
+
+const hasPostgres = Boolean(process.env['DATABASE_URL']);
+
+describe.skipIf(!hasPostgres)('hierarchy audit witnesses — real PostgreSQL', () => {
+  let handle: ReturnType<typeof createDb>;
+
+  beforeAll(() => {
+    handle = createDb(process.env['DATABASE_URL']!);
+  });
+
+  afterAll(async () => {
+    await handle.close();
+  });
+
+  witnessSuite(() => handle as unknown as AnyDb);
+});

packages/db/src/hierarchy-writer-coverage.test.ts

@@ -123,18 +123,20 @@
  *     tracked for M4-1b consideration); the counterfactual — flagging every
  *     bare identifier call — false-positives on essentially all callback
  *     code. Reviews of modules touching db handles carry this residual.
- *   - The scan perimeter is <root>/<pkg>/src for the three roots; production
- *     TS outside a src/ directory (e.g. packages/mosaic/framework/**) is not
- *     scanned (verified free of db/driver/execute references at review time).
- *     Files excluded from the scan — test files and out-of-src modules — are
- *     also invisible as import-graph CONDUITS: test files are emitted to
- *     dist, so a production module could launder a symbol or capability
- *     through a re-export in one. Importing a test module from production
- *     code is anomalous and review-visible; the blind spot is accepted as a
- *     residual, not closed.
+ *   - The scan perimeter is the full <root>/<pkg> tree for the three roots
+ *     (build output, tool caches, and dot-directories excluded), so
+ *     production TS outside src/ — package configs, e2e helpers,
+ *     packages/mosaic/framework/** — is scanned and conduit-visible
+ *     (widened from src/-only in M4-1b-i; the widened set was measured free
+ *     of every trigger token at the time). Files excluded from the scan —
+ *     test files — are still invisible as import-graph CONDUITS: test files
+ *     are emitted to dist, so a production module could launder a symbol or
+ *     capability through a re-export in one. Importing a test module from
+ *     production code is anomalous and review-visible; that blind spot is
+ *     accepted as a residual, not closed.
  *
  * The writer allowlist names hierarchy command/repository modules ONLY. It is
- * empty today: the hierarchy command family (M4-1b) has not landed, so no
+ * empty today: the hierarchy command family (M4-1b-ii) has not landed, so no
  * production module may write the class tables. The infrastructure register
  * holds legitimate non-hierarchy raw execution; registered modules are exempt
  * from prong (iii) only — prongs (i) and (ii) apply to them with no
@@ -179,7 +181,8 @@ const CLASS_TABLES = [
 
 /**
  * Writer allowlist (§6.3b): hierarchy command/repository modules only.
- * EMPTY until the hierarchy command family lands (M4-1b). Adding a module
+ * EMPTY until the hierarchy command family lands (M4-1b-ii; M4-1b-i ships
+ * only the audit/outbox machinery, which writes no class table). Adding a module
  * here is a contract-conformance decision reviewed under §5.1 — the module
  * must be part of the Gateway hierarchy command path, and it must not export
  * a function that executes caller-supplied SQL.
@@ -291,20 +294,23 @@ function isTestPath(rel: string): boolean {
   );
 }
 
+/** Directory names excluded from the walk: build output and tool caches only. */
+const EXCLUDED_DIRS = new Set(['node_modules', 'dist', 'build', 'coverage', 'test-results']);
+
 function collectSources(): string[] {
   const files: string[] = [];
   for (const root of SCAN_ROOTS) {
     const rootDir = join(REPO_ROOT, root);
     if (!existsSync(rootDir)) continue;
     for (const pkg of readdirSync(rootDir)) {
-      const srcDir = join(rootDir, pkg, 'src');
-      if (!existsSync(srcDir) || !statSync(srcDir).isDirectory()) continue;
+      const pkgDir = join(rootDir, pkg);
+      if (!statSync(pkgDir).isDirectory()) continue;
       const walk = (dir: string): void => {
         for (const entry of readdirSync(dir)) {
           const full = join(dir, entry);
           const st = statSync(full);
           if (st.isDirectory()) {
-            if (entry === 'node_modules' || entry === 'dist') continue;
+            if (EXCLUDED_DIRS.has(entry) || entry.startsWith('.')) continue;
             walk(full);
           } else if (EXTENSIONS.has(full.slice(full.lastIndexOf('.')))) {
             const rel = relative(REPO_ROOT, full).split(sep).join('/');
@@ -312,7 +318,7 @@ function collectSources(): string[] {
           }
         }
       };
-      walk(srcDir);
+      walk(pkgDir);
     }
   }
   return files.sort();

packages/db/src/schema.ts

@@ -4,6 +4,7 @@
  */
 
 import { sql } from 'drizzle-orm';
+import type { AnyPgColumn } from 'drizzle-orm/pg-core';
 import {
   pgTable,
   pgEnum,
@@ -1152,3 +1153,111 @@ export const hierarchyGrants = pgTable(
     index('hierarchy_grants_granted_by_idx').on(t.grantedBy),
   ],
 );
+
+// ─── Hierarchy audit events + outbox (contract 1 §5.2) ──────────────────────
+// NOT part of the record class (the class is exactly the five tables above).
+// Append-only semantic audit log for hierarchy mutations, with a dedicated
+// transactional outbox — hierarchy events are not workspace-scoped rows and
+// do not ride the workspace outbox. Deletion-safe linkage: events reference
+// their target by an immutable snapshot (id, slug, parent chain at event
+// time), never by a foreign key into the class tables, so append-only events
+// survive the deletion of their target. Append-only is enforced at the
+// application layer (the hierarchy audit repository exposes no update/delete
+// path for events); REQ-AUD-001's INSERT/SELECT-only database role is a
+// deployment concern outside this schema.
+
+export const HIERARCHY_AUDIT_VERBS = [
+  'create',
+  'rename',
+  'transfer',
+  'delete',
+  'grant_create',
+  'grant_change',
+  'grant_revoke',
+] as const;
+
+export const HIERARCHY_AUDIT_TARGET_KINDS = [
+  'company',
+  'estate',
+  'platform_project',
+  'grant',
+] as const;
+
+export const hierarchyAuditEvents = pgTable(
+  'hierarchy_audit_events',
+  {
+    id: uuid('id').primaryKey().defaultRandom(),
+    // Global append order; per-target ordering (REQ-AUD-001) is a filter on
+    // target_id ordered by seq.
+    seq: bigint('seq', { mode: 'number' }).notNull().generatedAlwaysAsIdentity(),
+    // No FK: audit events outlive every principal and every target (§5.2).
+    actorId: text('actor_id').notNull(),
+    verb: text('verb').notNull(),
+    targetKind: text('target_kind').notNull(),
+    targetId: uuid('target_id').notNull(),
+    // Immutable snapshot at event time. Node events: { id, slug, name,
+    // parentChain: [{ kind, id, slug }, …] root-first }. Grant events:
+    // { id, subject: { userId | teamId }, target: { kind, id }, role }
+    // (subject and role per contract 2 §4.4).
+    targetSnapshot: jsonb('target_snapshot').notNull(),
+    // Present exactly on transfers: snapshot of the source/destination
+    // parent ({ kind, id, slug }), CHECK-enforced below.
+    transferFrom: jsonb('transfer_from'),
+    transferTo: jsonb('transfer_to'),
+    correlationId: text('correlation_id').notNull(),
+    // Prior event in the causal chain (e.g. cascaded grant_revoke events
+    // caused by a node delete). Self-FK RESTRICT keeps the chain intact.
+    causationId: uuid('causation_id').references((): AnyPgColumn => hierarchyAuditEvents.id, {
+      onDelete: 'restrict',
+    }),
+    idempotencyKey: text('idempotency_key').notNull(),
+    occurredAt: timestamp('occurred_at', { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => [
+    uniqueIndex('hierarchy_audit_events_idempotency_idx').on(t.idempotencyKey),
+    uniqueIndex('hierarchy_audit_events_seq_idx').on(t.seq),
+    index('hierarchy_audit_events_target_seq_idx').on(t.targetId, t.seq),
+    index('hierarchy_audit_events_correlation_idx').on(t.correlationId),
+    check(
+      'hierarchy_audit_events_verb_check',
+      sql`verb IN ('create', 'rename', 'transfer', 'delete', 'grant_create', 'grant_change', 'grant_revoke')`,
+    ),
+    check(
+      'hierarchy_audit_events_target_kind_check',
+      sql`target_kind IN ('company', 'estate', 'platform_project', 'grant')`,
+    ),
+    check(
+      'hierarchy_audit_events_transfer_check',
+      sql`(verb = 'transfer') = (transfer_from IS NOT NULL AND transfer_to IS NOT NULL)`,
+    ),
+  ],
+);
+
+export const hierarchyOutboxStatusEnum = pgEnum('hierarchy_outbox_status', [
+  'pending',
+  'processing',
+  'delivered',
+]);
+
+export const hierarchyOutbox = pgTable(
+  'hierarchy_outbox',
+  {
+    id: uuid('id').primaryKey().defaultRandom(),
+    // FK into the append-only events table (not a class table): never
+    // dangles, so RESTRICT is safe and keeps event/outbox integrity.
+    eventId: uuid('event_id')
+      .notNull()
+      .references(() => hierarchyAuditEvents.id, { onDelete: 'restrict' }),
+    idempotencyKey: text('idempotency_key').notNull(),
+    correlationId: text('correlation_id').notNull(),
+    status: hierarchyOutboxStatusEnum('status').notNull().default('pending'),
+    createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
+    updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
+    deliveredAt: timestamp('delivered_at', { withTimezone: true }),
+  },
+  (t) => [
+    uniqueIndex('hierarchy_outbox_event_idx').on(t.eventId),
+    uniqueIndex('hierarchy_outbox_idempotency_idx').on(t.idempotencyKey),
+    index('hierarchy_outbox_status_created_idx').on(t.status, t.createdAt),
+  ],
+);