fix(#1392): hash-ledger migrations + install-time schema verification (closes #1392, closes #1402) (#1403)

Co-authored-by: ops-deploy-01 <[email protected]>

packages/db/src/index.ts

@@ -1,6 +1,15 @@
 export { createDb, type Db, type DbHandle } from './client.js';
 export { createPgliteDb } from './client-pglite.js';
-export { runMigrations, runPgliteMigrations } from './migrate.js';
+export {
+  runMigrations,
+  runPgliteMigrations,
+  getMigrationStatus,
+  readJournalTags,
+  applyMigrationsByHash,
+  type HashLedgerDeps,
+  type MigrationPlanEntry,
+  type MigrationStatus,
+} from './migrate.js';
 export * from './schema.js';
 export * from './federation.js';
 export {

packages/db/src/migrate.spec.ts

@@ -0,0 +1,169 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import {
+  applyMigrationsByHash,
+  readJournalTags,
+  type HashLedgerDeps,
+  type MigrationPlanEntry,
+} from './migrate.js';
+
+/* ------------------------------------------------------------------ */
+/*  In-memory hash-ledger harness                                      */
+/* ------------------------------------------------------------------ */
+
+interface LedgerHarness extends HashLedgerDeps {
+  ledger: Map<string, number>;
+  /** Recorded statement executions in order: `${hashPrefix}:${stmtIndex}`. */
+  executed: string[];
+  /** Optional: statements that should throw when executed. */
+  failOn?: (migrationHash: string, stmtIdx: number) => boolean;
+}
+
+function makeHarness(plan: MigrationPlanEntry[]): LedgerHarness {
+  const hashToEntry = new Map(plan.map((p) => [p.hash, p]));
+  const h: LedgerHarness = {
+    ledger: new Map(),
+    executed: [],
+    ensureLedger: async () => {},
+    appliedHashes: async () => [...h.ledger.keys()],
+    recordApplied: async (hash, folderMillis) => {
+      h.ledger.set(hash, folderMillis);
+    },
+    runStatement: async (statement) => {
+      void statement;
+      // runStatement does not know which migration it belongs to; the
+      // executed log is filled by the wrapper below.
+    },
+  };
+  // Wrap runStatement so the executed log records migration context. We
+  // reconstruct context by tracking a cursor the core advances per migration.
+  let cursor = 0;
+  const flat: Array<{ hash: string; idx: number }> = [];
+  for (const m of plan)
+    for (const [i] of m.statements.entries()) flat.push({ hash: m.hash, idx: i });
+  h.runStatement = async () => {
+    const at = flat[cursor] ?? { hash: '??', idx: -1 };
+    cursor += 1;
+    if (h.failOn && at.hash !== '??' && h.failOn(at.hash, at.idx)) {
+      throw new Error(`simulated failure in ${at.hash} #${at.idx.toString()}`);
+    }
+    h.executed.push(`${at.hash.slice(0, 6)}:${at.idx.toString()}`);
+  };
+  void hashToEntry;
+  return h;
+}
+
+/* ------------------------------------------------------------------ */
+/*  Fixtures                                                           */
+/* ------------------------------------------------------------------ */
+
+// Reproduces the REAL journal defect shape (#1402 D1): 0009/0010 carry
+// `when` timestamps BELOW 0008's. Under the old drizzle postgres-js
+// migrator these were silently skipped on any upgrade whose ledger was
+// last stamped in the 0008 era.
+const JOURNAL_FIXTURE: MigrationPlanEntry[] = [
+  { hash: 'aaaa0000', folderMillis: 1773368153122, statements: ['CREATE TABLE a (id int)'] },
+  { hash: 'bbbb0008', folderMillis: 1776822435828, statements: ['CREATE TABLE b (id int)'] },
+  // Backdated entries, exactly as shipped:
+  {
+    hash: 'cccc0009',
+    folderMillis: 1745280000000,
+    statements: ['ALTER TYPE t ADD VALUE', "CREATE TABLE c (s t DEFAULT 'pending')"],
+  },
+  { hash: 'dddd0010', folderMillis: 1745366400000, statements: ['CREATE TABLE d (id int)'] },
+];
+
+/** A ledger last stamped at the 0008 era: only pre-0009 hashes recorded. */
+const LEDGER_AT_0008_ERA = new Map<string, number>([
+  ['aaaa0000', 1773368153122],
+  ['bbbb0008', 1776822435828],
+]);
+
+/* ------------------------------------------------------------------ */
+/*  The core: apply-by-hash in journal order                           */
+/* ------------------------------------------------------------------ */
+
+describe('applyMigrationsByHash', () => {
+  it('applies backdated journal entries that a timestamp-based migrator would skip (#1402 D1)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    h.ledger = new Map(LEDGER_AT_0008_ERA);
+
+    const result = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+
+    // D1 in one sentence: 0009 and 0010 applied despite folderMillis < 0008.
+    expect(result).toEqual({ applied: 2, skipped: 2 });
+    expect(h.ledger.has('cccc0009')).toBe(true);
+    expect(h.ledger.has('dddd0010')).toBe(true);
+  });
+
+  it('executes statements individually (ALTER TYPE visibility, #1402 D2 shape)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    // 0009's two statements recorded as separate executions, in order.
+    expect(h.executed).toContain('cccc00:0');
+    expect(h.executed).toContain('cccc00:1');
+    expect(h.executed.indexOf('cccc00:0')).toBeLessThan(h.executed.indexOf('cccc00:1'));
+  });
+
+  it('is idempotent: a fully-applied ledger applies nothing', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    const first = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    const second = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    expect(first.applied).toBe(4);
+    expect(second).toEqual({ applied: 0, skipped: 4 });
+    expect(h.executed).toHaveLength(5); // 5 statements; second run executed NONE (not 10)
+  });
+
+  it('records no ledger row when a statement fails (crash prefix replays loudly)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    h.failOn = (hash, idx) => hash === 'cccc0009' && idx === 1;
+
+    await expect(applyMigrationsByHash(h, JOURNAL_FIXTURE)).rejects.toThrow(
+      /cccc0009 statement #1 failed: simulated failure/,
+    );
+    // Statement 0 of 0009 executed, but NO ledger row for 0009: the next run
+    // replays it and fails loudly on "already exists" instead of silently
+    // believing 0009 applied.
+    expect(h.ledger.has('cccc0009')).toBe(false);
+    expect(h.executed).toContain('cccc00:0');
+  });
+
+  it('applies in JOURNAL order, not timestamp order', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    // 5 statements total (0009 has two); prefix per migration: journal order,
+    // so 0009's pair sits between 0008 and 0010.
+    const order = h.executed.map((e) => e.slice(0, 4));
+    expect(order).toEqual(['aaaa', 'bbbb', 'cccc', 'cccc', 'dddd']);
+  });
+});
+
+/* ------------------------------------------------------------------ */
+/*  Journal integrity against the shipped folder                       */
+/* ------------------------------------------------------------------ */
+
+describe('readJournalTags', () => {
+  it('reads the shipped journal in order and sees the known backdated pair', () => {
+    const folder = resolve(__dirname, '../drizzle');
+    const tags = readJournalTags(folder);
+    expect(tags.length).toBeGreaterThan(0);
+    // The shipped defect (#1402 D1): these two entries carry April-2025
+    // timestamps below 0008's June-2026 one. If this assertion ever fails
+    // because the journal was FIXED (timestamps corrected or drizzle-kit
+    // regenerated), update #1402 — the hash-ledger core stays correct either
+    // way; this test pins the shipped reality the core was built for.
+    const t9 = tags.find((t) => t.startsWith('0009_'));
+    const t10 = tags.find((t) => t.startsWith('0010_'));
+    const t8 = tags.find((t) => t.startsWith('0008_'));
+    expect([t8, t9, t10]).toBeDefined();
+    const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as {
+      entries: Array<{ tag: string; when: number }>;
+    };
+    const when = new Map(journal.entries.map((e) => [e.tag, e.when]));
+    if (t8 && t9 && t10) {
+      expect(when.get(t9)!).toBeLessThan(when.get(t8)!); // backdated below 0008
+      expect(when.get(t10)!).toBeLessThan(when.get(t8)!); // backdated below 0008
+    }
+  });
+});

packages/db/src/migrate.ts

@@ -1,8 +1,7 @@
+import { readFileSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { sql } from 'drizzle-orm';
-import { drizzle as drizzlePostgres } from 'drizzle-orm/postgres-js';
-import { migrate as migratePostgres } from 'drizzle-orm/postgres-js/migrator';
 import { readMigrationFiles } from 'drizzle-orm/migrator';
 import postgres from 'postgres';
 import { DEFAULT_DATABASE_URL } from './defaults.js';
@@ -21,89 +20,243 @@ function migrationsFolder(): string {
   return resolve(here, '../drizzle');
 }
 
+/* ------------------------------------------------------------------ */
+/*  Shared hash-ledger migration core (#1392 / #1402)                  */
+/* ------------------------------------------------------------------ */
+//
+// Both tiers migrate through this single core, which applies migrations in
+// JOURNAL ORDER, one statement at a time, and skips by HASH — never by
+// folderMillis timestamp. The previous postgres path delegated to drizzle's
+// postgres-js migrator, which:
+//
+//   * applies only migrations with folderMillis > last-applied, silently
+//     skipping journal entries whose `when` is older than the ledger's newest
+//     stamp — 0009/0010 carry April-2025 timestamps below 0008's June-2026
+//     one, so any database last migrated in the 0008 era silently loses
+//     0009/0010 forever (#1402 D1); and
+//   * wraps each migration in ONE transaction, which breaks migrations that
+//     do `ALTER TYPE ADD VALUE` and then reference the new value in the same
+//     migration (0009) — Postgres' check_safe_enum_use rejects it (#1402 D2).
+//
+// Per-statement execution (each statement autocommits) and skip-by-hash fix
+// both. The PGlite path has run this way since it was written; this is the
+// TODO it left behind, now shared instead of duplicated.
+
+/** One migration as loaded from the shipped drizzle/ folder. */
+export interface MigrationPlanEntry {
+  hash: string;
+  folderMillis: number;
+  statements: string[];
+}
+
+/** The persistence operations the hash-ledger core needs, per tier. */
+export interface HashLedgerDeps {
+  /** Create the drizzle schema + ledger table if absent (idempotent). */
+  ensureLedger(): Promise<void>;
+  /** Hashes already recorded in the ledger. */
+  appliedHashes(): Promise<string[]>;
+  /** Record one fully-applied migration in the ledger. */
+  recordApplied(hash: string, folderMillis: number): Promise<void>;
+  /** Execute one SQL statement, autocommitting (never inside a wider tx). */
+  runStatement(statement: string): Promise<void>;
+}
+
+function loadPlan(): MigrationPlanEntry[] {
+  return readMigrationFiles({ migrationsFolder: migrationsFolder() }).map((m) => ({
+    hash: m.hash,
+    folderMillis: m.folderMillis,
+    statements: m.sql.map((s) => s.trim()).filter((s) => s.length > 0),
+  }));
+}
+
+/**
+ * Apply every unapplied migration in journal order, skipping by hash.
+ *
+ * Failure model: each statement autocommits, and the ledger row is written
+ * only after all statements of a migration succeed. A crash mid-migration
+ * leaves the prefix applied with no ledger entry, so the next boot replays
+ * those statements and fails loudly on "already exists". Recovery: drop the
+ * partially-applied objects, or insert the migration's hash into
+ * `drizzle.__drizzle_migrations` manually. The thrown error identifies the
+ * statement and migration that failed.
+ */
+export async function applyMigrationsByHash(
+  deps: HashLedgerDeps,
+  plan: MigrationPlanEntry[] = loadPlan(),
+): Promise<{ applied: number; skipped: number }> {
+  await deps.ensureLedger();
+  const alreadyApplied = new Set(await deps.appliedHashes());
+
+  let applied = 0;
+  let skipped = 0;
+  for (const migration of plan) {
+    if (alreadyApplied.has(migration.hash)) {
+      skipped += 1;
+      continue;
+    }
+    for (const [stmtIdx, stmt] of migration.statements.entries()) {
+      try {
+        await deps.runStatement(stmt);
+      } catch (err) {
+        const cause = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` +
+            `Statement: ${stmt.slice(0, 200)}${stmt.length > 200 ? '…' : ''}`,
+          { cause: err },
+        );
+      }
+    }
+    await deps.recordApplied(migration.hash, migration.folderMillis);
+    applied += 1;
+  }
+  return { applied, skipped };
+}
+
+const LEDGER_DDL = [
+  'CREATE SCHEMA IF NOT EXISTS drizzle',
+  `CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
+      id SERIAL PRIMARY KEY,
+      hash text NOT NULL,
+      created_at bigint
+    )`,
+];
+
+function connectionString(url?: string): string {
+  return url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL;
+}
+
+/**
+ * Apply Drizzle migrations against a postgres database, hash-ledger style.
+ * Idempotent: re-running against a fully-migrated database applies nothing.
+ */
 export async function runMigrations(url?: string): Promise<void> {
-  const connectionString = url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL;
-  const sqlClient = postgres(connectionString, { max: 1 });
-  const db = drizzlePostgres(sqlClient);
+  const sqlClient = postgres(connectionString(url), { max: 1 });
   try {
-    // TODO: postgres-tier first-install also fails because (a) Drizzle wraps every
-    // migration in one transaction (breaks 0009's ALTER TYPE ADD VALUE → SET DEFAULT
-    // sequence) and (b) drizzle/meta/_journal.json has 0009 ordered before 0008,
-    // which the postgres-js migrator skips by `created_at < folderMillis`. The
-    // PGlite path below sidesteps both. A follow-up should either share the
-    // per-statement loop (see runPgliteMigrations) or fix the journal ordering.
-    await migratePostgres(db, { migrationsFolder: migrationsFolder() });
+    await applyMigrationsByHash({
+      ensureLedger: async () => {
+        for (const ddl of LEDGER_DDL) await sqlClient.unsafe(ddl);
+      },
+      appliedHashes: async () => {
+        const rows = (await sqlClient.unsafe(
+          'SELECT hash FROM drizzle.__drizzle_migrations',
+        )) as Array<{ hash: string }>;
+        return rows.map((r) => String(r.hash));
+      },
+      recordApplied: async (hash, folderMillis) => {
+        await sqlClient.unsafe(
+          'INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ($1, $2)',
+          [hash, folderMillis],
+        );
+      },
+      runStatement: async (stmt) => {
+        await sqlClient.unsafe(stmt);
+      },
+    });
   } finally {
     await sqlClient.end();
   }
 }
 
-// Apply Drizzle migrations against an embedded PGlite database.
-//
-// We don't reuse drizzle's pglite migrator because it wraps ALL migrations in
-// one outer transaction, which breaks Postgres' `check_safe_enum_use` rule —
-// e.g. migration 0009 does `ALTER TYPE ADD VALUE 'pending'` then references
-// `'pending'` as a default in the same tx. PGlite's `exec()` runs each
-// statement under the Simple Query protocol, autocommitting between them.
-//
-// We still write to the standard `drizzle.__drizzle_migrations` ledger so the
-// result is interoperable with `runMigrations()` on a postgres-backed deploy
-// (modulo the journal-ordering bug noted above).
-//
-// We skip-by-hash rather than skip-by-folderMillis (which is what Drizzle's
-// postgres-js migrator does). That's deliberate — out-of-order timestamps in
-// `_journal.json` won't silently drop migrations.
-//
-// Failure model: each statement autocommits, and the ledger row is written
-// only after all statements in a migration succeed. A crash mid-migration
-// leaves the prefix applied with no ledger entry, so the next boot will
-// replay those statements and fail loudly on "already exists". Recovery:
-// drop the partially-applied objects, or insert the migration's hash into
-// `drizzle.__drizzle_migrations` manually. The error log identifies which
-// statement of which migration was the culprit.
+/**
+ * Apply Drizzle migrations against an embedded PGlite database.
+ *
+ * We don't reuse drizzle's pglite migrator for the same reasons as the
+ * postgres path (single-transaction wrap; folderMillis skip). PGlite's
+ * `exec()` runs each statement under the Simple Query protocol,
+ * autocommitting between them — exactly the semantics the shared core needs.
+ *
+ * The ledger rows this writes are interoperable with the postgres path (same
+ * schema, same hashes), because both consume the same shipped migrations.
+ */
 export async function runPgliteMigrations(handle: DbHandle): Promise<void> {
   const client = (handle.db as unknown as { $client?: PgliteExecutor }).$client;
   if (!client || typeof client.exec !== 'function') {
     throw new Error('runPgliteMigrations: handle.db is not backed by a PGlite client');
   }
 
-  await client.exec('CREATE SCHEMA IF NOT EXISTS drizzle');
-  await client.exec(`
-    CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
-      id SERIAL PRIMARY KEY,
-      hash text NOT NULL,
-      created_at bigint
-    )
-  `);
+  await applyMigrationsByHash({
+    ensureLedger: async () => {
+      for (const ddl of LEDGER_DDL) await client.exec(ddl);
+    },
+    appliedHashes: async () => {
+      const rows = (await handle.db.execute(
+        sql`SELECT hash FROM drizzle.__drizzle_migrations`,
+      )) as unknown as ExecuteRows<{ hash: string }>;
+      return rows.rows.map((r) => String(r.hash));
+    },
+    recordApplied: async (hash, folderMillis) => {
+      await handle.db.execute(
+        sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${hash}, ${folderMillis})`,
+      );
+    },
+    runStatement: async (stmt) => {
+      await client.exec(stmt);
+    },
+  });
+}
 
-  const appliedRows = (await handle.db.execute(
-    sql`SELECT hash FROM drizzle.__drizzle_migrations`,
-  )) as unknown as ExecuteRows<{ hash: string }>;
-  const applied = new Set(appliedRows.rows.map((r) => r.hash));
+/* ------------------------------------------------------------------ */
+/*  Migration status (#1392: the installer must VERIFY, not assume)     */
+/* ------------------------------------------------------------------ */
 
-  const migrations = readMigrationFiles({ migrationsFolder: migrationsFolder() });
-  for (const migration of migrations) {
-    if (applied.has(migration.hash)) continue;
+/** Read the journal tags (migration folder names) in journal order. */
+export function readJournalTags(folder: string = migrationsFolder()): string[] {
+  const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as {
+    entries?: Array<{ tag?: string }>;
+  };
+  return (journal.entries ?? []).map((e) => e.tag ?? '').filter((t) => t.length > 0);
+}
 
-    // Run each statement-breakpoint chunk in its own exec() call so PGlite
-    // commits between statements — this is what lets `ALTER TYPE ADD VALUE`
-    // become visible before a subsequent statement references the new value.
-    for (const [stmtIdx, stmt] of migration.sql.entries()) {
-      const trimmed = stmt.trim();
-      if (!trimmed) continue;
-      try {
-        await client.exec(trimmed);
-      } catch (err) {
-        const cause = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `runPgliteMigrations: migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` +
-            `Statement: ${trimmed.slice(0, 200)}${trimmed.length > 200 ? '…' : ''}`,
-          { cause: err },
-        );
-      }
+export interface MigrationStatus {
+  /** Hashes recorded in the database's ledger (0 if no ledger exists). */
+  appliedCount: number;
+  /** Migrations shipped in this package's drizzle/ folder. */
+  expectedCount: number;
+  /** The tag (folder name) of the last journal entry, for error messages. */
+  expectedLastTag: string;
+  /** True iff every shipped migration's hash is in the ledger. */
+  complete: boolean;
+}
+
+/**
+ * Report whether a postgres database carries the full shipped schema.
+ *
+ * Read-only apart from `ensureLedger` semantics: it never creates the ledger
+ * (unlike the migrators), so a database with NO ledger reports
+ * appliedCount=0 / complete=false — the exact #1392/#1389 signature (an
+ * install whose dependency set shipped no migrations at all).
+ */
+export async function getMigrationStatus(url?: string): Promise<MigrationStatus> {
+  const sqlClient = postgres(connectionString(url), { max: 1 });
+  try {
+    const plan = loadPlan();
+    const tags = readJournalTags();
+    const expectedHashes = new Set(plan.map((m) => m.hash));
+
+    let appliedHashes: string[] = [];
+    const regRows = (await sqlClient.unsafe(
+      "SELECT to_regclass('drizzle.__drizzle_migrations') AS reg",
+    )) as Array<{ reg: string | null }>;
+    if (regRows[0]?.reg) {
+      const rows = (await sqlClient.unsafe(
+        'SELECT hash FROM drizzle.__drizzle_migrations',
+      )) as Array<{ hash: string }>;
+      appliedHashes = rows.map((r) => String(r.hash));
     }
-    await handle.db.execute(
-      sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${migration.hash}, ${migration.folderMillis})`,
-    );
+
+    const appliedSet = new Set(appliedHashes);
+    return {
+      appliedCount: appliedHashes.length,
+      expectedCount: plan.length,
+      expectedLastTag: tags[tags.length - 1] ?? '',
+      complete:
+        plan.length > 0 &&
+        plan.every((m) => appliedSet.has(m.hash)) &&
+        // A ledger with entries OUTSIDE the shipped plan means the database
+        // came from a different (e.g. newer) build — not "complete" either.
+        appliedHashes.every((h) => expectedHashes.has(h)),
+    };
+  } finally {
+    await sqlClient.end();
   }
 }