fix(#1392): hash-ledger migrations + install-time schema verification (closes #1392, closes #1402) (#1403)

Co-authored-by: ops-deploy-01 <[email protected]>

apps/gateway/src/database/database.module.1392.spec.ts

@@ -0,0 +1,71 @@
+import { describe, it, expect, vi } from 'vitest';
+
+// The module under test imports @mosaicstack/db at module scope; we replace only the
+// pieces DatabaseModule uses (partial mock — the real module also exports the
+// schema the storage adapter's import chain needs) so the test pins the #1392
+// contract (refuse to start on an incomplete schema) without a live database.
+vi.mock('@mosaicstack/db', async (importOriginal) => {
+  const actual: object = await importOriginal();
+  return {
+    ...actual,
+    createDb: vi.fn(),
+    createPgliteDb: vi.fn(),
+    getMigrationStatus: vi.fn(),
+    runPgliteMigrations: vi.fn(),
+  };
+});
+
+import { DatabaseModule } from './database.module.js';
+import { getMigrationStatus } from '@mosaicstack/db';
+import type { DbHandle } from '@mosaicstack/db';
+import type { StorageAdapter } from '@mosaicstack/storage';
+import type { MosaicConfig } from '@mosaicstack/config';
+
+function makeModule(storageType: 'postgres' | 'pglite', tier: string) {
+  const storageAdapter = {
+    name: storageType,
+    migrate: vi.fn(),
+    close: vi.fn(),
+  } as unknown as StorageAdapter;
+  const handle = { close: vi.fn() } as unknown as DbHandle;
+  const config = {
+    tier,
+    storage: { type: storageType, url: 'postgresql://x' },
+  } as unknown as MosaicConfig;
+  return {
+    mod: new DatabaseModule(handle, storageAdapter, config),
+    storageAdapter,
+  };
+}
+
+describe('DatabaseModule.onModuleInit — #1392 schema verification', () => {
+  it('refuses to start when the postgres schema is incomplete', async () => {
+    const { mod, storageAdapter } = makeModule('postgres', 'standalone');
+    vi.mocked(getMigrationStatus).mockResolvedValue({
+      appliedCount: 15,
+      expectedCount: 17,
+      expectedLastTag: '0016_salty_morlocks',
+      complete: false,
+    });
+    await expect(mod.onModuleInit()).rejects.toThrow('Database schema incomplete: 15/17');
+    expect(storageAdapter.migrate).toHaveBeenCalled(); // migrations attempted first
+  });
+
+  it('starts normally when the schema is complete', async () => {
+    const { mod } = makeModule('postgres', 'standalone');
+    vi.mocked(getMigrationStatus).mockResolvedValue({
+      appliedCount: 17,
+      expectedCount: 17,
+      expectedLastTag: '0016_salty_morlocks',
+      complete: true,
+    });
+    await expect(mod.onModuleInit()).resolves.toBeUndefined();
+  });
+
+  it('does not verify postgres status for the local tier (PGlite migrates itself)', async () => {
+    const { mod } = makeModule('pglite', 'local');
+    vi.mocked(getMigrationStatus).mockClear();
+    await expect(mod.onModuleInit()).resolves.toBeUndefined();
+    expect(getMigrationStatus).not.toHaveBeenCalled();
+  });
+});

apps/gateway/src/database/database.module.ts

@@ -12,6 +12,7 @@ import {
 import {
   createDb,
   createPgliteDb,
+  getMigrationStatus,
   runPgliteMigrations,
   type Db,
   type DbHandle,
@@ -74,6 +75,11 @@ export class DatabaseModule implements OnApplicationShutdown, OnModuleInit {
   // the same DATABASE_URL, so a single call covers both the gateway DB and
   // the storage tables. We deliberately do NOT call runMigrations() here to
   // avoid opening a second short-lived connection and doubling startup cost.
+  //
+  // #1392: we DO verify afterwards (getMigrationStatus opens one short-lived
+  // connection) and refuse to start on an incomplete schema. A gateway that
+  // boots "healthy" on an empty or partial database is precisely the failure
+  // that shipped in the T63 batch: silent at startup, catastrophic later.
   async onModuleInit(): Promise<void> {
     if (this.config.tier === 'local') {
       this.logger.log('Applying PGlite schema migrations...');
@@ -81,6 +87,24 @@ export class DatabaseModule implements OnApplicationShutdown, OnModuleInit {
     }
     this.logger.log(`Initializing storage adapter (${this.storageAdapter.name})...`);
     await this.storageAdapter.migrate();
+
+    if (this.config.storage.type === 'postgres') {
+      const status = await getMigrationStatus(this.config.storage.url);
+      if (!status.complete) {
+        this.logger.error(
+          `Database schema incomplete: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied ` +
+            `(last expected: ${status.expectedLastTag}). ` +
+            'Refusing to start on a partial schema — see issues #1392/#1402. ' +
+            "Remediation: re-run 'mosaic gateway install' (it now verifies), or apply migrations manually.",
+        );
+        throw new Error(
+          `Database schema incomplete: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied`,
+        );
+      }
+      this.logger.log(
+        `Database schema verified: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied.`,
+      );
+    }
   }
 
   async onApplicationShutdown(): Promise<void> {

packages/db/src/index.ts

@@ -1,6 +1,15 @@
 export { createDb, type Db, type DbHandle } from './client.js';
 export { createPgliteDb } from './client-pglite.js';
-export { runMigrations, runPgliteMigrations } from './migrate.js';
+export {
+  runMigrations,
+  runPgliteMigrations,
+  getMigrationStatus,
+  readJournalTags,
+  applyMigrationsByHash,
+  type HashLedgerDeps,
+  type MigrationPlanEntry,
+  type MigrationStatus,
+} from './migrate.js';
 export * from './schema.js';
 export * from './federation.js';
 export {

packages/db/src/migrate.spec.ts

@@ -0,0 +1,169 @@
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import {
+  applyMigrationsByHash,
+  readJournalTags,
+  type HashLedgerDeps,
+  type MigrationPlanEntry,
+} from './migrate.js';
+
+/* ------------------------------------------------------------------ */
+/*  In-memory hash-ledger harness                                      */
+/* ------------------------------------------------------------------ */
+
+interface LedgerHarness extends HashLedgerDeps {
+  ledger: Map<string, number>;
+  /** Recorded statement executions in order: `${hashPrefix}:${stmtIndex}`. */
+  executed: string[];
+  /** Optional: statements that should throw when executed. */
+  failOn?: (migrationHash: string, stmtIdx: number) => boolean;
+}
+
+function makeHarness(plan: MigrationPlanEntry[]): LedgerHarness {
+  const hashToEntry = new Map(plan.map((p) => [p.hash, p]));
+  const h: LedgerHarness = {
+    ledger: new Map(),
+    executed: [],
+    ensureLedger: async () => {},
+    appliedHashes: async () => [...h.ledger.keys()],
+    recordApplied: async (hash, folderMillis) => {
+      h.ledger.set(hash, folderMillis);
+    },
+    runStatement: async (statement) => {
+      void statement;
+      // runStatement does not know which migration it belongs to; the
+      // executed log is filled by the wrapper below.
+    },
+  };
+  // Wrap runStatement so the executed log records migration context. We
+  // reconstruct context by tracking a cursor the core advances per migration.
+  let cursor = 0;
+  const flat: Array<{ hash: string; idx: number }> = [];
+  for (const m of plan)
+    for (const [i] of m.statements.entries()) flat.push({ hash: m.hash, idx: i });
+  h.runStatement = async () => {
+    const at = flat[cursor] ?? { hash: '??', idx: -1 };
+    cursor += 1;
+    if (h.failOn && at.hash !== '??' && h.failOn(at.hash, at.idx)) {
+      throw new Error(`simulated failure in ${at.hash} #${at.idx.toString()}`);
+    }
+    h.executed.push(`${at.hash.slice(0, 6)}:${at.idx.toString()}`);
+  };
+  void hashToEntry;
+  return h;
+}
+
+/* ------------------------------------------------------------------ */
+/*  Fixtures                                                           */
+/* ------------------------------------------------------------------ */
+
+// Reproduces the REAL journal defect shape (#1402 D1): 0009/0010 carry
+// `when` timestamps BELOW 0008's. Under the old drizzle postgres-js
+// migrator these were silently skipped on any upgrade whose ledger was
+// last stamped in the 0008 era.
+const JOURNAL_FIXTURE: MigrationPlanEntry[] = [
+  { hash: 'aaaa0000', folderMillis: 1773368153122, statements: ['CREATE TABLE a (id int)'] },
+  { hash: 'bbbb0008', folderMillis: 1776822435828, statements: ['CREATE TABLE b (id int)'] },
+  // Backdated entries, exactly as shipped:
+  {
+    hash: 'cccc0009',
+    folderMillis: 1745280000000,
+    statements: ['ALTER TYPE t ADD VALUE', "CREATE TABLE c (s t DEFAULT 'pending')"],
+  },
+  { hash: 'dddd0010', folderMillis: 1745366400000, statements: ['CREATE TABLE d (id int)'] },
+];
+
+/** A ledger last stamped at the 0008 era: only pre-0009 hashes recorded. */
+const LEDGER_AT_0008_ERA = new Map<string, number>([
+  ['aaaa0000', 1773368153122],
+  ['bbbb0008', 1776822435828],
+]);
+
+/* ------------------------------------------------------------------ */
+/*  The core: apply-by-hash in journal order                           */
+/* ------------------------------------------------------------------ */
+
+describe('applyMigrationsByHash', () => {
+  it('applies backdated journal entries that a timestamp-based migrator would skip (#1402 D1)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    h.ledger = new Map(LEDGER_AT_0008_ERA);
+
+    const result = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+
+    // D1 in one sentence: 0009 and 0010 applied despite folderMillis < 0008.
+    expect(result).toEqual({ applied: 2, skipped: 2 });
+    expect(h.ledger.has('cccc0009')).toBe(true);
+    expect(h.ledger.has('dddd0010')).toBe(true);
+  });
+
+  it('executes statements individually (ALTER TYPE visibility, #1402 D2 shape)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    // 0009's two statements recorded as separate executions, in order.
+    expect(h.executed).toContain('cccc00:0');
+    expect(h.executed).toContain('cccc00:1');
+    expect(h.executed.indexOf('cccc00:0')).toBeLessThan(h.executed.indexOf('cccc00:1'));
+  });
+
+  it('is idempotent: a fully-applied ledger applies nothing', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    const first = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    const second = await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    expect(first.applied).toBe(4);
+    expect(second).toEqual({ applied: 0, skipped: 4 });
+    expect(h.executed).toHaveLength(5); // 5 statements; second run executed NONE (not 10)
+  });
+
+  it('records no ledger row when a statement fails (crash prefix replays loudly)', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    h.failOn = (hash, idx) => hash === 'cccc0009' && idx === 1;
+
+    await expect(applyMigrationsByHash(h, JOURNAL_FIXTURE)).rejects.toThrow(
+      /cccc0009 statement #1 failed: simulated failure/,
+    );
+    // Statement 0 of 0009 executed, but NO ledger row for 0009: the next run
+    // replays it and fails loudly on "already exists" instead of silently
+    // believing 0009 applied.
+    expect(h.ledger.has('cccc0009')).toBe(false);
+    expect(h.executed).toContain('cccc00:0');
+  });
+
+  it('applies in JOURNAL order, not timestamp order', async () => {
+    const h = makeHarness(JOURNAL_FIXTURE);
+    await applyMigrationsByHash(h, JOURNAL_FIXTURE);
+    // 5 statements total (0009 has two); prefix per migration: journal order,
+    // so 0009's pair sits between 0008 and 0010.
+    const order = h.executed.map((e) => e.slice(0, 4));
+    expect(order).toEqual(['aaaa', 'bbbb', 'cccc', 'cccc', 'dddd']);
+  });
+});
+
+/* ------------------------------------------------------------------ */
+/*  Journal integrity against the shipped folder                       */
+/* ------------------------------------------------------------------ */
+
+describe('readJournalTags', () => {
+  it('reads the shipped journal in order and sees the known backdated pair', () => {
+    const folder = resolve(__dirname, '../drizzle');
+    const tags = readJournalTags(folder);
+    expect(tags.length).toBeGreaterThan(0);
+    // The shipped defect (#1402 D1): these two entries carry April-2025
+    // timestamps below 0008's June-2026 one. If this assertion ever fails
+    // because the journal was FIXED (timestamps corrected or drizzle-kit
+    // regenerated), update #1402 — the hash-ledger core stays correct either
+    // way; this test pins the shipped reality the core was built for.
+    const t9 = tags.find((t) => t.startsWith('0009_'));
+    const t10 = tags.find((t) => t.startsWith('0010_'));
+    const t8 = tags.find((t) => t.startsWith('0008_'));
+    expect([t8, t9, t10]).toBeDefined();
+    const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as {
+      entries: Array<{ tag: string; when: number }>;
+    };
+    const when = new Map(journal.entries.map((e) => [e.tag, e.when]));
+    if (t8 && t9 && t10) {
+      expect(when.get(t9)!).toBeLessThan(when.get(t8)!); // backdated below 0008
+      expect(when.get(t10)!).toBeLessThan(when.get(t8)!); // backdated below 0008
+    }
+  });
+});

packages/db/src/migrate.ts

@@ -1,8 +1,7 @@
+import { readFileSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { sql } from 'drizzle-orm';
-import { drizzle as drizzlePostgres } from 'drizzle-orm/postgres-js';
-import { migrate as migratePostgres } from 'drizzle-orm/postgres-js/migrator';
 import { readMigrationFiles } from 'drizzle-orm/migrator';
 import postgres from 'postgres';
 import { DEFAULT_DATABASE_URL } from './defaults.js';
@@ -21,89 +20,243 @@ function migrationsFolder(): string {
   return resolve(here, '../drizzle');
 }
 
+/* ------------------------------------------------------------------ */
+/*  Shared hash-ledger migration core (#1392 / #1402)                  */
+/* ------------------------------------------------------------------ */
+//
+// Both tiers migrate through this single core, which applies migrations in
+// JOURNAL ORDER, one statement at a time, and skips by HASH — never by
+// folderMillis timestamp. The previous postgres path delegated to drizzle's
+// postgres-js migrator, which:
+//
+//   * applies only migrations with folderMillis > last-applied, silently
+//     skipping journal entries whose `when` is older than the ledger's newest
+//     stamp — 0009/0010 carry April-2025 timestamps below 0008's June-2026
+//     one, so any database last migrated in the 0008 era silently loses
+//     0009/0010 forever (#1402 D1); and
+//   * wraps each migration in ONE transaction, which breaks migrations that
+//     do `ALTER TYPE ADD VALUE` and then reference the new value in the same
+//     migration (0009) — Postgres' check_safe_enum_use rejects it (#1402 D2).
+//
+// Per-statement execution (each statement autocommits) and skip-by-hash fix
+// both. The PGlite path has run this way since it was written; this is the
+// TODO it left behind, now shared instead of duplicated.
+
+/** One migration as loaded from the shipped drizzle/ folder. */
+export interface MigrationPlanEntry {
+  hash: string;
+  folderMillis: number;
+  statements: string[];
+}
+
+/** The persistence operations the hash-ledger core needs, per tier. */
+export interface HashLedgerDeps {
+  /** Create the drizzle schema + ledger table if absent (idempotent). */
+  ensureLedger(): Promise<void>;
+  /** Hashes already recorded in the ledger. */
+  appliedHashes(): Promise<string[]>;
+  /** Record one fully-applied migration in the ledger. */
+  recordApplied(hash: string, folderMillis: number): Promise<void>;
+  /** Execute one SQL statement, autocommitting (never inside a wider tx). */
+  runStatement(statement: string): Promise<void>;
+}
+
+function loadPlan(): MigrationPlanEntry[] {
+  return readMigrationFiles({ migrationsFolder: migrationsFolder() }).map((m) => ({
+    hash: m.hash,
+    folderMillis: m.folderMillis,
+    statements: m.sql.map((s) => s.trim()).filter((s) => s.length > 0),
+  }));
+}
+
+/**
+ * Apply every unapplied migration in journal order, skipping by hash.
+ *
+ * Failure model: each statement autocommits, and the ledger row is written
+ * only after all statements of a migration succeed. A crash mid-migration
+ * leaves the prefix applied with no ledger entry, so the next boot replays
+ * those statements and fails loudly on "already exists". Recovery: drop the
+ * partially-applied objects, or insert the migration's hash into
+ * `drizzle.__drizzle_migrations` manually. The thrown error identifies the
+ * statement and migration that failed.
+ */
+export async function applyMigrationsByHash(
+  deps: HashLedgerDeps,
+  plan: MigrationPlanEntry[] = loadPlan(),
+): Promise<{ applied: number; skipped: number }> {
+  await deps.ensureLedger();
+  const alreadyApplied = new Set(await deps.appliedHashes());
+
+  let applied = 0;
+  let skipped = 0;
+  for (const migration of plan) {
+    if (alreadyApplied.has(migration.hash)) {
+      skipped += 1;
+      continue;
+    }
+    for (const [stmtIdx, stmt] of migration.statements.entries()) {
+      try {
+        await deps.runStatement(stmt);
+      } catch (err) {
+        const cause = err instanceof Error ? err.message : String(err);
+        throw new Error(
+          `migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` +
+            `Statement: ${stmt.slice(0, 200)}${stmt.length > 200 ? '…' : ''}`,
+          { cause: err },
+        );
+      }
+    }
+    await deps.recordApplied(migration.hash, migration.folderMillis);
+    applied += 1;
+  }
+  return { applied, skipped };
+}
+
+const LEDGER_DDL = [
+  'CREATE SCHEMA IF NOT EXISTS drizzle',
+  `CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
+      id SERIAL PRIMARY KEY,
+      hash text NOT NULL,
+      created_at bigint
+    )`,
+];
+
+function connectionString(url?: string): string {
+  return url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL;
+}
+
+/**
+ * Apply Drizzle migrations against a postgres database, hash-ledger style.
+ * Idempotent: re-running against a fully-migrated database applies nothing.
+ */
 export async function runMigrations(url?: string): Promise<void> {
-  const connectionString = url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL;
-  const sqlClient = postgres(connectionString, { max: 1 });
-  const db = drizzlePostgres(sqlClient);
+  const sqlClient = postgres(connectionString(url), { max: 1 });
   try {
-    // TODO: postgres-tier first-install also fails because (a) Drizzle wraps every
-    // migration in one transaction (breaks 0009's ALTER TYPE ADD VALUE → SET DEFAULT
-    // sequence) and (b) drizzle/meta/_journal.json has 0009 ordered before 0008,
-    // which the postgres-js migrator skips by `created_at < folderMillis`. The
-    // PGlite path below sidesteps both. A follow-up should either share the
-    // per-statement loop (see runPgliteMigrations) or fix the journal ordering.
-    await migratePostgres(db, { migrationsFolder: migrationsFolder() });
+    await applyMigrationsByHash({
+      ensureLedger: async () => {
+        for (const ddl of LEDGER_DDL) await sqlClient.unsafe(ddl);
+      },
+      appliedHashes: async () => {
+        const rows = (await sqlClient.unsafe(
+          'SELECT hash FROM drizzle.__drizzle_migrations',
+        )) as Array<{ hash: string }>;
+        return rows.map((r) => String(r.hash));
+      },
+      recordApplied: async (hash, folderMillis) => {
+        await sqlClient.unsafe(
+          'INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ($1, $2)',
+          [hash, folderMillis],
+        );
+      },
+      runStatement: async (stmt) => {
+        await sqlClient.unsafe(stmt);
+      },
+    });
   } finally {
     await sqlClient.end();
   }
 }
 
-// Apply Drizzle migrations against an embedded PGlite database.
-//
-// We don't reuse drizzle's pglite migrator because it wraps ALL migrations in
-// one outer transaction, which breaks Postgres' `check_safe_enum_use` rule —
-// e.g. migration 0009 does `ALTER TYPE ADD VALUE 'pending'` then references
-// `'pending'` as a default in the same tx. PGlite's `exec()` runs each
-// statement under the Simple Query protocol, autocommitting between them.
-//
-// We still write to the standard `drizzle.__drizzle_migrations` ledger so the
-// result is interoperable with `runMigrations()` on a postgres-backed deploy
-// (modulo the journal-ordering bug noted above).
-//
-// We skip-by-hash rather than skip-by-folderMillis (which is what Drizzle's
-// postgres-js migrator does). That's deliberate — out-of-order timestamps in
-// `_journal.json` won't silently drop migrations.
-//
-// Failure model: each statement autocommits, and the ledger row is written
-// only after all statements in a migration succeed. A crash mid-migration
-// leaves the prefix applied with no ledger entry, so the next boot will
-// replay those statements and fail loudly on "already exists". Recovery:
-// drop the partially-applied objects, or insert the migration's hash into
-// `drizzle.__drizzle_migrations` manually. The error log identifies which
-// statement of which migration was the culprit.
+/**
+ * Apply Drizzle migrations against an embedded PGlite database.
+ *
+ * We don't reuse drizzle's pglite migrator for the same reasons as the
+ * postgres path (single-transaction wrap; folderMillis skip). PGlite's
+ * `exec()` runs each statement under the Simple Query protocol,
+ * autocommitting between them — exactly the semantics the shared core needs.
+ *
+ * The ledger rows this writes are interoperable with the postgres path (same
+ * schema, same hashes), because both consume the same shipped migrations.
+ */
 export async function runPgliteMigrations(handle: DbHandle): Promise<void> {
   const client = (handle.db as unknown as { $client?: PgliteExecutor }).$client;
   if (!client || typeof client.exec !== 'function') {
     throw new Error('runPgliteMigrations: handle.db is not backed by a PGlite client');
   }
 
-  await client.exec('CREATE SCHEMA IF NOT EXISTS drizzle');
-  await client.exec(`
-    CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
-      id SERIAL PRIMARY KEY,
-      hash text NOT NULL,
-      created_at bigint
-    )
-  `);
+  await applyMigrationsByHash({
+    ensureLedger: async () => {
+      for (const ddl of LEDGER_DDL) await client.exec(ddl);
+    },
+    appliedHashes: async () => {
+      const rows = (await handle.db.execute(
+        sql`SELECT hash FROM drizzle.__drizzle_migrations`,
+      )) as unknown as ExecuteRows<{ hash: string }>;
+      return rows.rows.map((r) => String(r.hash));
+    },
+    recordApplied: async (hash, folderMillis) => {
+      await handle.db.execute(
+        sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${hash}, ${folderMillis})`,
+      );
+    },
+    runStatement: async (stmt) => {
+      await client.exec(stmt);
+    },
+  });
+}
 
-  const appliedRows = (await handle.db.execute(
-    sql`SELECT hash FROM drizzle.__drizzle_migrations`,
-  )) as unknown as ExecuteRows<{ hash: string }>;
-  const applied = new Set(appliedRows.rows.map((r) => r.hash));
+/* ------------------------------------------------------------------ */
+/*  Migration status (#1392: the installer must VERIFY, not assume)     */
+/* ------------------------------------------------------------------ */
 
-  const migrations = readMigrationFiles({ migrationsFolder: migrationsFolder() });
-  for (const migration of migrations) {
-    if (applied.has(migration.hash)) continue;
+/** Read the journal tags (migration folder names) in journal order. */
+export function readJournalTags(folder: string = migrationsFolder()): string[] {
+  const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as {
+    entries?: Array<{ tag?: string }>;
+  };
+  return (journal.entries ?? []).map((e) => e.tag ?? '').filter((t) => t.length > 0);
+}
 
-    // Run each statement-breakpoint chunk in its own exec() call so PGlite
-    // commits between statements — this is what lets `ALTER TYPE ADD VALUE`
-    // become visible before a subsequent statement references the new value.
-    for (const [stmtIdx, stmt] of migration.sql.entries()) {
-      const trimmed = stmt.trim();
-      if (!trimmed) continue;
-      try {
-        await client.exec(trimmed);
-      } catch (err) {
-        const cause = err instanceof Error ? err.message : String(err);
-        throw new Error(
-          `runPgliteMigrations: migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` +
-            `Statement: ${trimmed.slice(0, 200)}${trimmed.length > 200 ? '…' : ''}`,
-          { cause: err },
-        );
-      }
+export interface MigrationStatus {
+  /** Hashes recorded in the database's ledger (0 if no ledger exists). */
+  appliedCount: number;
+  /** Migrations shipped in this package's drizzle/ folder. */
+  expectedCount: number;
+  /** The tag (folder name) of the last journal entry, for error messages. */
+  expectedLastTag: string;
+  /** True iff every shipped migration's hash is in the ledger. */
+  complete: boolean;
+}
+
+/**
+ * Report whether a postgres database carries the full shipped schema.
+ *
+ * Read-only apart from `ensureLedger` semantics: it never creates the ledger
+ * (unlike the migrators), so a database with NO ledger reports
+ * appliedCount=0 / complete=false — the exact #1392/#1389 signature (an
+ * install whose dependency set shipped no migrations at all).
+ */
+export async function getMigrationStatus(url?: string): Promise<MigrationStatus> {
+  const sqlClient = postgres(connectionString(url), { max: 1 });
+  try {
+    const plan = loadPlan();
+    const tags = readJournalTags();
+    const expectedHashes = new Set(plan.map((m) => m.hash));
+
+    let appliedHashes: string[] = [];
+    const regRows = (await sqlClient.unsafe(
+      "SELECT to_regclass('drizzle.__drizzle_migrations') AS reg",
+    )) as Array<{ reg: string | null }>;
+    if (regRows[0]?.reg) {
+      const rows = (await sqlClient.unsafe(
+        'SELECT hash FROM drizzle.__drizzle_migrations',
+      )) as Array<{ hash: string }>;
+      appliedHashes = rows.map((r) => String(r.hash));
     }
-    await handle.db.execute(
-      sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${migration.hash}, ${migration.folderMillis})`,
-    );
+
+    const appliedSet = new Set(appliedHashes);
+    return {
+      appliedCount: appliedHashes.length,
+      expectedCount: plan.length,
+      expectedLastTag: tags[tags.length - 1] ?? '',
+      complete:
+        plan.length > 0 &&
+        plan.every((m) => appliedSet.has(m.hash)) &&
+        // A ledger with entries OUTSIDE the shipped plan means the database
+        // came from a different (e.g. newer) build — not "complete" either.
+        appliedHashes.every((h) => expectedHashes.has(h)),
+    };
+  } finally {
+    await sqlClient.end();
   }
 }

packages/mosaic/src/commands/gateway/install.ts

@@ -15,6 +15,7 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { ClackPrompter } from '../../prompter/clack-prompter.js';
+import type { VerifyResult } from './verify.js';
 import type { WizardState } from '../../types.js';
 
 interface InstallOpts {
@@ -89,13 +90,23 @@ export async function runInstall(opts: InstallOpts): Promise<void> {
     prompter.log(`  Logs:      mosaic gateway logs`);
     prompter.log(`  Status:    mosaic gateway status`);
 
-    // Post-install verification (CU-07-03) — non-fatal.
+    // Post-install verification (CU-07-03). Health/token/bootstrap failures
+    // stay non-fatal (courtesy checks), but a FAILED database schema check is
+    // fatal (#1392): an install that reports success over an empty/partial
+    // database is the exact T63 failure this command must never reproduce.
+    let verifyResult: VerifyResult | undefined;
     try {
       const { runPostInstallVerification } = await import('./verify.js');
-      await runPostInstallVerification(configResult.host, configResult.port);
+      verifyResult = await runPostInstallVerification(configResult.host, configResult.port);
     } catch {
       // Non-fatal — verification is a courtesy
     }
+    if (verifyResult && verifyResult.schemaMigrated === false) {
+      prompter.warn(
+        'Gateway install ABORTED: database schema verification failed (remediation above).',
+      );
+      process.exit(1);
+    }
   } catch (err) {
     // Stages normally return structured results for expected failures.
     // Anything that reaches here is an unexpected runtime error — render a

packages/mosaic/src/commands/gateway/schema-check.spec.ts

@@ -0,0 +1,156 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  checkDatabaseSchema,
+  SCHEMA_FAIL_REMEDIATION,
+  type SchemaCheckDeps,
+} from './schema-check.js';
+
+/* ------------------------------------------------------------------ */
+/*  Fixture config                                                     */
+/* ------------------------------------------------------------------ */
+
+const tmpDirs: string[] = [];
+
+function writeConfig(cfg: Record<string, unknown>): string {
+  const dir = mkdtempSync(join(tmpdir(), 'schema-check-'));
+  tmpDirs.push(dir);
+  const path = join(dir, 'mosaic.config.json');
+  writeFileSync(path, JSON.stringify(cfg));
+  return path;
+}
+
+const STANDALONE_CFG = {
+  tier: 'standalone',
+  storage: { type: 'postgres', url: 'postgresql://u:p@localhost:5434/x' },
+  queue: { type: 'bullmq', url: 'redis://localhost:6380' },
+  memory: { type: 'keyword' },
+};
+
+const LOCAL_CFG = {
+  tier: 'local',
+  storage: { type: 'pglite', dataDir: '.mosaic/storage-pglite' },
+  queue: { type: 'local', dataDir: '.mosaic/queue' },
+  memory: { type: 'keyword' },
+};
+
+function deps(overrides: Partial<SchemaCheckDeps> = {}): SchemaCheckDeps {
+  return {
+    runMigrations: vi.fn().mockResolvedValue(undefined),
+    getMigrationStatus: vi.fn().mockResolvedValue({
+      appliedCount: 17,
+      expectedCount: 17,
+      expectedLastTag: '0016_x',
+      complete: true,
+    }),
+    ...overrides,
+  };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Tests                                                              */
+/* ------------------------------------------------------------------ */
+
+describe('checkDatabaseSchema', () => {
+  beforeEach(() => {
+    process.env['MOSAIC_CONFIG'] = '';
+    vi.stubEnv('DATABASE_URL', 'postgresql://env:env@localhost:9999/env');
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
+  });
+
+  it('passes when the ledger matches the shipped journal', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps();
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('ok');
+    expect(result.detail).toContain('17/17');
+    expect(d.runMigrations).toHaveBeenCalledWith(STANDALONE_CFG.storage.url);
+    expect(d.getMigrationStatus).toHaveBeenCalledWith(STANDALONE_CFG.storage.url);
+  });
+
+  it('FAILS when the ledger is incomplete — the #1389 empty-database signature', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      getMigrationStatus: vi.fn().mockResolvedValue({
+        appliedCount: 0,
+        expectedCount: 17,
+        expectedLastTag: '0016_x',
+        complete: false,
+      }),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    if (result.status !== 'fail') throw new Error('expected fail');
+    expect(result.detail).toContain('0/17');
+    expect(result.remediation).toBe(SCHEMA_FAIL_REMEDIATION);
+    expect(result.remediation).toContain('#1389');
+  });
+
+  it('FAILS when the ledger is only PARTIALLY migrated (#1402 upgrade case)', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      getMigrationStatus: vi.fn().mockResolvedValue({
+        appliedCount: 9,
+        expectedCount: 17,
+        expectedLastTag: '0016_x',
+        complete: false,
+      }),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    expect(result.detail).toContain('9/17');
+  });
+
+  it('FAILS (never crashes) when the migration run itself throws', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      runMigrations: vi.fn().mockRejectedValue(new Error('connection refused')),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    expect(result.detail).toContain('connection refused');
+  });
+
+  it('skips the local tier (gateway migrates its own PGlite at startup)', async () => {
+    const cfg = writeConfig(LOCAL_CFG);
+    const d = deps();
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('skipped');
+    expect(d.runMigrations).not.toHaveBeenCalled();
+  });
+});
+
+/* ------------------------------------------------------------------ */
+/*  Config resolution priority                                         */
+/* ------------------------------------------------------------------ */
+
+describe('resolveSchemaCheckConfigPath', () => {
+  it('prefers the daemon-written gateway config over cwd copies', async () => {
+    const { resolveSchemaCheckConfigPath } = await import('./schema-check.js');
+    const cwdCfg = writeConfig(STANDALONE_CFG); // in a temp dir
+    const daemonDir = mkdtempSync(join(tmpdir(), 'schema-check-daemon-'));
+    tmpDirs.push(daemonDir);
+    const daemonHome = join(daemonDir, '.config', 'mosaic', 'gateway');
+    mkdirSync(daemonHome, { recursive: true });
+    writeFileSync(join(daemonHome, 'mosaic.config.json'), JSON.stringify(LOCAL_CFG));
+
+    const prevHome = process.env['HOME'];
+    vi.stubEnv('HOME', daemonDir);
+    vi.stubEnv('MOSAIC_CONFIG', '');
+    try {
+      const resolved = resolveSchemaCheckConfigPath();
+      // Must NOT pick the cwd copy (cwd is the vitest project dir, not our
+      // temp dir — so the only resolvable candidates are daemon + $HOME/.mosaic).
+      expect(resolved).toBe(join(daemonHome, 'mosaic.config.json'));
+      void cwdCfg;
+    } finally {
+      if (prevHome !== undefined) vi.stubEnv('HOME', prevHome);
+    }
+  });
+});

packages/mosaic/src/commands/gateway/schema-check.ts

@@ -0,0 +1,107 @@
+/**
+ * Install-time database schema verification (#1392).
+ *
+ * Fresh standalone (postgres) installs once ended "healthy" with a completely
+ * empty database: the resolved dependency set shipped no migrations at all
+ * (#1389), the gateway started fine, and nothing failed until the first real
+ * query. The issue's own conclusion: the installer must "instruct and verify".
+ *
+ * This check runs AFTER migrations have been (re-)applied, so the only way it
+ * fails is a genuinely broken migration set or database — which is exactly
+ * when install must not report success. Failure is fatal by design (fast-fail
+ * STANDARDS); callers print the remediation text and exit non-zero.
+ */
+
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { loadConfig } from '@mosaicstack/config';
+
+export interface SchemaStatusCounts {
+  appliedCount: number;
+  expectedCount: number;
+  expectedLastTag: string;
+  complete: boolean;
+}
+
+/** Injectable migration surface — keeps this unit-testable without a DB. */
+export interface SchemaCheckDeps {
+  runMigrations(url: string): Promise<void>;
+  getMigrationStatus(url: string): Promise<SchemaStatusCounts>;
+}
+
+export type SchemaCheckResult =
+  | { status: 'ok'; detail: string }
+  | { status: 'skipped'; detail: string }
+  | { status: 'fail'; detail: string; remediation: string };
+
+export const SCHEMA_FAIL_REMEDIATION = [
+  'The gateway database does not carry the full schema.',
+  'Causes seen in the wild: dependency set resolved without migrations (#1389), or a partially-migrated database (#1402).',
+  'Remediation:',
+  '  1. Re-run: mosaic gateway install  (applies migrations and verifies again)',
+  '  2. Check the resolved @mosaicstack/db version is the same pipeline as the gateway (npm ls -g @mosaicstack/db)',
+  '  3. Manual apply: run runMigrations() from @mosaicstack/db against the storage URL, then re-verify',
+].join('\n');
+
+/**
+ * Resolve the config the INSTALLED gateway would use — same priority the
+ * daemon applies (apps/gateway/src/env.ts resolveGatewayConfigPath), minus
+ * the source-tree anchors that do not exist on an installed host. Verifying
+ * against any other config could green-light a database the daemon never
+ * reads (#1392: verify what runs, not what happens to lie in cwd).
+ */
+export function resolveSchemaCheckConfigPath(explicit?: string): string | undefined {
+  if (explicit) return resolve(explicit);
+  const candidates = [
+    process.env['MOSAIC_CONFIG'],
+    join(homedir(), '.config', 'mosaic', 'gateway', 'mosaic.config.json'), // daemon-written
+    resolve(process.cwd(), 'mosaic.config.json'),
+    join(homedir(), '.mosaic', 'mosaic.config.json'),
+  ];
+  for (const c of candidates) {
+    if (c && existsSync(c)) return c;
+  }
+  return undefined; // loadConfig falls back to env-var detection
+}
+
+export async function checkDatabaseSchema(
+  deps: SchemaCheckDeps,
+  configPath?: string,
+): Promise<SchemaCheckResult> {
+  const config = loadConfig(resolveSchemaCheckConfigPath(configPath));
+
+  // Local tier: the gateway itself runs PGlite migrations at startup (see
+  // DatabaseModule.onModuleInit), and a broken local tier fails the health
+  // check instead. Nothing for the installer to verify here.
+  if (config.storage.type !== 'postgres') {
+    return {
+      status: 'skipped',
+      detail: 'database schema (local tier — migrated by gateway at startup)',
+    };
+  }
+
+  const url = config.storage.url;
+  try {
+    await deps.runMigrations(url);
+    const status = await deps.getMigrationStatus(url);
+    if (status.complete) {
+      return {
+        status: 'ok',
+        detail: `database schema (${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations)`,
+      };
+    }
+    return {
+      status: 'fail',
+      detail: `database schema incomplete (${status.appliedCount.toString()}/${status.expectedCount.toString()} applied, last expected: ${status.expectedLastTag})`,
+      remediation: SCHEMA_FAIL_REMEDIATION,
+    };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      status: 'fail',
+      detail: `database schema check errored: ${msg}`,
+      remediation: SCHEMA_FAIL_REMEDIATION,
+    };
+  }
+}

packages/mosaic/src/commands/gateway/verify.ts

@@ -44,6 +44,8 @@ export interface VerifyResult {
   gatewayHealthy: boolean;
   adminTokenOnFile: boolean;
   bootstrapReachable: boolean;
+  /** False only on a FAILED postgres schema check; true when ok or skipped. */
+  schemaMigrated: boolean;
   allPassed: boolean;
 }
 
@@ -89,7 +91,36 @@ export async function runPostInstallVerification(
     fail('bootstrap endpoint reach', 'Run: mosaic gateway status  /  mosaic gateway logs');
   }
 
-  const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable;
+  // ─── Check 4: Database schema migrated (#1392) ────────────────────────────
+  // Fatal-on-failure for install: the #1389 failure mode was an install that
+  // reported success over an empty database. Local tiers skip (the gateway
+  // migrates its own PGlite at startup and would fail health if it couldn't).
+  let schemaMigrated = true;
+  try {
+    const { checkDatabaseSchema } = await import('./schema-check.js');
+    const { runMigrations, getMigrationStatus } = await import('@mosaicstack/db');
+    const result = await checkDatabaseSchema(
+      { runMigrations, getMigrationStatus },
+      process.env['MOSAIC_CONFIG'],
+    );
+    if (result.status === 'ok') {
+      ok(result.detail);
+    } else if (result.status === 'skipped') {
+      ok(result.detail);
+    } else {
+      fail(result.detail, result.remediation);
+      schemaMigrated = false;
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    fail(
+      `database schema check errored: ${msg}`,
+      'See #1392/#1389; re-run: mosaic gateway install',
+    );
+    schemaMigrated = false;
+  }
+
+  const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable && schemaMigrated;
 
   if (!allPassed) {
     console.log(
@@ -98,7 +129,7 @@ export async function runPostInstallVerification(
     );
   }
 
-  return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, allPassed };
+  return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, schemaMigrated, allPassed };
 }
 
 /**