diff --git a/apps/gateway/src/database/database.module.1392.spec.ts b/apps/gateway/src/database/database.module.1392.spec.ts new file mode 100644 index 00000000..f339e65c --- /dev/null +++ b/apps/gateway/src/database/database.module.1392.spec.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi } from 'vitest'; + +// The module under test imports @mosaicstack/db at module scope; we replace only the +// pieces DatabaseModule uses (partial mock — the real module also exports the +// schema the storage adapter's import chain needs) so the test pins the #1392 +// contract (refuse to start on an incomplete schema) without a live database. +vi.mock('@mosaicstack/db', async (importOriginal) => { + const actual: object = await importOriginal(); + return { + ...actual, + createDb: vi.fn(), + createPgliteDb: vi.fn(), + getMigrationStatus: vi.fn(), + runPgliteMigrations: vi.fn(), + }; +}); + +import { DatabaseModule } from './database.module.js'; +import { getMigrationStatus } from '@mosaicstack/db'; +import type { DbHandle } from '@mosaicstack/db'; +import type { StorageAdapter } from '@mosaicstack/storage'; +import type { MosaicConfig } from '@mosaicstack/config'; + +function makeModule(storageType: 'postgres' | 'pglite', tier: string) { + const storageAdapter = { + name: storageType, + migrate: vi.fn(), + close: vi.fn(), + } as unknown as StorageAdapter; + const handle = { close: vi.fn() } as unknown as DbHandle; + const config = { + tier, + storage: { type: storageType, url: 'postgresql://x' }, + } as unknown as MosaicConfig; + return { + mod: new DatabaseModule(handle, storageAdapter, config), + storageAdapter, + }; +} + +describe('DatabaseModule.onModuleInit — #1392 schema verification', () => { + it('refuses to start when the postgres schema is incomplete', async () => { + const { mod, storageAdapter } = makeModule('postgres', 'standalone'); + vi.mocked(getMigrationStatus).mockResolvedValue({ + appliedCount: 15, + expectedCount: 17, + expectedLastTag: '0016_salty_morlocks', + complete: false, + }); + await expect(mod.onModuleInit()).rejects.toThrow('Database schema incomplete: 15/17'); + expect(storageAdapter.migrate).toHaveBeenCalled(); // migrations attempted first + }); + + it('starts normally when the schema is complete', async () => { + const { mod } = makeModule('postgres', 'standalone'); + vi.mocked(getMigrationStatus).mockResolvedValue({ + appliedCount: 17, + expectedCount: 17, + expectedLastTag: '0016_salty_morlocks', + complete: true, + }); + await expect(mod.onModuleInit()).resolves.toBeUndefined(); + }); + + it('does not verify postgres status for the local tier (PGlite migrates itself)', async () => { + const { mod } = makeModule('pglite', 'local'); + vi.mocked(getMigrationStatus).mockClear(); + await expect(mod.onModuleInit()).resolves.toBeUndefined(); + expect(getMigrationStatus).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/gateway/src/database/database.module.ts b/apps/gateway/src/database/database.module.ts index 0e6f6480..d0188c6f 100644 --- a/apps/gateway/src/database/database.module.ts +++ b/apps/gateway/src/database/database.module.ts @@ -12,6 +12,7 @@ import { import { createDb, createPgliteDb, + getMigrationStatus, runPgliteMigrations, type Db, type DbHandle, @@ -74,6 +75,11 @@ export class DatabaseModule implements OnApplicationShutdown, OnModuleInit { // the same DATABASE_URL, so a single call covers both the gateway DB and // the storage tables. We deliberately do NOT call runMigrations() here to // avoid opening a second short-lived connection and doubling startup cost. + // + // #1392: we DO verify afterwards (getMigrationStatus opens one short-lived + // connection) and refuse to start on an incomplete schema. A gateway that + // boots "healthy" on an empty or partial database is precisely the failure + // that shipped in the T63 batch: silent at startup, catastrophic later. async onModuleInit(): Promise { if (this.config.tier === 'local') { this.logger.log('Applying PGlite schema migrations...'); @@ -81,6 +87,24 @@ export class DatabaseModule implements OnApplicationShutdown, OnModuleInit { } this.logger.log(`Initializing storage adapter (${this.storageAdapter.name})...`); await this.storageAdapter.migrate(); + + if (this.config.storage.type === 'postgres') { + const status = await getMigrationStatus(this.config.storage.url); + if (!status.complete) { + this.logger.error( + `Database schema incomplete: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied ` + + `(last expected: ${status.expectedLastTag}). ` + + 'Refusing to start on a partial schema — see issues #1392/#1402. ' + + "Remediation: re-run 'mosaic gateway install' (it now verifies), or apply migrations manually.", + ); + throw new Error( + `Database schema incomplete: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied`, + ); + } + this.logger.log( + `Database schema verified: ${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations applied.`, + ); + } } async onApplicationShutdown(): Promise { diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 9d823ad7..d50ec561 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,6 +1,15 @@ export { createDb, type Db, type DbHandle } from './client.js'; export { createPgliteDb } from './client-pglite.js'; -export { runMigrations, runPgliteMigrations } from './migrate.js'; +export { + runMigrations, + runPgliteMigrations, + getMigrationStatus, + readJournalTags, + applyMigrationsByHash, + type HashLedgerDeps, + type MigrationPlanEntry, + type MigrationStatus, +} from './migrate.js'; export * from './schema.js'; export * from './federation.js'; export { diff --git a/packages/db/src/migrate.spec.ts b/packages/db/src/migrate.spec.ts new file mode 100644 index 00000000..fb91e0a3 --- /dev/null +++ b/packages/db/src/migrate.spec.ts @@ -0,0 +1,169 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { + applyMigrationsByHash, + readJournalTags, + type HashLedgerDeps, + type MigrationPlanEntry, +} from './migrate.js'; + +/* ------------------------------------------------------------------ */ +/* In-memory hash-ledger harness */ +/* ------------------------------------------------------------------ */ + +interface LedgerHarness extends HashLedgerDeps { + ledger: Map; + /** Recorded statement executions in order: `${hashPrefix}:${stmtIndex}`. */ + executed: string[]; + /** Optional: statements that should throw when executed. */ + failOn?: (migrationHash: string, stmtIdx: number) => boolean; +} + +function makeHarness(plan: MigrationPlanEntry[]): LedgerHarness { + const hashToEntry = new Map(plan.map((p) => [p.hash, p])); + const h: LedgerHarness = { + ledger: new Map(), + executed: [], + ensureLedger: async () => {}, + appliedHashes: async () => [...h.ledger.keys()], + recordApplied: async (hash, folderMillis) => { + h.ledger.set(hash, folderMillis); + }, + runStatement: async (statement) => { + void statement; + // runStatement does not know which migration it belongs to; the + // executed log is filled by the wrapper below. + }, + }; + // Wrap runStatement so the executed log records migration context. We + // reconstruct context by tracking a cursor the core advances per migration. + let cursor = 0; + const flat: Array<{ hash: string; idx: number }> = []; + for (const m of plan) + for (const [i] of m.statements.entries()) flat.push({ hash: m.hash, idx: i }); + h.runStatement = async () => { + const at = flat[cursor] ?? { hash: '??', idx: -1 }; + cursor += 1; + if (h.failOn && at.hash !== '??' && h.failOn(at.hash, at.idx)) { + throw new Error(`simulated failure in ${at.hash} #${at.idx.toString()}`); + } + h.executed.push(`${at.hash.slice(0, 6)}:${at.idx.toString()}`); + }; + void hashToEntry; + return h; +} + +/* ------------------------------------------------------------------ */ +/* Fixtures */ +/* ------------------------------------------------------------------ */ + +// Reproduces the REAL journal defect shape (#1402 D1): 0009/0010 carry +// `when` timestamps BELOW 0008's. Under the old drizzle postgres-js +// migrator these were silently skipped on any upgrade whose ledger was +// last stamped in the 0008 era. +const JOURNAL_FIXTURE: MigrationPlanEntry[] = [ + { hash: 'aaaa0000', folderMillis: 1773368153122, statements: ['CREATE TABLE a (id int)'] }, + { hash: 'bbbb0008', folderMillis: 1776822435828, statements: ['CREATE TABLE b (id int)'] }, + // Backdated entries, exactly as shipped: + { + hash: 'cccc0009', + folderMillis: 1745280000000, + statements: ['ALTER TYPE t ADD VALUE', "CREATE TABLE c (s t DEFAULT 'pending')"], + }, + { hash: 'dddd0010', folderMillis: 1745366400000, statements: ['CREATE TABLE d (id int)'] }, +]; + +/** A ledger last stamped at the 0008 era: only pre-0009 hashes recorded. */ +const LEDGER_AT_0008_ERA = new Map([ + ['aaaa0000', 1773368153122], + ['bbbb0008', 1776822435828], +]); + +/* ------------------------------------------------------------------ */ +/* The core: apply-by-hash in journal order */ +/* ------------------------------------------------------------------ */ + +describe('applyMigrationsByHash', () => { + it('applies backdated journal entries that a timestamp-based migrator would skip (#1402 D1)', async () => { + const h = makeHarness(JOURNAL_FIXTURE); + h.ledger = new Map(LEDGER_AT_0008_ERA); + + const result = await applyMigrationsByHash(h, JOURNAL_FIXTURE); + + // D1 in one sentence: 0009 and 0010 applied despite folderMillis < 0008. + expect(result).toEqual({ applied: 2, skipped: 2 }); + expect(h.ledger.has('cccc0009')).toBe(true); + expect(h.ledger.has('dddd0010')).toBe(true); + }); + + it('executes statements individually (ALTER TYPE visibility, #1402 D2 shape)', async () => { + const h = makeHarness(JOURNAL_FIXTURE); + await applyMigrationsByHash(h, JOURNAL_FIXTURE); + // 0009's two statements recorded as separate executions, in order. + expect(h.executed).toContain('cccc00:0'); + expect(h.executed).toContain('cccc00:1'); + expect(h.executed.indexOf('cccc00:0')).toBeLessThan(h.executed.indexOf('cccc00:1')); + }); + + it('is idempotent: a fully-applied ledger applies nothing', async () => { + const h = makeHarness(JOURNAL_FIXTURE); + const first = await applyMigrationsByHash(h, JOURNAL_FIXTURE); + const second = await applyMigrationsByHash(h, JOURNAL_FIXTURE); + expect(first.applied).toBe(4); + expect(second).toEqual({ applied: 0, skipped: 4 }); + expect(h.executed).toHaveLength(5); // 5 statements; second run executed NONE (not 10) + }); + + it('records no ledger row when a statement fails (crash prefix replays loudly)', async () => { + const h = makeHarness(JOURNAL_FIXTURE); + h.failOn = (hash, idx) => hash === 'cccc0009' && idx === 1; + + await expect(applyMigrationsByHash(h, JOURNAL_FIXTURE)).rejects.toThrow( + /cccc0009 statement #1 failed: simulated failure/, + ); + // Statement 0 of 0009 executed, but NO ledger row for 0009: the next run + // replays it and fails loudly on "already exists" instead of silently + // believing 0009 applied. + expect(h.ledger.has('cccc0009')).toBe(false); + expect(h.executed).toContain('cccc00:0'); + }); + + it('applies in JOURNAL order, not timestamp order', async () => { + const h = makeHarness(JOURNAL_FIXTURE); + await applyMigrationsByHash(h, JOURNAL_FIXTURE); + // 5 statements total (0009 has two); prefix per migration: journal order, + // so 0009's pair sits between 0008 and 0010. + const order = h.executed.map((e) => e.slice(0, 4)); + expect(order).toEqual(['aaaa', 'bbbb', 'cccc', 'cccc', 'dddd']); + }); +}); + +/* ------------------------------------------------------------------ */ +/* Journal integrity against the shipped folder */ +/* ------------------------------------------------------------------ */ + +describe('readJournalTags', () => { + it('reads the shipped journal in order and sees the known backdated pair', () => { + const folder = resolve(__dirname, '../drizzle'); + const tags = readJournalTags(folder); + expect(tags.length).toBeGreaterThan(0); + // The shipped defect (#1402 D1): these two entries carry April-2025 + // timestamps below 0008's June-2026 one. If this assertion ever fails + // because the journal was FIXED (timestamps corrected or drizzle-kit + // regenerated), update #1402 — the hash-ledger core stays correct either + // way; this test pins the shipped reality the core was built for. + const t9 = tags.find((t) => t.startsWith('0009_')); + const t10 = tags.find((t) => t.startsWith('0010_')); + const t8 = tags.find((t) => t.startsWith('0008_')); + expect([t8, t9, t10]).toBeDefined(); + const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as { + entries: Array<{ tag: string; when: number }>; + }; + const when = new Map(journal.entries.map((e) => [e.tag, e.when])); + if (t8 && t9 && t10) { + expect(when.get(t9)!).toBeLessThan(when.get(t8)!); // backdated below 0008 + expect(when.get(t10)!).toBeLessThan(when.get(t8)!); // backdated below 0008 + } + }); +}); diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index 73af1742..13dd9d04 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -1,8 +1,7 @@ +import { readFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { sql } from 'drizzle-orm'; -import { drizzle as drizzlePostgres } from 'drizzle-orm/postgres-js'; -import { migrate as migratePostgres } from 'drizzle-orm/postgres-js/migrator'; import { readMigrationFiles } from 'drizzle-orm/migrator'; import postgres from 'postgres'; import { DEFAULT_DATABASE_URL } from './defaults.js'; @@ -21,89 +20,243 @@ function migrationsFolder(): string { return resolve(here, '../drizzle'); } +/* ------------------------------------------------------------------ */ +/* Shared hash-ledger migration core (#1392 / #1402) */ +/* ------------------------------------------------------------------ */ +// +// Both tiers migrate through this single core, which applies migrations in +// JOURNAL ORDER, one statement at a time, and skips by HASH — never by +// folderMillis timestamp. The previous postgres path delegated to drizzle's +// postgres-js migrator, which: +// +// * applies only migrations with folderMillis > last-applied, silently +// skipping journal entries whose `when` is older than the ledger's newest +// stamp — 0009/0010 carry April-2025 timestamps below 0008's June-2026 +// one, so any database last migrated in the 0008 era silently loses +// 0009/0010 forever (#1402 D1); and +// * wraps each migration in ONE transaction, which breaks migrations that +// do `ALTER TYPE ADD VALUE` and then reference the new value in the same +// migration (0009) — Postgres' check_safe_enum_use rejects it (#1402 D2). +// +// Per-statement execution (each statement autocommits) and skip-by-hash fix +// both. The PGlite path has run this way since it was written; this is the +// TODO it left behind, now shared instead of duplicated. + +/** One migration as loaded from the shipped drizzle/ folder. */ +export interface MigrationPlanEntry { + hash: string; + folderMillis: number; + statements: string[]; +} + +/** The persistence operations the hash-ledger core needs, per tier. */ +export interface HashLedgerDeps { + /** Create the drizzle schema + ledger table if absent (idempotent). */ + ensureLedger(): Promise; + /** Hashes already recorded in the ledger. */ + appliedHashes(): Promise; + /** Record one fully-applied migration in the ledger. */ + recordApplied(hash: string, folderMillis: number): Promise; + /** Execute one SQL statement, autocommitting (never inside a wider tx). */ + runStatement(statement: string): Promise; +} + +function loadPlan(): MigrationPlanEntry[] { + return readMigrationFiles({ migrationsFolder: migrationsFolder() }).map((m) => ({ + hash: m.hash, + folderMillis: m.folderMillis, + statements: m.sql.map((s) => s.trim()).filter((s) => s.length > 0), + })); +} + +/** + * Apply every unapplied migration in journal order, skipping by hash. + * + * Failure model: each statement autocommits, and the ledger row is written + * only after all statements of a migration succeed. A crash mid-migration + * leaves the prefix applied with no ledger entry, so the next boot replays + * those statements and fails loudly on "already exists". Recovery: drop the + * partially-applied objects, or insert the migration's hash into + * `drizzle.__drizzle_migrations` manually. The thrown error identifies the + * statement and migration that failed. + */ +export async function applyMigrationsByHash( + deps: HashLedgerDeps, + plan: MigrationPlanEntry[] = loadPlan(), +): Promise<{ applied: number; skipped: number }> { + await deps.ensureLedger(); + const alreadyApplied = new Set(await deps.appliedHashes()); + + let applied = 0; + let skipped = 0; + for (const migration of plan) { + if (alreadyApplied.has(migration.hash)) { + skipped += 1; + continue; + } + for (const [stmtIdx, stmt] of migration.statements.entries()) { + try { + await deps.runStatement(stmt); + } catch (err) { + const cause = err instanceof Error ? err.message : String(err); + throw new Error( + `migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` + + `Statement: ${stmt.slice(0, 200)}${stmt.length > 200 ? '…' : ''}`, + { cause: err }, + ); + } + } + await deps.recordApplied(migration.hash, migration.folderMillis); + applied += 1; + } + return { applied, skipped }; +} + +const LEDGER_DDL = [ + 'CREATE SCHEMA IF NOT EXISTS drizzle', + `CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations ( + id SERIAL PRIMARY KEY, + hash text NOT NULL, + created_at bigint + )`, +]; + +function connectionString(url?: string): string { + return url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL; +} + +/** + * Apply Drizzle migrations against a postgres database, hash-ledger style. + * Idempotent: re-running against a fully-migrated database applies nothing. + */ export async function runMigrations(url?: string): Promise { - const connectionString = url ?? process.env['DATABASE_URL'] ?? DEFAULT_DATABASE_URL; - const sqlClient = postgres(connectionString, { max: 1 }); - const db = drizzlePostgres(sqlClient); + const sqlClient = postgres(connectionString(url), { max: 1 }); try { - // TODO: postgres-tier first-install also fails because (a) Drizzle wraps every - // migration in one transaction (breaks 0009's ALTER TYPE ADD VALUE → SET DEFAULT - // sequence) and (b) drizzle/meta/_journal.json has 0009 ordered before 0008, - // which the postgres-js migrator skips by `created_at < folderMillis`. The - // PGlite path below sidesteps both. A follow-up should either share the - // per-statement loop (see runPgliteMigrations) or fix the journal ordering. - await migratePostgres(db, { migrationsFolder: migrationsFolder() }); + await applyMigrationsByHash({ + ensureLedger: async () => { + for (const ddl of LEDGER_DDL) await sqlClient.unsafe(ddl); + }, + appliedHashes: async () => { + const rows = (await sqlClient.unsafe( + 'SELECT hash FROM drizzle.__drizzle_migrations', + )) as Array<{ hash: string }>; + return rows.map((r) => String(r.hash)); + }, + recordApplied: async (hash, folderMillis) => { + await sqlClient.unsafe( + 'INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ($1, $2)', + [hash, folderMillis], + ); + }, + runStatement: async (stmt) => { + await sqlClient.unsafe(stmt); + }, + }); } finally { await sqlClient.end(); } } -// Apply Drizzle migrations against an embedded PGlite database. -// -// We don't reuse drizzle's pglite migrator because it wraps ALL migrations in -// one outer transaction, which breaks Postgres' `check_safe_enum_use` rule — -// e.g. migration 0009 does `ALTER TYPE ADD VALUE 'pending'` then references -// `'pending'` as a default in the same tx. PGlite's `exec()` runs each -// statement under the Simple Query protocol, autocommitting between them. -// -// We still write to the standard `drizzle.__drizzle_migrations` ledger so the -// result is interoperable with `runMigrations()` on a postgres-backed deploy -// (modulo the journal-ordering bug noted above). -// -// We skip-by-hash rather than skip-by-folderMillis (which is what Drizzle's -// postgres-js migrator does). That's deliberate — out-of-order timestamps in -// `_journal.json` won't silently drop migrations. -// -// Failure model: each statement autocommits, and the ledger row is written -// only after all statements in a migration succeed. A crash mid-migration -// leaves the prefix applied with no ledger entry, so the next boot will -// replay those statements and fail loudly on "already exists". Recovery: -// drop the partially-applied objects, or insert the migration's hash into -// `drizzle.__drizzle_migrations` manually. The error log identifies which -// statement of which migration was the culprit. +/** + * Apply Drizzle migrations against an embedded PGlite database. + * + * We don't reuse drizzle's pglite migrator for the same reasons as the + * postgres path (single-transaction wrap; folderMillis skip). PGlite's + * `exec()` runs each statement under the Simple Query protocol, + * autocommitting between them — exactly the semantics the shared core needs. + * + * The ledger rows this writes are interoperable with the postgres path (same + * schema, same hashes), because both consume the same shipped migrations. + */ export async function runPgliteMigrations(handle: DbHandle): Promise { const client = (handle.db as unknown as { $client?: PgliteExecutor }).$client; if (!client || typeof client.exec !== 'function') { throw new Error('runPgliteMigrations: handle.db is not backed by a PGlite client'); } - await client.exec('CREATE SCHEMA IF NOT EXISTS drizzle'); - await client.exec(` - CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations ( - id SERIAL PRIMARY KEY, - hash text NOT NULL, - created_at bigint - ) - `); + await applyMigrationsByHash({ + ensureLedger: async () => { + for (const ddl of LEDGER_DDL) await client.exec(ddl); + }, + appliedHashes: async () => { + const rows = (await handle.db.execute( + sql`SELECT hash FROM drizzle.__drizzle_migrations`, + )) as unknown as ExecuteRows<{ hash: string }>; + return rows.rows.map((r) => String(r.hash)); + }, + recordApplied: async (hash, folderMillis) => { + await handle.db.execute( + sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${hash}, ${folderMillis})`, + ); + }, + runStatement: async (stmt) => { + await client.exec(stmt); + }, + }); +} - const appliedRows = (await handle.db.execute( - sql`SELECT hash FROM drizzle.__drizzle_migrations`, - )) as unknown as ExecuteRows<{ hash: string }>; - const applied = new Set(appliedRows.rows.map((r) => r.hash)); +/* ------------------------------------------------------------------ */ +/* Migration status (#1392: the installer must VERIFY, not assume) */ +/* ------------------------------------------------------------------ */ - const migrations = readMigrationFiles({ migrationsFolder: migrationsFolder() }); - for (const migration of migrations) { - if (applied.has(migration.hash)) continue; +/** Read the journal tags (migration folder names) in journal order. */ +export function readJournalTags(folder: string = migrationsFolder()): string[] { + const journal = JSON.parse(readFileSync(resolve(folder, 'meta', '_journal.json'), 'utf8')) as { + entries?: Array<{ tag?: string }>; + }; + return (journal.entries ?? []).map((e) => e.tag ?? '').filter((t) => t.length > 0); +} - // Run each statement-breakpoint chunk in its own exec() call so PGlite - // commits between statements — this is what lets `ALTER TYPE ADD VALUE` - // become visible before a subsequent statement references the new value. - for (const [stmtIdx, stmt] of migration.sql.entries()) { - const trimmed = stmt.trim(); - if (!trimmed) continue; - try { - await client.exec(trimmed); - } catch (err) { - const cause = err instanceof Error ? err.message : String(err); - throw new Error( - `runPgliteMigrations: migration hash=${migration.hash} statement #${stmtIdx} failed: ${cause}\n` + - `Statement: ${trimmed.slice(0, 200)}${trimmed.length > 200 ? '…' : ''}`, - { cause: err }, - ); - } +export interface MigrationStatus { + /** Hashes recorded in the database's ledger (0 if no ledger exists). */ + appliedCount: number; + /** Migrations shipped in this package's drizzle/ folder. */ + expectedCount: number; + /** The tag (folder name) of the last journal entry, for error messages. */ + expectedLastTag: string; + /** True iff every shipped migration's hash is in the ledger. */ + complete: boolean; +} + +/** + * Report whether a postgres database carries the full shipped schema. + * + * Read-only apart from `ensureLedger` semantics: it never creates the ledger + * (unlike the migrators), so a database with NO ledger reports + * appliedCount=0 / complete=false — the exact #1392/#1389 signature (an + * install whose dependency set shipped no migrations at all). + */ +export async function getMigrationStatus(url?: string): Promise { + const sqlClient = postgres(connectionString(url), { max: 1 }); + try { + const plan = loadPlan(); + const tags = readJournalTags(); + const expectedHashes = new Set(plan.map((m) => m.hash)); + + let appliedHashes: string[] = []; + const regRows = (await sqlClient.unsafe( + "SELECT to_regclass('drizzle.__drizzle_migrations') AS reg", + )) as Array<{ reg: string | null }>; + if (regRows[0]?.reg) { + const rows = (await sqlClient.unsafe( + 'SELECT hash FROM drizzle.__drizzle_migrations', + )) as Array<{ hash: string }>; + appliedHashes = rows.map((r) => String(r.hash)); } - await handle.db.execute( - sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES (${migration.hash}, ${migration.folderMillis})`, - ); + + const appliedSet = new Set(appliedHashes); + return { + appliedCount: appliedHashes.length, + expectedCount: plan.length, + expectedLastTag: tags[tags.length - 1] ?? '', + complete: + plan.length > 0 && + plan.every((m) => appliedSet.has(m.hash)) && + // A ledger with entries OUTSIDE the shipped plan means the database + // came from a different (e.g. newer) build — not "complete" either. + appliedHashes.every((h) => expectedHashes.has(h)), + }; + } finally { + await sqlClient.end(); } } diff --git a/packages/mosaic/src/commands/gateway/install.ts b/packages/mosaic/src/commands/gateway/install.ts index 84b07c04..186f7b2e 100644 --- a/packages/mosaic/src/commands/gateway/install.ts +++ b/packages/mosaic/src/commands/gateway/install.ts @@ -15,6 +15,7 @@ import { homedir } from 'node:os'; import { join } from 'node:path'; import { ClackPrompter } from '../../prompter/clack-prompter.js'; +import type { VerifyResult } from './verify.js'; import type { WizardState } from '../../types.js'; interface InstallOpts { @@ -89,13 +90,23 @@ export async function runInstall(opts: InstallOpts): Promise { prompter.log(` Logs: mosaic gateway logs`); prompter.log(` Status: mosaic gateway status`); - // Post-install verification (CU-07-03) — non-fatal. + // Post-install verification (CU-07-03). Health/token/bootstrap failures + // stay non-fatal (courtesy checks), but a FAILED database schema check is + // fatal (#1392): an install that reports success over an empty/partial + // database is the exact T63 failure this command must never reproduce. + let verifyResult: VerifyResult | undefined; try { const { runPostInstallVerification } = await import('./verify.js'); - await runPostInstallVerification(configResult.host, configResult.port); + verifyResult = await runPostInstallVerification(configResult.host, configResult.port); } catch { // Non-fatal — verification is a courtesy } + if (verifyResult && verifyResult.schemaMigrated === false) { + prompter.warn( + 'Gateway install ABORTED: database schema verification failed (remediation above).', + ); + process.exit(1); + } } catch (err) { // Stages normally return structured results for expected failures. // Anything that reaches here is an unexpected runtime error — render a diff --git a/packages/mosaic/src/commands/gateway/schema-check.spec.ts b/packages/mosaic/src/commands/gateway/schema-check.spec.ts new file mode 100644 index 00000000..199317b9 --- /dev/null +++ b/packages/mosaic/src/commands/gateway/schema-check.spec.ts @@ -0,0 +1,156 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + checkDatabaseSchema, + SCHEMA_FAIL_REMEDIATION, + type SchemaCheckDeps, +} from './schema-check.js'; + +/* ------------------------------------------------------------------ */ +/* Fixture config */ +/* ------------------------------------------------------------------ */ + +const tmpDirs: string[] = []; + +function writeConfig(cfg: Record): string { + const dir = mkdtempSync(join(tmpdir(), 'schema-check-')); + tmpDirs.push(dir); + const path = join(dir, 'mosaic.config.json'); + writeFileSync(path, JSON.stringify(cfg)); + return path; +} + +const STANDALONE_CFG = { + tier: 'standalone', + storage: { type: 'postgres', url: 'postgresql://u:p@localhost:5434/x' }, + queue: { type: 'bullmq', url: 'redis://localhost:6380' }, + memory: { type: 'keyword' }, +}; + +const LOCAL_CFG = { + tier: 'local', + storage: { type: 'pglite', dataDir: '.mosaic/storage-pglite' }, + queue: { type: 'local', dataDir: '.mosaic/queue' }, + memory: { type: 'keyword' }, +}; + +function deps(overrides: Partial = {}): SchemaCheckDeps { + return { + runMigrations: vi.fn().mockResolvedValue(undefined), + getMigrationStatus: vi.fn().mockResolvedValue({ + appliedCount: 17, + expectedCount: 17, + expectedLastTag: '0016_x', + complete: true, + }), + ...overrides, + }; +} + +/* ------------------------------------------------------------------ */ +/* Tests */ +/* ------------------------------------------------------------------ */ + +describe('checkDatabaseSchema', () => { + beforeEach(() => { + process.env['MOSAIC_CONFIG'] = ''; + vi.stubEnv('DATABASE_URL', 'postgresql://env:env@localhost:9999/env'); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true }); + }); + + it('passes when the ledger matches the shipped journal', async () => { + const cfg = writeConfig(STANDALONE_CFG); + const d = deps(); + const result = await checkDatabaseSchema(d, cfg); + expect(result.status).toBe('ok'); + expect(result.detail).toContain('17/17'); + expect(d.runMigrations).toHaveBeenCalledWith(STANDALONE_CFG.storage.url); + expect(d.getMigrationStatus).toHaveBeenCalledWith(STANDALONE_CFG.storage.url); + }); + + it('FAILS when the ledger is incomplete — the #1389 empty-database signature', async () => { + const cfg = writeConfig(STANDALONE_CFG); + const d = deps({ + getMigrationStatus: vi.fn().mockResolvedValue({ + appliedCount: 0, + expectedCount: 17, + expectedLastTag: '0016_x', + complete: false, + }), + }); + const result = await checkDatabaseSchema(d, cfg); + expect(result.status).toBe('fail'); + if (result.status !== 'fail') throw new Error('expected fail'); + expect(result.detail).toContain('0/17'); + expect(result.remediation).toBe(SCHEMA_FAIL_REMEDIATION); + expect(result.remediation).toContain('#1389'); + }); + + it('FAILS when the ledger is only PARTIALLY migrated (#1402 upgrade case)', async () => { + const cfg = writeConfig(STANDALONE_CFG); + const d = deps({ + getMigrationStatus: vi.fn().mockResolvedValue({ + appliedCount: 9, + expectedCount: 17, + expectedLastTag: '0016_x', + complete: false, + }), + }); + const result = await checkDatabaseSchema(d, cfg); + expect(result.status).toBe('fail'); + expect(result.detail).toContain('9/17'); + }); + + it('FAILS (never crashes) when the migration run itself throws', async () => { + const cfg = writeConfig(STANDALONE_CFG); + const d = deps({ + runMigrations: vi.fn().mockRejectedValue(new Error('connection refused')), + }); + const result = await checkDatabaseSchema(d, cfg); + expect(result.status).toBe('fail'); + expect(result.detail).toContain('connection refused'); + }); + + it('skips the local tier (gateway migrates its own PGlite at startup)', async () => { + const cfg = writeConfig(LOCAL_CFG); + const d = deps(); + const result = await checkDatabaseSchema(d, cfg); + expect(result.status).toBe('skipped'); + expect(d.runMigrations).not.toHaveBeenCalled(); + }); +}); + +/* ------------------------------------------------------------------ */ +/* Config resolution priority */ +/* ------------------------------------------------------------------ */ + +describe('resolveSchemaCheckConfigPath', () => { + it('prefers the daemon-written gateway config over cwd copies', async () => { + const { resolveSchemaCheckConfigPath } = await import('./schema-check.js'); + const cwdCfg = writeConfig(STANDALONE_CFG); // in a temp dir + const daemonDir = mkdtempSync(join(tmpdir(), 'schema-check-daemon-')); + tmpDirs.push(daemonDir); + const daemonHome = join(daemonDir, '.config', 'mosaic', 'gateway'); + mkdirSync(daemonHome, { recursive: true }); + writeFileSync(join(daemonHome, 'mosaic.config.json'), JSON.stringify(LOCAL_CFG)); + + const prevHome = process.env['HOME']; + vi.stubEnv('HOME', daemonDir); + vi.stubEnv('MOSAIC_CONFIG', ''); + try { + const resolved = resolveSchemaCheckConfigPath(); + // Must NOT pick the cwd copy (cwd is the vitest project dir, not our + // temp dir — so the only resolvable candidates are daemon + $HOME/.mosaic). + expect(resolved).toBe(join(daemonHome, 'mosaic.config.json')); + void cwdCfg; + } finally { + if (prevHome !== undefined) vi.stubEnv('HOME', prevHome); + } + }); +}); diff --git a/packages/mosaic/src/commands/gateway/schema-check.ts b/packages/mosaic/src/commands/gateway/schema-check.ts new file mode 100644 index 00000000..b4ab810a --- /dev/null +++ b/packages/mosaic/src/commands/gateway/schema-check.ts @@ -0,0 +1,107 @@ +/** + * Install-time database schema verification (#1392). + * + * Fresh standalone (postgres) installs once ended "healthy" with a completely + * empty database: the resolved dependency set shipped no migrations at all + * (#1389), the gateway started fine, and nothing failed until the first real + * query. The issue's own conclusion: the installer must "instruct and verify". + * + * This check runs AFTER migrations have been (re-)applied, so the only way it + * fails is a genuinely broken migration set or database — which is exactly + * when install must not report success. Failure is fatal by design (fast-fail + * STANDARDS); callers print the remediation text and exit non-zero. + */ + +import { existsSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { loadConfig } from '@mosaicstack/config'; + +export interface SchemaStatusCounts { + appliedCount: number; + expectedCount: number; + expectedLastTag: string; + complete: boolean; +} + +/** Injectable migration surface — keeps this unit-testable without a DB. */ +export interface SchemaCheckDeps { + runMigrations(url: string): Promise; + getMigrationStatus(url: string): Promise; +} + +export type SchemaCheckResult = + | { status: 'ok'; detail: string } + | { status: 'skipped'; detail: string } + | { status: 'fail'; detail: string; remediation: string }; + +export const SCHEMA_FAIL_REMEDIATION = [ + 'The gateway database does not carry the full schema.', + 'Causes seen in the wild: dependency set resolved without migrations (#1389), or a partially-migrated database (#1402).', + 'Remediation:', + ' 1. Re-run: mosaic gateway install (applies migrations and verifies again)', + ' 2. Check the resolved @mosaicstack/db version is the same pipeline as the gateway (npm ls -g @mosaicstack/db)', + ' 3. Manual apply: run runMigrations() from @mosaicstack/db against the storage URL, then re-verify', +].join('\n'); + +/** + * Resolve the config the INSTALLED gateway would use — same priority the + * daemon applies (apps/gateway/src/env.ts resolveGatewayConfigPath), minus + * the source-tree anchors that do not exist on an installed host. Verifying + * against any other config could green-light a database the daemon never + * reads (#1392: verify what runs, not what happens to lie in cwd). + */ +export function resolveSchemaCheckConfigPath(explicit?: string): string | undefined { + if (explicit) return resolve(explicit); + const candidates = [ + process.env['MOSAIC_CONFIG'], + join(homedir(), '.config', 'mosaic', 'gateway', 'mosaic.config.json'), // daemon-written + resolve(process.cwd(), 'mosaic.config.json'), + join(homedir(), '.mosaic', 'mosaic.config.json'), + ]; + for (const c of candidates) { + if (c && existsSync(c)) return c; + } + return undefined; // loadConfig falls back to env-var detection +} + +export async function checkDatabaseSchema( + deps: SchemaCheckDeps, + configPath?: string, +): Promise { + const config = loadConfig(resolveSchemaCheckConfigPath(configPath)); + + // Local tier: the gateway itself runs PGlite migrations at startup (see + // DatabaseModule.onModuleInit), and a broken local tier fails the health + // check instead. Nothing for the installer to verify here. + if (config.storage.type !== 'postgres') { + return { + status: 'skipped', + detail: 'database schema (local tier — migrated by gateway at startup)', + }; + } + + const url = config.storage.url; + try { + await deps.runMigrations(url); + const status = await deps.getMigrationStatus(url); + if (status.complete) { + return { + status: 'ok', + detail: `database schema (${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations)`, + }; + } + return { + status: 'fail', + detail: `database schema incomplete (${status.appliedCount.toString()}/${status.expectedCount.toString()} applied, last expected: ${status.expectedLastTag})`, + remediation: SCHEMA_FAIL_REMEDIATION, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { + status: 'fail', + detail: `database schema check errored: ${msg}`, + remediation: SCHEMA_FAIL_REMEDIATION, + }; + } +} diff --git a/packages/mosaic/src/commands/gateway/verify.ts b/packages/mosaic/src/commands/gateway/verify.ts index e001bb22..5ac83dc9 100644 --- a/packages/mosaic/src/commands/gateway/verify.ts +++ b/packages/mosaic/src/commands/gateway/verify.ts @@ -44,6 +44,8 @@ export interface VerifyResult { gatewayHealthy: boolean; adminTokenOnFile: boolean; bootstrapReachable: boolean; + /** False only on a FAILED postgres schema check; true when ok or skipped. */ + schemaMigrated: boolean; allPassed: boolean; } @@ -89,7 +91,36 @@ export async function runPostInstallVerification( fail('bootstrap endpoint reach', 'Run: mosaic gateway status / mosaic gateway logs'); } - const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable; + // ─── Check 4: Database schema migrated (#1392) ──────────────────────────── + // Fatal-on-failure for install: the #1389 failure mode was an install that + // reported success over an empty database. Local tiers skip (the gateway + // migrates its own PGlite at startup and would fail health if it couldn't). + let schemaMigrated = true; + try { + const { checkDatabaseSchema } = await import('./schema-check.js'); + const { runMigrations, getMigrationStatus } = await import('@mosaicstack/db'); + const result = await checkDatabaseSchema( + { runMigrations, getMigrationStatus }, + process.env['MOSAIC_CONFIG'], + ); + if (result.status === 'ok') { + ok(result.detail); + } else if (result.status === 'skipped') { + ok(result.detail); + } else { + fail(result.detail, result.remediation); + schemaMigrated = false; + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + fail( + `database schema check errored: ${msg}`, + 'See #1392/#1389; re-run: mosaic gateway install', + ); + schemaMigrated = false; + } + + const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable && schemaMigrated; if (!allPassed) { console.log( @@ -98,7 +129,7 @@ export async function runPostInstallVerification( ); } - return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, allPassed }; + return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, schemaMigrated, allPassed }; } /**