fix(#1392): hash-ledger migrations + install-time schema verification (closes #1392, closes #1402) (#1403)

Co-authored-by: ops-deploy-01 <[email protected]>

packages/mosaic/src/commands/gateway/install.ts

@@ -15,6 +15,7 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { ClackPrompter } from '../../prompter/clack-prompter.js';
+import type { VerifyResult } from './verify.js';
 import type { WizardState } from '../../types.js';
 
 interface InstallOpts {
@@ -89,13 +90,23 @@ export async function runInstall(opts: InstallOpts): Promise<void> {
     prompter.log(`  Logs:      mosaic gateway logs`);
     prompter.log(`  Status:    mosaic gateway status`);
 
-    // Post-install verification (CU-07-03) — non-fatal.
+    // Post-install verification (CU-07-03). Health/token/bootstrap failures
+    // stay non-fatal (courtesy checks), but a FAILED database schema check is
+    // fatal (#1392): an install that reports success over an empty/partial
+    // database is the exact T63 failure this command must never reproduce.
+    let verifyResult: VerifyResult | undefined;
     try {
       const { runPostInstallVerification } = await import('./verify.js');
-      await runPostInstallVerification(configResult.host, configResult.port);
+      verifyResult = await runPostInstallVerification(configResult.host, configResult.port);
     } catch {
       // Non-fatal — verification is a courtesy
     }
+    if (verifyResult && verifyResult.schemaMigrated === false) {
+      prompter.warn(
+        'Gateway install ABORTED: database schema verification failed (remediation above).',
+      );
+      process.exit(1);
+    }
   } catch (err) {
     // Stages normally return structured results for expected failures.
     // Anything that reaches here is an unexpected runtime error — render a

packages/mosaic/src/commands/gateway/schema-check.spec.ts

@@ -0,0 +1,156 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  checkDatabaseSchema,
+  SCHEMA_FAIL_REMEDIATION,
+  type SchemaCheckDeps,
+} from './schema-check.js';
+
+/* ------------------------------------------------------------------ */
+/*  Fixture config                                                     */
+/* ------------------------------------------------------------------ */
+
+const tmpDirs: string[] = [];
+
+function writeConfig(cfg: Record<string, unknown>): string {
+  const dir = mkdtempSync(join(tmpdir(), 'schema-check-'));
+  tmpDirs.push(dir);
+  const path = join(dir, 'mosaic.config.json');
+  writeFileSync(path, JSON.stringify(cfg));
+  return path;
+}
+
+const STANDALONE_CFG = {
+  tier: 'standalone',
+  storage: { type: 'postgres', url: 'postgresql://u:p@localhost:5434/x' },
+  queue: { type: 'bullmq', url: 'redis://localhost:6380' },
+  memory: { type: 'keyword' },
+};
+
+const LOCAL_CFG = {
+  tier: 'local',
+  storage: { type: 'pglite', dataDir: '.mosaic/storage-pglite' },
+  queue: { type: 'local', dataDir: '.mosaic/queue' },
+  memory: { type: 'keyword' },
+};
+
+function deps(overrides: Partial<SchemaCheckDeps> = {}): SchemaCheckDeps {
+  return {
+    runMigrations: vi.fn().mockResolvedValue(undefined),
+    getMigrationStatus: vi.fn().mockResolvedValue({
+      appliedCount: 17,
+      expectedCount: 17,
+      expectedLastTag: '0016_x',
+      complete: true,
+    }),
+    ...overrides,
+  };
+}
+
+/* ------------------------------------------------------------------ */
+/*  Tests                                                              */
+/* ------------------------------------------------------------------ */
+
+describe('checkDatabaseSchema', () => {
+  beforeEach(() => {
+    process.env['MOSAIC_CONFIG'] = '';
+    vi.stubEnv('DATABASE_URL', 'postgresql://env:env@localhost:9999/env');
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    for (const d of tmpDirs.splice(0)) rmSync(d, { recursive: true, force: true });
+  });
+
+  it('passes when the ledger matches the shipped journal', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps();
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('ok');
+    expect(result.detail).toContain('17/17');
+    expect(d.runMigrations).toHaveBeenCalledWith(STANDALONE_CFG.storage.url);
+    expect(d.getMigrationStatus).toHaveBeenCalledWith(STANDALONE_CFG.storage.url);
+  });
+
+  it('FAILS when the ledger is incomplete — the #1389 empty-database signature', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      getMigrationStatus: vi.fn().mockResolvedValue({
+        appliedCount: 0,
+        expectedCount: 17,
+        expectedLastTag: '0016_x',
+        complete: false,
+      }),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    if (result.status !== 'fail') throw new Error('expected fail');
+    expect(result.detail).toContain('0/17');
+    expect(result.remediation).toBe(SCHEMA_FAIL_REMEDIATION);
+    expect(result.remediation).toContain('#1389');
+  });
+
+  it('FAILS when the ledger is only PARTIALLY migrated (#1402 upgrade case)', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      getMigrationStatus: vi.fn().mockResolvedValue({
+        appliedCount: 9,
+        expectedCount: 17,
+        expectedLastTag: '0016_x',
+        complete: false,
+      }),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    expect(result.detail).toContain('9/17');
+  });
+
+  it('FAILS (never crashes) when the migration run itself throws', async () => {
+    const cfg = writeConfig(STANDALONE_CFG);
+    const d = deps({
+      runMigrations: vi.fn().mockRejectedValue(new Error('connection refused')),
+    });
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('fail');
+    expect(result.detail).toContain('connection refused');
+  });
+
+  it('skips the local tier (gateway migrates its own PGlite at startup)', async () => {
+    const cfg = writeConfig(LOCAL_CFG);
+    const d = deps();
+    const result = await checkDatabaseSchema(d, cfg);
+    expect(result.status).toBe('skipped');
+    expect(d.runMigrations).not.toHaveBeenCalled();
+  });
+});
+
+/* ------------------------------------------------------------------ */
+/*  Config resolution priority                                         */
+/* ------------------------------------------------------------------ */
+
+describe('resolveSchemaCheckConfigPath', () => {
+  it('prefers the daemon-written gateway config over cwd copies', async () => {
+    const { resolveSchemaCheckConfigPath } = await import('./schema-check.js');
+    const cwdCfg = writeConfig(STANDALONE_CFG); // in a temp dir
+    const daemonDir = mkdtempSync(join(tmpdir(), 'schema-check-daemon-'));
+    tmpDirs.push(daemonDir);
+    const daemonHome = join(daemonDir, '.config', 'mosaic', 'gateway');
+    mkdirSync(daemonHome, { recursive: true });
+    writeFileSync(join(daemonHome, 'mosaic.config.json'), JSON.stringify(LOCAL_CFG));
+
+    const prevHome = process.env['HOME'];
+    vi.stubEnv('HOME', daemonDir);
+    vi.stubEnv('MOSAIC_CONFIG', '');
+    try {
+      const resolved = resolveSchemaCheckConfigPath();
+      // Must NOT pick the cwd copy (cwd is the vitest project dir, not our
+      // temp dir — so the only resolvable candidates are daemon + $HOME/.mosaic).
+      expect(resolved).toBe(join(daemonHome, 'mosaic.config.json'));
+      void cwdCfg;
+    } finally {
+      if (prevHome !== undefined) vi.stubEnv('HOME', prevHome);
+    }
+  });
+});

packages/mosaic/src/commands/gateway/schema-check.ts

@@ -0,0 +1,107 @@
+/**
+ * Install-time database schema verification (#1392).
+ *
+ * Fresh standalone (postgres) installs once ended "healthy" with a completely
+ * empty database: the resolved dependency set shipped no migrations at all
+ * (#1389), the gateway started fine, and nothing failed until the first real
+ * query. The issue's own conclusion: the installer must "instruct and verify".
+ *
+ * This check runs AFTER migrations have been (re-)applied, so the only way it
+ * fails is a genuinely broken migration set or database — which is exactly
+ * when install must not report success. Failure is fatal by design (fast-fail
+ * STANDARDS); callers print the remediation text and exit non-zero.
+ */
+
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { loadConfig } from '@mosaicstack/config';
+
+export interface SchemaStatusCounts {
+  appliedCount: number;
+  expectedCount: number;
+  expectedLastTag: string;
+  complete: boolean;
+}
+
+/** Injectable migration surface — keeps this unit-testable without a DB. */
+export interface SchemaCheckDeps {
+  runMigrations(url: string): Promise<void>;
+  getMigrationStatus(url: string): Promise<SchemaStatusCounts>;
+}
+
+export type SchemaCheckResult =
+  | { status: 'ok'; detail: string }
+  | { status: 'skipped'; detail: string }
+  | { status: 'fail'; detail: string; remediation: string };
+
+export const SCHEMA_FAIL_REMEDIATION = [
+  'The gateway database does not carry the full schema.',
+  'Causes seen in the wild: dependency set resolved without migrations (#1389), or a partially-migrated database (#1402).',
+  'Remediation:',
+  '  1. Re-run: mosaic gateway install  (applies migrations and verifies again)',
+  '  2. Check the resolved @mosaicstack/db version is the same pipeline as the gateway (npm ls -g @mosaicstack/db)',
+  '  3. Manual apply: run runMigrations() from @mosaicstack/db against the storage URL, then re-verify',
+].join('\n');
+
+/**
+ * Resolve the config the INSTALLED gateway would use — same priority the
+ * daemon applies (apps/gateway/src/env.ts resolveGatewayConfigPath), minus
+ * the source-tree anchors that do not exist on an installed host. Verifying
+ * against any other config could green-light a database the daemon never
+ * reads (#1392: verify what runs, not what happens to lie in cwd).
+ */
+export function resolveSchemaCheckConfigPath(explicit?: string): string | undefined {
+  if (explicit) return resolve(explicit);
+  const candidates = [
+    process.env['MOSAIC_CONFIG'],
+    join(homedir(), '.config', 'mosaic', 'gateway', 'mosaic.config.json'), // daemon-written
+    resolve(process.cwd(), 'mosaic.config.json'),
+    join(homedir(), '.mosaic', 'mosaic.config.json'),
+  ];
+  for (const c of candidates) {
+    if (c && existsSync(c)) return c;
+  }
+  return undefined; // loadConfig falls back to env-var detection
+}
+
+export async function checkDatabaseSchema(
+  deps: SchemaCheckDeps,
+  configPath?: string,
+): Promise<SchemaCheckResult> {
+  const config = loadConfig(resolveSchemaCheckConfigPath(configPath));
+
+  // Local tier: the gateway itself runs PGlite migrations at startup (see
+  // DatabaseModule.onModuleInit), and a broken local tier fails the health
+  // check instead. Nothing for the installer to verify here.
+  if (config.storage.type !== 'postgres') {
+    return {
+      status: 'skipped',
+      detail: 'database schema (local tier — migrated by gateway at startup)',
+    };
+  }
+
+  const url = config.storage.url;
+  try {
+    await deps.runMigrations(url);
+    const status = await deps.getMigrationStatus(url);
+    if (status.complete) {
+      return {
+        status: 'ok',
+        detail: `database schema (${status.appliedCount.toString()}/${status.expectedCount.toString()} migrations)`,
+      };
+    }
+    return {
+      status: 'fail',
+      detail: `database schema incomplete (${status.appliedCount.toString()}/${status.expectedCount.toString()} applied, last expected: ${status.expectedLastTag})`,
+      remediation: SCHEMA_FAIL_REMEDIATION,
+    };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      status: 'fail',
+      detail: `database schema check errored: ${msg}`,
+      remediation: SCHEMA_FAIL_REMEDIATION,
+    };
+  }
+}

packages/mosaic/src/commands/gateway/verify.ts

@@ -44,6 +44,8 @@ export interface VerifyResult {
   gatewayHealthy: boolean;
   adminTokenOnFile: boolean;
   bootstrapReachable: boolean;
+  /** False only on a FAILED postgres schema check; true when ok or skipped. */
+  schemaMigrated: boolean;
   allPassed: boolean;
 }
 
@@ -89,7 +91,36 @@ export async function runPostInstallVerification(
     fail('bootstrap endpoint reach', 'Run: mosaic gateway status  /  mosaic gateway logs');
   }
 
-  const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable;
+  // ─── Check 4: Database schema migrated (#1392) ────────────────────────────
+  // Fatal-on-failure for install: the #1389 failure mode was an install that
+  // reported success over an empty database. Local tiers skip (the gateway
+  // migrates its own PGlite at startup and would fail health if it couldn't).
+  let schemaMigrated = true;
+  try {
+    const { checkDatabaseSchema } = await import('./schema-check.js');
+    const { runMigrations, getMigrationStatus } = await import('@mosaicstack/db');
+    const result = await checkDatabaseSchema(
+      { runMigrations, getMigrationStatus },
+      process.env['MOSAIC_CONFIG'],
+    );
+    if (result.status === 'ok') {
+      ok(result.detail);
+    } else if (result.status === 'skipped') {
+      ok(result.detail);
+    } else {
+      fail(result.detail, result.remediation);
+      schemaMigrated = false;
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    fail(
+      `database schema check errored: ${msg}`,
+      'See #1392/#1389; re-run: mosaic gateway install',
+    );
+    schemaMigrated = false;
+  }
+
+  const allPassed = gatewayHealthy && adminTokenOnFile && bootstrapReachable && schemaMigrated;
 
   if (!allPassed) {
     console.log(
@@ -98,7 +129,7 @@ export async function runPostInstallVerification(
     );
   }
 
-  return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, allPassed };
+  return { gatewayHealthy, adminTokenOnFile, bootstrapReachable, schemaMigrated, allPassed };
 }
 
 /**