feat(gateway,storage): mosaic gateway doctor with tier health JSON (FED-M1-06) (#475)

packages/storage/src/tier-detection.spec.ts

@@ -0,0 +1,546 @@
+/**
+ * Unit tests for tier-detection.ts.
+ *
+ * All external I/O (postgres, ioredis) is mocked — no live services required.
+ *
+ * Note on hoisting: vi.mock() factories are hoisted above all imports by vitest.
+ * Variables referenced inside factory callbacks must be declared via vi.hoisted()
+ * so they are available at hoist time.
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+/* ------------------------------------------------------------------ */
+/*  Hoist shared mock state so factories can reference it              */
+/* ------------------------------------------------------------------ */
+
+const mocks = vi.hoisted(() => {
+  const mockSqlFn = vi.fn();
+  const mockEnd = vi.fn().mockResolvedValue(undefined);
+  const mockPostgresConstructor = vi.fn(() => {
+    const sql = mockSqlFn as ReturnType<typeof mockSqlFn>;
+    (sql as unknown as Record<string, unknown>)['end'] = mockEnd;
+    return sql;
+  });
+
+  const mockRedisConnect = vi.fn().mockResolvedValue(undefined);
+  const mockRedisPing = vi.fn().mockResolvedValue('PONG');
+  const mockRedisDisconnect = vi.fn();
+  const MockRedis = vi.fn().mockImplementation(() => ({
+    connect: mockRedisConnect,
+    ping: mockRedisPing,
+    disconnect: mockRedisDisconnect,
+  }));
+
+  return {
+    mockSqlFn,
+    mockEnd,
+    mockPostgresConstructor,
+    mockRedisConnect,
+    mockRedisPing,
+    mockRedisDisconnect,
+    MockRedis,
+  };
+});
+
+/* ------------------------------------------------------------------ */
+/*  Module mocks (registered at hoist time)                            */
+/* ------------------------------------------------------------------ */
+
+vi.mock('postgres', () => ({
+  default: mocks.mockPostgresConstructor,
+}));
+
+vi.mock('ioredis', () => ({
+  Redis: mocks.MockRedis,
+}));
+
+/* ------------------------------------------------------------------ */
+/*  Import SUT after mocks are registered                              */
+/* ------------------------------------------------------------------ */
+
+import { detectAndAssertTier, probeServiceHealth, TierDetectionError } from './tier-detection.js';
+import type { TierConfig } from './tier-detection.js';
+
+/* ------------------------------------------------------------------ */
+/*  Config fixtures                                                     */
+/* ------------------------------------------------------------------ */
+
+const LOCAL_CONFIG: TierConfig = {
+  tier: 'local',
+  storage: { type: 'pglite', dataDir: '.mosaic/pglite' },
+  queue: { type: 'local' },
+};
+
+const STANDALONE_CONFIG: TierConfig = {
+  tier: 'standalone',
+  storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@db-host:5432/mosaic' },
+  queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
+};
+
+const FEDERATED_CONFIG: TierConfig = {
+  tier: 'federated',
+  storage: {
+    type: 'postgres',
+    url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
+    enableVector: true,
+  },
+  queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
+};
+
+/* ------------------------------------------------------------------ */
+/*  Tests                                                               */
+/* ------------------------------------------------------------------ */
+
+describe('detectAndAssertTier', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    // Default: all probes succeed.
+    mocks.mockSqlFn.mockResolvedValue([]);
+    mocks.mockEnd.mockResolvedValue(undefined);
+    mocks.mockRedisConnect.mockResolvedValue(undefined);
+    mocks.mockRedisPing.mockResolvedValue('PONG');
+
+    // Re-wire constructor to return a fresh sql-like object each time.
+    mocks.mockPostgresConstructor.mockImplementation(() => {
+      const sql = mocks.mockSqlFn as ReturnType<typeof mocks.mockSqlFn>;
+      (sql as unknown as Record<string, unknown>)['end'] = mocks.mockEnd;
+      return sql;
+    });
+    mocks.MockRedis.mockImplementation(() => ({
+      connect: mocks.mockRedisConnect,
+      ping: mocks.mockRedisPing,
+      disconnect: mocks.mockRedisDisconnect,
+    }));
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  1. local — no-op                                                 */
+  /* ---------------------------------------------------------------- */
+
+  it('resolves immediately for tier=local without touching postgres or ioredis', async () => {
+    await expect(detectAndAssertTier(LOCAL_CONFIG)).resolves.toBeUndefined();
+    expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
+    expect(mocks.MockRedis).not.toHaveBeenCalled();
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  2. standalone — happy path                                       */
+  /* ---------------------------------------------------------------- */
+
+  it('resolves for tier=standalone when postgres and valkey are reachable', async () => {
+    await expect(detectAndAssertTier(STANDALONE_CONFIG)).resolves.toBeUndefined();
+
+    // Postgres was probed (SELECT 1 only — no pgvector check).
+    expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(1);
+    expect(mocks.mockSqlFn).toHaveBeenCalledTimes(1);
+    // Valkey was probed.
+    expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
+    expect(mocks.mockRedisPing).toHaveBeenCalledTimes(1);
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  3. standalone — postgres unreachable                             */
+  /* ---------------------------------------------------------------- */
+
+  it('throws TierDetectionError with service=postgres when postgres query rejects', async () => {
+    mocks.mockSqlFn.mockRejectedValueOnce(new Error('connection refused'));
+
+    const promise = detectAndAssertTier(STANDALONE_CONFIG);
+    await expect(promise).rejects.toBeInstanceOf(TierDetectionError);
+
+    // Confirm no valkey probe happened (fail fast on first error).
+    expect(mocks.MockRedis).not.toHaveBeenCalled();
+  });
+
+  it('sets service=postgres on the error when postgres fails', async () => {
+    mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
+
+    try {
+      await detectAndAssertTier(STANDALONE_CONFIG);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('postgres');
+      expect(typed.remediation).toContain('docker compose');
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  4. standalone — valkey unreachable                               */
+  /* ---------------------------------------------------------------- */
+
+  it('throws TierDetectionError with service=valkey when ping fails', async () => {
+    // Postgres probe succeeds; valkey connect fails.
+    mocks.mockSqlFn.mockResolvedValue([]);
+    mocks.mockRedisConnect.mockRejectedValue(new Error('ECONNREFUSED'));
+
+    try {
+      await detectAndAssertTier(STANDALONE_CONFIG);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('valkey');
+      expect(typed.message).toContain('valkey');
+      expect(typed.remediation).toContain('valkey-federated');
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  5. federated — happy path                                        */
+  /* ---------------------------------------------------------------- */
+
+  it('resolves for tier=federated when all three checks pass', async () => {
+    // SELECT 1 and CREATE EXTENSION both succeed.
+    mocks.mockSqlFn.mockResolvedValue([]);
+
+    await expect(detectAndAssertTier(FEDERATED_CONFIG)).resolves.toBeUndefined();
+
+    // postgres probe (SELECT 1) + pgvector probe (CREATE EXTENSION) = 2 postgres constructors.
+    expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(2);
+    expect(mocks.mockSqlFn).toHaveBeenCalledTimes(2);
+    // Valkey probed once.
+    expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  6. federated — pgvector not installable                          */
+  /* ---------------------------------------------------------------- */
+
+  it('throws TierDetectionError with service=pgvector when CREATE EXTENSION fails', async () => {
+    // SELECT 1 succeeds (first call), CREATE EXTENSION fails (second call).
+    mocks.mockSqlFn
+      .mockResolvedValueOnce([]) // SELECT 1
+      .mockRejectedValueOnce(new Error('extension "vector" is not available'));
+
+    try {
+      await detectAndAssertTier(FEDERATED_CONFIG);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('pgvector');
+      expect(typed.message).toContain('pgvector');
+      expect(typed.remediation).toContain('pgvector/pgvector');
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  7. probeValkey honors connectTimeout and lazyConnect            */
+  /* ---------------------------------------------------------------- */
+
+  it('constructs the ioredis Redis client with connectTimeout: 5000', async () => {
+    await detectAndAssertTier(STANDALONE_CONFIG);
+
+    expect(mocks.MockRedis).toHaveBeenCalledOnce();
+    expect(mocks.MockRedis).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.objectContaining({ connectTimeout: 5000, lazyConnect: true }),
+    );
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  8. probePgvector — library-not-installed remediation             */
+  /* ---------------------------------------------------------------- */
+
+  it('includes pgvector/pgvector:pg17 in remediation when pgvector library is missing', async () => {
+    // SELECT 1 succeeds; CREATE EXTENSION fails with the canonical library-missing message.
+    mocks.mockSqlFn
+      .mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
+      .mockRejectedValueOnce(new Error('extension "vector" is not available')); // probePgvector
+
+    try {
+      await detectAndAssertTier(FEDERATED_CONFIG);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('pgvector');
+      expect(typed.remediation).toContain('pgvector/pgvector:pg17');
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  9. probePgvector — permission / other error remediation          */
+  /* ---------------------------------------------------------------- */
+
+  it('mentions CREATE permission or superuser in remediation for a generic pgvector error', async () => {
+    // SELECT 1 succeeds; CREATE EXTENSION fails with a permission error.
+    mocks.mockSqlFn
+      .mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
+      .mockRejectedValueOnce(new Error('permission denied to create extension'));
+
+    try {
+      await detectAndAssertTier(FEDERATED_CONFIG);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('pgvector');
+      // Must NOT point to the image fix — that's only for the library-missing case.
+      expect(typed.remediation).not.toContain('pgvector/pgvector:pg17');
+      // Must mention permissions or superuser.
+      expect(typed.remediation).toMatch(/CREATE|superuser/i);
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  10. federated tier rejects non-bullmq queue.type                 */
+  /* ---------------------------------------------------------------- */
+
+  it('throws TierDetectionError with service=config for federated tier with queue.type !== bullmq', async () => {
+    const badConfig: TierConfig = {
+      tier: 'federated',
+      storage: {
+        type: 'postgres',
+        url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
+        enableVector: true,
+      },
+      queue: { type: 'local' },
+    };
+
+    try {
+      await detectAndAssertTier(badConfig);
+      expect.fail('should have thrown');
+    } catch (err) {
+      expect(err).toBeInstanceOf(TierDetectionError);
+      const typed = err as TierDetectionError;
+      expect(typed.service).toBe('config');
+      expect(typed.remediation).toContain('bullmq');
+    }
+
+    // No network probes should have been attempted.
+    expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
+    expect(mocks.MockRedis).not.toHaveBeenCalled();
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  11. Error fields populated                                        */
+  /* ---------------------------------------------------------------- */
+
+  it('populates host, port, and remediation on a thrown TierDetectionError', async () => {
+    mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
+
+    let caught: TierDetectionError | undefined;
+    try {
+      await detectAndAssertTier(STANDALONE_CONFIG);
+    } catch (err) {
+      caught = err as TierDetectionError;
+    }
+
+    expect(caught).toBeInstanceOf(TierDetectionError);
+    expect(caught!.service).toBe('postgres');
+    // Host and port are extracted from the Postgres URL in STANDALONE_CONFIG.
+    expect(caught!.host).toBe('db-host');
+    expect(caught!.port).toBe(5432);
+    expect(caught!.remediation).toMatch(/docker compose/i);
+    expect(caught!.message).toContain('db-host:5432');
+  });
+});
+
+/* ------------------------------------------------------------------ */
+/*  probeServiceHealth tests                                           */
+/* ------------------------------------------------------------------ */
+
+describe('probeServiceHealth', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    mocks.mockSqlFn.mockResolvedValue([]);
+    mocks.mockEnd.mockResolvedValue(undefined);
+    mocks.mockRedisConnect.mockResolvedValue(undefined);
+    mocks.mockRedisPing.mockResolvedValue('PONG');
+
+    mocks.mockPostgresConstructor.mockImplementation(() => {
+      const sql = mocks.mockSqlFn as ReturnType<typeof mocks.mockSqlFn>;
+      (sql as unknown as Record<string, unknown>)['end'] = mocks.mockEnd;
+      return sql;
+    });
+    mocks.MockRedis.mockImplementation(() => ({
+      connect: mocks.mockRedisConnect,
+      ping: mocks.mockRedisPing,
+      disconnect: mocks.mockRedisDisconnect,
+    }));
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  12. local tier — all skipped, green                              */
+  /* ---------------------------------------------------------------- */
+
+  it('returns all services as skipped and overall green for local tier', async () => {
+    const report = await probeServiceHealth(LOCAL_CONFIG);
+
+    expect(report.tier).toBe('local');
+    expect(report.overall).toBe('green');
+    expect(report.services).toHaveLength(3);
+    for (const svc of report.services) {
+      expect(svc.status).toBe('skipped');
+    }
+    expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
+    expect(mocks.MockRedis).not.toHaveBeenCalled();
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  13. postgres fails, valkey ok → red                              */
+  /* ---------------------------------------------------------------- */
+
+  it('returns red overall with postgres fail and valkey ok for standalone when postgres fails', async () => {
+    mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
+
+    const report = await probeServiceHealth(STANDALONE_CONFIG);
+
+    expect(report.overall).toBe('red');
+
+    const pgCheck = report.services.find((s) => s.name === 'postgres');
+    expect(pgCheck?.status).toBe('fail');
+    expect(pgCheck?.error).toBeDefined();
+    expect(pgCheck?.error?.remediation).toContain('docker compose');
+
+    const vkCheck = report.services.find((s) => s.name === 'valkey');
+    expect(vkCheck?.status).toBe('ok');
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  14. federated all green → 3 services ok                         */
+  /* ---------------------------------------------------------------- */
+
+  it('returns green overall with all 3 services ok for federated when all pass', async () => {
+    mocks.mockSqlFn.mockResolvedValue([]);
+
+    const report = await probeServiceHealth(FEDERATED_CONFIG);
+
+    expect(report.tier).toBe('federated');
+    expect(report.overall).toBe('green');
+    expect(report.services).toHaveLength(3);
+
+    for (const svc of report.services) {
+      expect(svc.status).toBe('ok');
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  15. durationMs is a non-negative number for every service check  */
+  /* ---------------------------------------------------------------- */
+
+  it('sets durationMs as a non-negative number for every service check', async () => {
+    mocks.mockSqlFn.mockResolvedValue([]);
+
+    const report = await probeServiceHealth(FEDERATED_CONFIG);
+
+    for (const svc of report.services) {
+      expect(typeof svc.durationMs).toBe('number');
+      expect(svc.durationMs).toBeGreaterThanOrEqual(0);
+    }
+  });
+
+  it('sets durationMs >= 0 even for skipped services (local tier)', async () => {
+    const report = await probeServiceHealth(LOCAL_CONFIG);
+
+    for (const svc of report.services) {
+      expect(typeof svc.durationMs).toBe('number');
+      expect(svc.durationMs).toBeGreaterThanOrEqual(0);
+    }
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  16. configPath is passed through to the report                   */
+  /* ---------------------------------------------------------------- */
+
+  it('includes configPath in the report when provided', async () => {
+    const report = await probeServiceHealth(LOCAL_CONFIG, '/etc/mosaic/mosaic.config.json');
+    expect(report.configPath).toBe('/etc/mosaic/mosaic.config.json');
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  17. standalone — valkey fails, postgres ok → red                 */
+  /* ---------------------------------------------------------------- */
+
+  it('returns red with valkey fail and postgres ok for standalone when valkey fails', async () => {
+    mocks.mockSqlFn.mockResolvedValue([]);
+    mocks.mockRedisConnect.mockRejectedValue(new Error('ECONNREFUSED'));
+
+    const report = await probeServiceHealth(STANDALONE_CONFIG);
+
+    expect(report.overall).toBe('red');
+
+    const pgCheck = report.services.find((s) => s.name === 'postgres');
+    expect(pgCheck?.status).toBe('ok');
+
+    const vkCheck = report.services.find((s) => s.name === 'valkey');
+    expect(vkCheck?.status).toBe('fail');
+    expect(vkCheck?.error).toBeDefined();
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  18. federated — pgvector fails → red with remediation           */
+  /* ---------------------------------------------------------------- */
+
+  it('returns red with pgvector fail for federated when pgvector probe fails', async () => {
+    mocks.mockSqlFn
+      .mockResolvedValueOnce([]) // postgres SELECT 1
+      .mockRejectedValueOnce(new Error('extension "vector" is not available')); // pgvector
+
+    const report = await probeServiceHealth(FEDERATED_CONFIG);
+
+    expect(report.overall).toBe('red');
+
+    const pvCheck = report.services.find((s) => s.name === 'pgvector');
+    expect(pvCheck?.status).toBe('fail');
+    expect(pvCheck?.error?.remediation).toContain('pgvector/pgvector:pg17');
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  19. federated — non-bullmq queue → red config error, no network */
+  /* ---------------------------------------------------------------- */
+
+  it('returns red overall with config error when federated tier has non-bullmq queue (no network call)', async () => {
+    const federatedBadQueueConfig: TierConfig = {
+      tier: 'federated',
+      storage: {
+        type: 'postgres',
+        url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
+        enableVector: true,
+      },
+      queue: { type: 'local' },
+    };
+
+    const report = await probeServiceHealth(federatedBadQueueConfig);
+
+    expect(report.overall).toBe('red');
+
+    const valkey = report.services.find((s) => s.name === 'valkey');
+    expect(valkey?.status).toBe('fail');
+    expect(valkey?.error?.remediation).toMatch(/bullmq/i);
+
+    // Critically: no network call was made — MockRedis constructor must NOT have been called.
+    expect(mocks.MockRedis).not.toHaveBeenCalled();
+  });
+
+  /* ---------------------------------------------------------------- */
+  /*  20. durationMs actually measures real elapsed time               */
+  /* ---------------------------------------------------------------- */
+
+  it('measures real elapsed time for service probes', async () => {
+    const DELAY_MS = 25;
+
+    // Make the postgres mock introduce a real wall-clock delay.
+    mocks.mockSqlFn.mockImplementation(
+      () =>
+        new Promise((resolve) =>
+          setTimeout(() => {
+            resolve([]);
+          }, DELAY_MS),
+        ),
+    );
+
+    const report = await probeServiceHealth(STANDALONE_CONFIG);
+
+    const pgCheck = report.services.find((s) => s.name === 'postgres');
+    expect(pgCheck).toBeDefined();
+    // Must be >= 20ms (small slack for jitter). Would be 0 if timer were stubbed.
+    expect(pgCheck!.durationMs).toBeGreaterThanOrEqual(20);
+  });
+});