Compare commits

..

6 Commits

Author SHA1 Message Date
Jarvis
f74fc9687f docs(federation): mark FED-M1-02 done (PR #471 merged)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/pr/ci Pipeline was successful
2026-04-19 19:02:39 -05:00
Jarvis
7383380f64 feat(gateway): tier-detector with fail-fast PG/Valkey/pgvector probes (FED-M1-04)
Implements `apps/gateway/src/bootstrap/tier-detector.ts` invoked from
`main.ts` before NestJS bootstraps. For each tier:

- `local`: no-op (PGlite is in-process)
- `standalone`: probe Postgres + Valkey
- `federated`: probe Postgres + Valkey + pgvector extension; reject
  config upfront if `queue.type !== 'bullmq'`

Each probe has a 5-second hard cap and emits a structured
`TierDetectionError` with service / host / port / remediation. The
remediation field discriminates pgvector failure modes ("library not
available" vs "permission denied") so operators get actionable hints
without leaking credentials.

Adds `postgres` and `ioredis` as direct gateway deps; previously only
transitive. 12 unit tests cover happy paths and each fail-fast branch.

Refs #460
2026-04-19 19:02:12 -05:00
58169f9979 feat(storage): pgvector adapter support gated on tier=federated (FED-M1-03) (#472)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/publish Pipeline was successful
2026-04-19 23:42:18 +00:00
51402bdb6d feat(infra): docker-compose.federated.yml overlay (FED-M1-02) (#471)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/publish Pipeline was successful
2026-04-19 23:21:31 +00:00
9c89c32684 feat(config): add federated tier + rename team→standalone (FED-M1-01) (#470)
Some checks failed
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/publish Pipeline failed
2026-04-19 23:11:11 +00:00
8aabb8c5b2 docs(mission): author MVP rollup manifest, archive install-ux-v2 (#469)
All checks were successful
ci/woodpecker/push/ci Pipeline was successful
ci/woodpecker/push/publish Pipeline was successful
2026-04-19 22:51:11 +00:00
19 changed files with 1217 additions and 46 deletions

View File

@@ -63,8 +63,10 @@
"class-validator": "^0.15.1", "class-validator": "^0.15.1",
"dotenv": "^17.3.1", "dotenv": "^17.3.1",
"fastify": "^5.0.0", "fastify": "^5.0.0",
"ioredis": "^5.10.0",
"node-cron": "^4.2.1", "node-cron": "^4.2.1",
"openai": "^6.32.0", "openai": "^6.32.0",
"postgres": "^3.4.8",
"reflect-metadata": "^0.2.0", "reflect-metadata": "^0.2.0",
"rxjs": "^7.8.0", "rxjs": "^7.8.0",
"socket.io": "^4.8.0", "socket.io": "^4.8.0",

View File

@@ -0,0 +1,344 @@
/**
* Unit tests for tier-detector.ts.
*
* All external I/O (postgres, ioredis) is mocked — no live services required.
*
* Note on hoisting: vi.mock() factories are hoisted above all imports by vitest.
* Variables referenced inside factory callbacks must be declared via vi.hoisted()
* so they are available at hoist time.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
/* ------------------------------------------------------------------ */
/* Hoist shared mock state so factories can reference it */
/* ------------------------------------------------------------------ */
const mocks = vi.hoisted(() => {
const mockSqlFn = vi.fn();
const mockEnd = vi.fn().mockResolvedValue(undefined);
const mockPostgresConstructor = vi.fn(() => {
const sql = mockSqlFn as ReturnType<typeof mockSqlFn>;
(sql as unknown as Record<string, unknown>)['end'] = mockEnd;
return sql;
});
const mockRedisConnect = vi.fn().mockResolvedValue(undefined);
const mockRedisPing = vi.fn().mockResolvedValue('PONG');
const mockRedisDisconnect = vi.fn();
const MockRedis = vi.fn().mockImplementation(() => ({
connect: mockRedisConnect,
ping: mockRedisPing,
disconnect: mockRedisDisconnect,
}));
return {
mockSqlFn,
mockEnd,
mockPostgresConstructor,
mockRedisConnect,
mockRedisPing,
mockRedisDisconnect,
MockRedis,
};
});
/* ------------------------------------------------------------------ */
/* Module mocks (registered at hoist time) */
/* ------------------------------------------------------------------ */
vi.mock('postgres', () => ({
default: mocks.mockPostgresConstructor,
}));
vi.mock('ioredis', () => ({
Redis: mocks.MockRedis,
}));
/* ------------------------------------------------------------------ */
/* Import SUT after mocks are registered */
/* ------------------------------------------------------------------ */
import { detectAndAssertTier, TierDetectionError } from './tier-detector.js';
import type { MosaicConfig } from '@mosaicstack/config';
/* ------------------------------------------------------------------ */
/* Config fixtures */
/* ------------------------------------------------------------------ */
const LOCAL_CONFIG: MosaicConfig = {
tier: 'local',
storage: { type: 'pglite', dataDir: '.mosaic/pglite' },
queue: { type: 'local', dataDir: '.mosaic/queue' },
memory: { type: 'keyword' },
};
const STANDALONE_CONFIG: MosaicConfig = {
tier: 'standalone',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@db-host:5432/mosaic' },
queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
memory: { type: 'keyword' },
};
const FEDERATED_CONFIG: MosaicConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
enableVector: true,
},
queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
memory: { type: 'pgvector' },
};
/* ------------------------------------------------------------------ */
/* Tests */
/* ------------------------------------------------------------------ */
describe('detectAndAssertTier', () => {
beforeEach(() => {
vi.clearAllMocks();
// Default: all probes succeed.
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockEnd.mockResolvedValue(undefined);
mocks.mockRedisConnect.mockResolvedValue(undefined);
mocks.mockRedisPing.mockResolvedValue('PONG');
// Re-wire constructor to return a fresh sql-like object each time.
mocks.mockPostgresConstructor.mockImplementation(() => {
const sql = mocks.mockSqlFn as ReturnType<typeof mocks.mockSqlFn>;
(sql as unknown as Record<string, unknown>)['end'] = mocks.mockEnd;
return sql;
});
mocks.MockRedis.mockImplementation(() => ({
connect: mocks.mockRedisConnect,
ping: mocks.mockRedisPing,
disconnect: mocks.mockRedisDisconnect,
}));
});
/* ---------------------------------------------------------------- */
/* 1. local — no-op */
/* ---------------------------------------------------------------- */
it('resolves immediately for tier=local without touching postgres or ioredis', async () => {
await expect(detectAndAssertTier(LOCAL_CONFIG)).resolves.toBeUndefined();
expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 2. standalone — happy path */
/* ---------------------------------------------------------------- */
it('resolves for tier=standalone when postgres and valkey are reachable', async () => {
await expect(detectAndAssertTier(STANDALONE_CONFIG)).resolves.toBeUndefined();
// Postgres was probed (SELECT 1 only — no pgvector check).
expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(1);
expect(mocks.mockSqlFn).toHaveBeenCalledTimes(1);
// Valkey was probed.
expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
expect(mocks.mockRedisPing).toHaveBeenCalledTimes(1);
});
/* ---------------------------------------------------------------- */
/* 3. standalone — postgres unreachable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=postgres when postgres query rejects', async () => {
mocks.mockSqlFn.mockRejectedValueOnce(new Error('connection refused'));
const promise = detectAndAssertTier(STANDALONE_CONFIG);
await expect(promise).rejects.toBeInstanceOf(TierDetectionError);
// Confirm no valkey probe happened (fail fast on first error).
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
it('sets service=postgres on the error when postgres fails', async () => {
mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
try {
await detectAndAssertTier(STANDALONE_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('postgres');
expect(typed.remediation).toContain('docker compose');
}
});
/* ---------------------------------------------------------------- */
/* 4. standalone — valkey unreachable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=valkey when ping fails', async () => {
// Postgres probe succeeds; valkey connect fails.
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockRedisConnect.mockRejectedValue(new Error('ECONNREFUSED'));
try {
await detectAndAssertTier(STANDALONE_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('valkey');
expect(typed.message).toContain('valkey');
expect(typed.remediation).toContain('valkey-federated');
}
});
/* ---------------------------------------------------------------- */
/* 5. federated — happy path */
/* ---------------------------------------------------------------- */
it('resolves for tier=federated when all three checks pass', async () => {
// SELECT 1 and CREATE EXTENSION both succeed.
mocks.mockSqlFn.mockResolvedValue([]);
await expect(detectAndAssertTier(FEDERATED_CONFIG)).resolves.toBeUndefined();
// postgres probe (SELECT 1) + pgvector probe (CREATE EXTENSION) = 2 postgres constructors.
expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(2);
expect(mocks.mockSqlFn).toHaveBeenCalledTimes(2);
// Valkey probed once.
expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
});
/* ---------------------------------------------------------------- */
/* 6. federated — pgvector not installable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=pgvector when CREATE EXTENSION fails', async () => {
// SELECT 1 succeeds (first call), CREATE EXTENSION fails (second call).
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1
.mockRejectedValueOnce(new Error('extension "vector" is not available'));
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
expect(typed.message).toContain('pgvector');
expect(typed.remediation).toContain('pgvector/pgvector');
}
});
/* ---------------------------------------------------------------- */
/* 7. probeValkey honors connectTimeout */
/* ---------------------------------------------------------------- */
it('constructs the ioredis Redis client with connectTimeout: 5000', async () => {
await detectAndAssertTier(STANDALONE_CONFIG);
expect(mocks.MockRedis).toHaveBeenCalledOnce();
const [, options] = mocks.MockRedis.mock.calls[0] as [string, Record<string, unknown>];
expect(options).toMatchObject({ connectTimeout: 5000 });
});
/* ---------------------------------------------------------------- */
/* 8. probePgvector — library-not-installed remediation */
/* ---------------------------------------------------------------- */
it('includes pgvector/pgvector:pg17 in remediation when pgvector library is missing', async () => {
// SELECT 1 succeeds; CREATE EXTENSION fails with the canonical library-missing message.
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
.mockRejectedValueOnce(new Error('extension "vector" is not available')); // probePgvector
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
expect(typed.remediation).toContain('pgvector/pgvector:pg17');
}
});
/* ---------------------------------------------------------------- */
/* 9. probePgvector — permission / other error remediation */
/* ---------------------------------------------------------------- */
it('mentions CREATE permission or superuser in remediation for a generic pgvector error', async () => {
// SELECT 1 succeeds; CREATE EXTENSION fails with a permission error (not the library-missing message).
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
.mockRejectedValueOnce(new Error('permission denied to create extension'));
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
// Must NOT point to the image fix — that's only for the library-missing case.
expect(typed.remediation).not.toContain('pgvector/pgvector:pg17');
// Must mention permissions or superuser.
expect(typed.remediation).toMatch(/CREATE|superuser/i);
}
});
/* ---------------------------------------------------------------- */
/* 10. federated tier rejects non-bullmq queue.type */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=config for federated tier with queue.type !== bullmq', async () => {
const badConfig: MosaicConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
enableVector: true,
},
queue: { type: 'local', dataDir: '.mosaic/queue' },
memory: { type: 'pgvector' },
};
try {
await detectAndAssertTier(badConfig);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('config');
expect(typed.remediation).toContain('bullmq');
}
// No network probes should have been attempted.
expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 11. Error fields populated */
/* ---------------------------------------------------------------- */
it('populates host, port, and remediation on a thrown TierDetectionError', async () => {
mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
let caught: TierDetectionError | undefined;
try {
await detectAndAssertTier(STANDALONE_CONFIG);
} catch (err) {
caught = err as TierDetectionError;
}
expect(caught).toBeInstanceOf(TierDetectionError);
expect(caught!.service).toBe('postgres');
// Host and port are extracted from the Postgres URL in STANDALONE_CONFIG.
expect(caught!.host).toBe('db-host');
expect(caught!.port).toBe(5432);
expect(caught!.remediation).toMatch(/docker compose/i);
expect(caught!.message).toContain('db-host:5432');
});
});

View File

@@ -0,0 +1,220 @@
/**
* Tier Detector — pre-flight service reachability assertions.
*
* Runs BEFORE NestFactory.create() to surface actionable errors immediately
* rather than crashing mid-boot with an opaque stack trace.
*
* Library choices:
* - Postgres: `postgres` npm package (already a dep via @mosaicstack/db / drizzle-orm).
* The spec mentions `pg`, but only `postgres` is installed in this monorepo.
* - Valkey: `ioredis` (already a dep via @mosaicstack/queue → bullmq; same URL
* convention used by the bullmq adapter).
*/
import postgres from 'postgres';
import { Redis } from 'ioredis';
import type { MosaicConfig } from '@mosaicstack/config';
/* ------------------------------------------------------------------ */
/* Structured error type */
/* ------------------------------------------------------------------ */
export class TierDetectionError extends Error {
public readonly service: 'postgres' | 'valkey' | 'pgvector' | 'config';
public readonly host: string;
public readonly port: number;
public readonly remediation: string;
constructor(opts: {
service: 'postgres' | 'valkey' | 'pgvector' | 'config';
host: string;
port: number;
remediation: string;
cause?: unknown;
}) {
const message =
`[tier-detector] ${opts.service} unreachable or unusable at ` +
`${opts.host}:${opts.port}${opts.remediation}`;
super(message, { cause: opts.cause });
this.name = 'TierDetectionError';
this.service = opts.service;
this.host = opts.host;
this.port = opts.port;
this.remediation = opts.remediation;
}
}
/* ------------------------------------------------------------------ */
/* URL helpers */
/* ------------------------------------------------------------------ */
/** Extract host and port from a URL string, returning safe fallbacks on parse failure. */
function parseHostPort(url: string, defaultPort: number): { host: string; port: number } {
try {
const parsed = new URL(url);
const host = parsed.hostname || 'unknown';
const port = parsed.port ? parseInt(parsed.port, 10) : defaultPort;
return { host, port };
} catch {
return { host: 'unknown', port: defaultPort };
}
}
/* ------------------------------------------------------------------ */
/* Postgres probe */
/* ------------------------------------------------------------------ */
async function probePostgres(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 5432);
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
// Run a trivial query to confirm connectivity.
await sql`SELECT 1`;
} catch (cause) {
throw new TierDetectionError({
service: 'postgres',
host,
port,
remediation:
'Start Postgres: `docker compose -f docker-compose.federated.yml --profile federated up -d postgres-federated`',
cause,
});
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {
// Ignore cleanup errors — we already have what we need.
});
}
}
}
/* ------------------------------------------------------------------ */
/* pgvector probe */
/* ------------------------------------------------------------------ */
async function probePgvector(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 5432);
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
// This succeeds whether the extension is already installed or freshly created.
// It errors only if the pgvector shared library is missing from the Postgres binary.
await sql`CREATE EXTENSION IF NOT EXISTS vector`;
} catch (cause) {
const causeMsg = cause instanceof Error ? cause.message.toLowerCase() : '';
const isLibraryMissing = causeMsg.includes('extension "vector" is not available');
const remediation = isLibraryMissing
? 'Use the `pgvector/pgvector:pg17` image, not the stock `postgres:17` image. See `docker-compose.federated.yml`.'
: 'The database role lacks permission to CREATE EXTENSION. Grant `CREATE` on the database, or run as a superuser.';
throw new TierDetectionError({
service: 'pgvector',
host,
port,
remediation,
cause,
});
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {
// Ignore cleanup errors.
});
}
}
}
/* ------------------------------------------------------------------ */
/* Valkey probe */
/* ------------------------------------------------------------------ */
const DEFAULT_VALKEY_URL = 'redis://localhost:6380';
async function probeValkey(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 6380);
const client = new Redis(url, {
enableReadyCheck: false,
maxRetriesPerRequest: 0,
retryStrategy: () => null, // no retries — fail fast
lazyConnect: true,
connectTimeout: 5000, // fail-fast: 5-second hard cap on connection attempt
});
try {
await client.connect();
const pong = await client.ping();
if (pong !== 'PONG') {
throw new Error(`Unexpected PING response: ${pong}`);
}
} catch (cause) {
throw new TierDetectionError({
service: 'valkey',
host,
port,
remediation:
'Start Valkey: `docker compose -f docker-compose.federated.yml --profile federated up -d valkey-federated`',
cause,
});
} finally {
client.disconnect();
}
}
/* ------------------------------------------------------------------ */
/* Public entry point */
/* ------------------------------------------------------------------ */
/**
* Assert that all services required by `config.tier` are reachable.
*
* - `local` — no-op (PGlite is in-process; no external services).
* - `standalone` — assert Postgres + Valkey (if queue.type === 'bullmq').
* - `federated` — assert Postgres + Valkey + pgvector installability.
*
* Throws `TierDetectionError` on the first failure with host:port and
* a remediation hint.
*/
export async function detectAndAssertTier(config: MosaicConfig): Promise<void> {
if (config.tier === 'local') {
// PGlite runs in-process — nothing to probe.
return;
}
const pgUrl =
config.storage.type === 'postgres' ? config.storage.url : 'postgresql://localhost:5432/mosaic';
const valkeyUrl =
config.queue.type === 'bullmq' ? (config.queue.url ?? DEFAULT_VALKEY_URL) : null;
if (config.tier === 'standalone') {
await probePostgres(pgUrl);
if (valkeyUrl) {
await probeValkey(valkeyUrl);
}
return;
}
// tier === 'federated'
// Reject misconfigured queue upfront — federated requires bullmq + a Valkey URL.
if (config.queue.type !== 'bullmq') {
throw new TierDetectionError({
service: 'config',
host: 'localhost',
port: 0,
remediation:
"Federated tier requires queue.type === 'bullmq'. " +
"Set queue: { type: 'bullmq', url: 'redis://...' } in your mosaic.config.json.",
});
}
const federatedValkeyUrl = config.queue.url ?? DEFAULT_VALKEY_URL;
await probePostgres(pgUrl);
await probeValkey(federatedValkeyUrl);
await probePgvector(pgUrl);
}

View File

@@ -20,10 +20,12 @@ import { Logger, ValidationPipe } from '@nestjs/common';
import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify'; import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify';
import helmet from '@fastify/helmet'; import helmet from '@fastify/helmet';
import { listSsoStartupWarnings } from '@mosaicstack/auth'; import { listSsoStartupWarnings } from '@mosaicstack/auth';
import { loadConfig } from '@mosaicstack/config';
import { AppModule } from './app.module.js'; import { AppModule } from './app.module.js';
import { mountAuthHandler } from './auth/auth.controller.js'; import { mountAuthHandler } from './auth/auth.controller.js';
import { mountMcpHandler } from './mcp/mcp.controller.js'; import { mountMcpHandler } from './mcp/mcp.controller.js';
import { McpService } from './mcp/mcp.service.js'; import { McpService } from './mcp/mcp.service.js';
import { detectAndAssertTier, TierDetectionError } from './bootstrap/tier-detector.js';
async function bootstrap(): Promise<void> { async function bootstrap(): Promise<void> {
const logger = new Logger('Bootstrap'); const logger = new Logger('Bootstrap');
@@ -32,6 +34,20 @@ async function bootstrap(): Promise<void> {
throw new Error('BETTER_AUTH_SECRET is required'); throw new Error('BETTER_AUTH_SECRET is required');
} }
// Pre-flight: assert all external services required by the configured tier
// are reachable. Runs before NestFactory.create() so failures are visible
// immediately with actionable remediation hints.
const mosaicConfig = loadConfig();
try {
await detectAndAssertTier(mosaicConfig);
} catch (err) {
if (err instanceof TierDetectionError) {
logger.error(`Tier detection failed: ${err.message}`);
logger.error(`Remediation: ${err.remediation}`);
}
throw err;
}
for (const warning of listSsoStartupWarnings()) { for (const warning of listSsoStartupWarnings()) {
logger.warn(warning); logger.warn(warning);
} }

View File

@@ -0,0 +1,60 @@
# docker-compose.federated.yml — Federated tier overlay
#
# USAGE:
# docker compose -f docker-compose.federated.yml --profile federated up -d
#
# This file is a standalone overlay for the Mosaic federated tier.
# It is NOT an extension of docker-compose.yml — it defines its own services
# and named volumes so it can run independently of the base dev stack.
#
# IMPORTANT — HOST PORT CONFLICTS:
# The federated services bind the same host ports as the base dev stack
# (5433 for Postgres, 6380 for Valkey). You must stop the base dev stack
# before starting the federated stack on the same machine:
# docker compose down
# docker compose -f docker-compose.federated.yml --profile federated up -d
#
# pgvector extension:
# The vector extension is created automatically at first boot via
# ./infra/pg-init/01-extensions.sql (CREATE EXTENSION IF NOT EXISTS vector).
#
# Tier configuration:
# Used by `mosaic` instances configured with `tier: federated`.
# DEFAULT_FEDERATED_CONFIG points at:
# postgresql://mosaic:mosaic@localhost:5433/mosaic
services:
postgres-federated:
image: pgvector/pgvector:pg17
profiles: [federated]
ports:
- '${PG_FEDERATED_HOST_PORT:-5433}:5432'
environment:
POSTGRES_USER: mosaic
POSTGRES_PASSWORD: mosaic
POSTGRES_DB: mosaic
volumes:
- pg_federated_data:/var/lib/postgresql/data
- ./infra/pg-init:/docker-entrypoint-initdb.d:ro
healthcheck:
test: ['CMD-SHELL', 'pg_isready -U mosaic']
interval: 5s
timeout: 3s
retries: 5
valkey-federated:
image: valkey/valkey:8-alpine
profiles: [federated]
ports:
- '${VALKEY_FEDERATED_HOST_PORT:-6380}:6379'
volumes:
- valkey_federated_data:/data
healthcheck:
test: ['CMD', 'valkey-cli', 'ping']
interval: 5s
timeout: 3s
retries: 5
volumes:
pg_federated_data:
valkey_federated_data:

View File

@@ -28,7 +28,7 @@ These are MVP-level checks that don't belong to any single workstream. Updated b
| MVP-T02 | done | Archive install-ux-v2 mission state to `docs/archive/missions/install-ux-v2-20260405/` | IUV-M03 retroactively closed (shipped via PR #446 + releases 0.0.27→0.0.29) | | MVP-T02 | done | Archive install-ux-v2 mission state to `docs/archive/missions/install-ux-v2-20260405/` | IUV-M03 retroactively closed (shipped via PR #446 + releases 0.0.27→0.0.29) |
| MVP-T03 | done | Land federation v1 planning artifacts on `main` | PR #468 merged 2026-04-19 (commit `66512550`) | | MVP-T03 | done | Land federation v1 planning artifacts on `main` | PR #468 merged 2026-04-19 (commit `66512550`) |
| MVP-T04 | not-started | Sync `.mosaic/orchestrator/mission.json` MVP slot with this manifest (milestone enumeration, etc.) | Coord state file; consider whether to repopulate via `mosaic coord` or accept hand-edit | | MVP-T04 | not-started | Sync `.mosaic/orchestrator/mission.json` MVP slot with this manifest (milestone enumeration, etc.) | Coord state file; consider whether to repopulate via `mosaic coord` or accept hand-edit |
| MVP-T05 | not-started | Kick off W1 / FED-M1 — federated tier infrastructure | First execution task in MVP | | MVP-T05 | in-progress | Kick off W1 / FED-M1 — federated tier infrastructure | Session 16 (2026-04-19): FED-M1-01 in-progress on `feat/federation-m1-tier-config` |
| MVP-T06 | not-started | Declare additional workstreams (web dashboard, TUI/CLI parity, remote control, etc.) as scope solidifies | Track each new workstream by adding a row to the Workstream Rollup | | MVP-T06 | not-started | Declare additional workstreams (web dashboard, TUI/CLI parity, remote control, etc.) as scope solidifies | Track each new workstream by adding a row to the Workstream Rollup |
## Pointer to Active Workstream ## Pointer to Active Workstream

View File

@@ -15,20 +15,20 @@
Goal: Gateway runs in `federated` tier with containerized PG+pgvector+Valkey. No federation logic yet. Existing standalone behavior does not regress. Goal: Gateway runs in `federated` tier with containerized PG+pgvector+Valkey. No federation logic yet. Existing standalone behavior does not regress.
| id | status | description | issue | agent | branch | depends_on | estimate | notes | | id | status | description | issue | agent | branch | depends_on | estimate | notes |
| --------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----- | ------ | ------------------------------- | ---------- | -------- | ----------------------------------------------------------------------------------------------------------------- | | --------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----- | ------ | ------------------------------- | ---------- | -------- | -------------------------------------------------------------------------------------------------------------------------------- |
| FED-M1-01 | not-started | Extend `mosaic.config.json` schema: add `"federated"` to `tier` enum in validator + TS types. Keep `local` and `standalone` working. Update schema docs/README where referenced. | #460 | codex | feat/federation-m1-tier-config | — | 4K | Schema lives in `packages/types`; validator in gateway bootstrap. No behavior change yet — enum only. | | FED-M1-01 | done | Extend `mosaic.config.json` schema: add `"federated"` to `tier` enum in validator + TS types. Keep `local` and `standalone` working. Update schema docs/README where referenced. | #460 | sonnet | feat/federation-m1-tier-config | — | 4K | Shipped in PR #470. Renamed `team``standalone`; added `team` deprecation alias; added `DEFAULT_FEDERATED_CONFIG`. |
| FED-M1-02 | not-started | Author `docker-compose.federated.yml` as an overlay profile: Postgres 16 + pgvector extension (port 5433), Valkey (6380), named volumes, healthchecks. Compose-up should boot cleanly on a clean machine. | #460 | codex | feat/federation-m1-compose | FED-M1-01 | 5K | Overlay on existing `docker-compose.yml`; no changes to base file. Add `profile: federated` gating. | | FED-M1-02 | done | Author `docker-compose.federated.yml` as an overlay profile: Postgres 17 + pgvector extension (port 5433), Valkey (6380), named volumes, healthchecks. Compose-up should boot cleanly on a clean machine. | #460 | sonnet | feat/federation-m1-compose | FED-M1-01 | 5K | Shipped in PR #471. Overlay defines `postgres-federated`/`valkey-federated`, profile-gated, with pg-init for pgvector extension. |
| FED-M1-03 | not-started | Add pgvector support to `packages/storage/src/adapters/postgres.ts`: create extension on init (idempotent), expose vector column type in schema helpers. No adapter changes for non-federated tiers. | #460 | codex | feat/federation-m1-pgvector | FED-M1-02 | 8K | Extension create is idempotent `CREATE EXTENSION IF NOT EXISTS vector`. Gate on tier = federated. | | FED-M1-03 | done | Add pgvector support to `packages/storage/src/adapters/postgres.ts`: create extension on init (idempotent), expose vector column type in schema helpers. No adapter changes for non-federated tiers. | #460 | sonnet | feat/federation-m1-pgvector | FED-M1-02 | 8K | Shipped in PR #472. `enableVector` flag on postgres StorageConfig; idempotent CREATE EXTENSION before migrations. |
| FED-M1-04 | not-started | Implement `apps/gateway/src/bootstrap/tier-detector.ts`: reads config, asserts PG/Valkey/pgvector reachable for `federated`, fail-fast with actionable error message on failure. Unit tests for each failure mode. | #460 | codex | feat/federation-m1-detector | FED-M1-03 | 8K | Structured error type with remediation hints. Logs which service failed, with host:port attempted. | | FED-M1-04 | in-progress | Implement `apps/gateway/src/bootstrap/tier-detector.ts`: reads config, asserts PG/Valkey/pgvector reachable for `federated`, fail-fast with actionable error message on failure. Unit tests for each failure mode. | #460 | sonnet | feat/federation-m1-detector | FED-M1-03 | 8K | Worker delivered; reviewer flagged 3 issues (Valkey timeout, pgvector error discrimination, federated/non-bullmq guard) — fixed. |
| FED-M1-05 | not-started | Write `scripts/migrate-to-federated.ts`: one-way migration from `local` (PGlite) / `standalone` (PG without pgvector) → `federated`. Dumps, transforms, loads; dry-run + confirm UX. Idempotent on re-run. | #460 | codex | feat/federation-m1-migrate | FED-M1-04 | 10K | Do NOT run automatically. CLI subcommand `mosaic migrate tier --to federated --dry-run`. Safety rails. | | FED-M1-05 | not-started | Write `scripts/migrate-to-federated.ts`: one-way migration from `local` (PGlite) / `standalone` (PG without pgvector) → `federated`. Dumps, transforms, loads; dry-run + confirm UX. Idempotent on re-run. | #460 | codex | feat/federation-m1-migrate | FED-M1-04 | 10K | Do NOT run automatically. CLI subcommand `mosaic migrate tier --to federated --dry-run`. Safety rails. |
| FED-M1-06 | not-started | Update `mosaic doctor`: report current tier, required services, actual health per service, pgvector presence, overall green/yellow/red. Machine-readable JSON output flag for CI use. | #460 | sonnet | feat/federation-m1-doctor | FED-M1-04 | 6K | Existing doctor output evolves; add `--json` flag. Green/yellow/red + remediation suggestions per issue. | | FED-M1-06 | not-started | Update `mosaic doctor`: report current tier, required services, actual health per service, pgvector presence, overall green/yellow/red. Machine-readable JSON output flag for CI use. | #460 | sonnet | feat/federation-m1-doctor | FED-M1-04 | 6K | Existing doctor output evolves; add `--json` flag. Green/yellow/red + remediation suggestions per issue. |
| FED-M1-07 | not-started | Integration test: gateway boots in `federated` tier with docker-compose `federated` profile; refuses to boot when PG unreachable (asserts fail-fast); pgvector extension query succeeds. | #460 | sonnet | feat/federation-m1-integration | FED-M1-04 | 8K | Vitest + docker-compose test profile. One test file per assertion; real services, no mocks. | | FED-M1-07 | not-started | Integration test: gateway boots in `federated` tier with docker-compose `federated` profile; refuses to boot when PG unreachable (asserts fail-fast); pgvector extension query succeeds. | #460 | sonnet | feat/federation-m1-integration | FED-M1-04 | 8K | Vitest + docker-compose test profile. One test file per assertion; real services, no mocks. |
| FED-M1-08 | not-started | Integration test for migration script: seed a local PGlite with representative data (tasks, notes, users, teams), run migration, assert row counts + key samples equal on federated PG. | #460 | sonnet | feat/federation-m1-migrate-test | FED-M1-05 | 6K | Runs against docker-compose federated profile; uses temp PGlite file; deterministic seed. | | FED-M1-08 | not-started | Integration test for migration script: seed a local PGlite with representative data (tasks, notes, users, teams), run migration, assert row counts + key samples equal on federated PG. | #460 | sonnet | feat/federation-m1-migrate-test | FED-M1-05 | 6K | Runs against docker-compose federated profile; uses temp PGlite file; deterministic seed. |
| FED-M1-09 | not-started | Standalone regression: full agent-session E2E on existing `standalone` tier with a gateway built from this branch. Must pass without referencing any federation module. | #460 | haiku | feat/federation-m1-regression | FED-M1-07 | 4K | Reuse existing e2e harness; just re-point at the federation branch build. Canary that we didn't break it. | | FED-M1-09 | not-started | Standalone regression: full agent-session E2E on existing `standalone` tier with a gateway built from this branch. Must pass without referencing any federation module. | #460 | haiku | feat/federation-m1-regression | FED-M1-07 | 4K | Reuse existing e2e harness; just re-point at the federation branch build. Canary that we didn't break it. |
| FED-M1-10 | not-started | Code review pass: security-focused on the migration script (data-at-rest during migration) + tier detector (error-message sensitivity leakage). Independent reviewer, not authors of tasks 01-09. | #460 | sonnet | — | FED-M1-09 | 8K | Use `feature-dev:code-reviewer` agent. Specifically: no secrets in error messages; no partial-migration footguns. | | FED-M1-10 | not-started | Code review pass: security-focused on the migration script (data-at-rest during migration) + tier detector (error-message sensitivity leakage). Independent reviewer, not authors of tasks 01-09. | #460 | sonnet | — | FED-M1-09 | 8K | Use `feature-dev:code-reviewer` agent. Specifically: no secrets in error messages; no partial-migration footguns. |
| FED-M1-11 | not-started | Docs update: `docs/federation/` operator notes for tier setup; README blurb on federated tier; `docs/guides/` entry for migration. Do NOT touch runbook yet (deferred to FED-M7). | #460 | haiku | feat/federation-m1-docs | FED-M1-10 | 4K | Short, actionable. Link from MISSION-MANIFEST. No decisions captured here — those belong in PRD. | | FED-M1-11 | not-started | Docs update: `docs/federation/` operator notes for tier setup; README blurb on federated tier; `docs/guides/` entry for migration. Do NOT touch runbook yet (deferred to FED-M7). | #460 | haiku | feat/federation-m1-docs | FED-M1-10 | 4K | Short, actionable. Link from MISSION-MANIFEST. No decisions captured here — those belong in PRD. |
| FED-M1-12 | not-started | PR, CI green, merge to main, close #460. | #460 | — | (aggregate) | FED-M1-11 | 3K | Queue-guard before push; wait for green; merge squashed; tea `issue-close` #460. | | FED-M1-12 | not-started | PR, CI green, merge to main, close #460. | #460 | — | (aggregate) | FED-M1-11 | 3K | Queue-guard before push; wait for green; merge squashed; tea `issue-close` #460. |
**M1 total estimate:** ~74K tokens (over-budget vs 20K PRD estimate — explanation below) **M1 total estimate:** ~74K tokens (over-budget vs 20K PRD estimate — explanation below)

View File

@@ -305,3 +305,77 @@ Issues closed: #52, #55, #57, #58, #120-#134
| Install-ux-v2 manifest + tasks + scratchpad + iuv-m03-design | moved to `docs/archive/missions/install-ux-v2-20260405/` with status corrected to complete | | Install-ux-v2 manifest + tasks + scratchpad + iuv-m03-design | moved to `docs/archive/missions/install-ux-v2-20260405/` with status corrected to complete |
**Next:** PR `docs/mvp-mission-manifest` → merge to `main` → next session begins W1 / FED-M1 from clean state. **Next:** PR `docs/mvp-mission-manifest` → merge to `main` → next session begins W1 / FED-M1 from clean state.
---
## Session 16 — 2026-04-19 — claude
**Mode:** Delivery (W1 / FED-M1 execution)
**Branch:** `feat/federation-m1-tier-config`
**Context budget:** 200K, currently ~45% used (compaction-aware)
**Goal:** FED-M1-01 — extend `mosaic.config.json` schema: add `"federated"` to tier enum.
**Critical reconciliation surfaced during pre-flight:**
The federation PRD (`docs/federation/PRD.md` line 247) defines three tiers: `local | standalone | federated`.
The existing code (`packages/config/src/mosaic-config.ts`, `packages/mosaic/src/types.ts`, `packages/mosaic/src/stages/gateway-config.ts`) uses `local | team`.
`team` is the same conceptual tier as PRD `standalone` (Postgres + Valkey, no pgvector). Rather than carrying a confusing alias forever, FED-M1-01 will rename `team``standalone` and add `federated` as a third value, so all downstream federation work has a coherent vocabulary.
Affected files (storage-tier semantics only — Team/workspace usages unaffected):
- `packages/config/src/mosaic-config.ts` (StorageTier type, validator enum, defaults)
- `packages/mosaic/src/types.ts` (GatewayStorageTier)
- `packages/mosaic/src/stages/gateway-config.ts` (~10 references)
- `packages/mosaic/src/stages/gateway-config.spec.ts` (test references)
- Possibly `tools/e2e-install-test.sh` (referenced grep) and headless env hint string
**Worker plan:**
1. Spawn sonnet subagent with explicit task spec + the reconciliation context above.
2. Worker delivers diff; orchestrator runs `pnpm typecheck && pnpm lint && pnpm format:check`.
3. Independent `feature-dev:code-reviewer` subagent reviews diff.
4. Second independent verification subagent (general-purpose, sonnet) verifies reviewer's claims and confirms all `'team'` storage-tier references migrated, no `Team`/workspace bleed.
5. Open PR via tea CLI; wait for CI; queue-guard; squash merge; record actuals.
**Open items:**
- `MVP-T04` (sync `.mosaic/orchestrator/mission.json`) still deferred.
- `team` tier rename touches install wizard headless env vars (`MOSAIC_STORAGE_TIER=team`); will need 0.0.x deprecation note in scratchpad if release notes are written this milestone.
---
## Session 17 — 2026-04-19 — claude
**Mode:** Delivery (W1 / FED-M1 execution; resumed after compaction)
**Branches landed this run:** `feat/federation-m1-tier-config` (PR #470), `feat/federation-m1-compose` (PR #471), `feat/federation-m1-pgvector` (PR #472)
**Branch active at end:** `feat/federation-m1-detector` (FED-M1-04, ready to push)
**Tasks closed:** FED-M1-01, FED-M1-02, FED-M1-03 (all merged to `main` via squash, CI green, issue #460 still open as milestone).
**FED-M1-04 — tier-detector:** Worker delivered `apps/gateway/src/bootstrap/tier-detector.ts` (~210 lines) + `tier-detector.spec.ts` (12 tests). Independent code review (sonnet) returned `changes-required` with 3 issues:
1. CRITICAL: `probeValkey` missing `connectTimeout: 5000` on the ioredis Redis client (defaulted to 10s, violated fail-fast spec).
2. IMPORTANT: `probePgvector` catch block did not discriminate "library not installed" (use `pgvector/pgvector:pg17`) from permission errors.
3. IMPORTANT: Federated tier silently skipped Valkey probe when `queue.type !== 'bullmq'` (computed Valkey URL conditionally).
Worker fix-up round addressed all three:
- L147: `connectTimeout: 5000` added to Redis options
- L113-117: catch block branches on `extension "vector" is not available` substring → distinct remediation per failure mode
- L206-215: federated branch fails fast with `service: 'config'` if `queue.type !== 'bullmq'`, then probes Valkey unconditionally
- 4 new tests (8 → 12 total) cover each fix specifically
Independent verifier (haiku) confirmed all 6 verification claims (line numbers, test presence, suite green: 12/12 PASS).
**Process note — review pipeline working as designed:**
Initial verifier (haiku) on the first delivery returned "OK to ship" but missed the 3 deeper issues that the sonnet code-reviewer caught. This validates the user's "always verify subagent claims independently with another subagent" rule — but specifically with the **right tier** for the task: code review needs sonnet-level reasoning, while haiku is fine for verifying surface claims (line counts, file existence) once review issues are known. Going forward: code review uses sonnet (`feature-dev:code-reviewer`), claim verification uses haiku.
**Followup tasks tracked but deferred:**
- #7: `tier=local` hardcoded in gateway-config resume branches (~262, ~317) — pre-existing bug, fix during M1-06 (doctor) or M1-09 (regression).
- #8: confirm `packages/config/dist` not git-tracked.
**Next:** PR for FED-M1-04 → CI wait → merge. Then FED-M1-05 (migration script, codex/sonnet, 10K).

View File

@@ -1,7 +1,9 @@
export type { MosaicConfig, StorageTier, MemoryConfigRef } from './mosaic-config.js'; export type { MosaicConfig, StorageTier, MemoryConfigRef } from './mosaic-config.js';
export { export {
DEFAULT_LOCAL_CONFIG, DEFAULT_LOCAL_CONFIG,
DEFAULT_TEAM_CONFIG, DEFAULT_STANDALONE_CONFIG,
DEFAULT_FEDERATED_CONFIG,
loadConfig, loadConfig,
validateConfig, validateConfig,
detectFromEnv,
} from './mosaic-config.js'; } from './mosaic-config.js';

View File

@@ -0,0 +1,170 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import {
validateConfig,
detectFromEnv,
DEFAULT_LOCAL_CONFIG,
DEFAULT_STANDALONE_CONFIG,
DEFAULT_FEDERATED_CONFIG,
} from './mosaic-config.js';
describe('validateConfig — tier enum', () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let stderrSpy: any;
beforeEach(() => {
stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
});
afterEach(() => {
stderrSpy.mockRestore();
});
it('accepts tier="local"', () => {
const result = validateConfig({
tier: 'local',
storage: { type: 'pglite', dataDir: '.mosaic/storage-pglite' },
queue: { type: 'local', dataDir: '.mosaic/queue' },
memory: { type: 'keyword' },
});
expect(result.tier).toBe('local');
});
it('accepts tier="standalone"', () => {
const result = validateConfig({
tier: 'standalone',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5432/mosaic' },
queue: { type: 'bullmq' },
memory: { type: 'keyword' },
});
expect(result.tier).toBe('standalone');
});
it('accepts tier="federated"', () => {
const result = validateConfig({
tier: 'federated',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5433/mosaic' },
queue: { type: 'bullmq' },
memory: { type: 'pgvector' },
});
expect(result.tier).toBe('federated');
});
it('accepts deprecated tier="team" as alias for "standalone" and emits a deprecation warning', () => {
const result = validateConfig({
tier: 'team',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5432/mosaic' },
queue: { type: 'bullmq' },
memory: { type: 'keyword' },
});
expect(result.tier).toBe('standalone');
expect(stderrSpy).toHaveBeenCalledWith(expect.stringContaining('DEPRECATED'));
});
it('rejects an invalid tier with an error listing all three valid values', () => {
expect(() =>
validateConfig({
tier: 'invalid',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5432/mosaic' },
queue: { type: 'bullmq' },
memory: { type: 'keyword' },
}),
).toThrow(/local.*standalone.*federated|federated.*standalone.*local/);
});
it('error message for invalid tier mentions all three valid values', () => {
let message = '';
try {
validateConfig({
tier: 'invalid',
storage: { type: 'postgres', url: 'postgresql://...' },
queue: { type: 'bullmq' },
memory: { type: 'keyword' },
});
} catch (err) {
message = err instanceof Error ? err.message : String(err);
}
expect(message).toContain('"local"');
expect(message).toContain('"standalone"');
expect(message).toContain('"federated"');
});
});
describe('DEFAULT_* config constants', () => {
it('DEFAULT_LOCAL_CONFIG has tier="local"', () => {
expect(DEFAULT_LOCAL_CONFIG.tier).toBe('local');
});
it('DEFAULT_STANDALONE_CONFIG has tier="standalone"', () => {
expect(DEFAULT_STANDALONE_CONFIG.tier).toBe('standalone');
});
it('DEFAULT_FEDERATED_CONFIG has tier="federated" and pgvector memory', () => {
expect(DEFAULT_FEDERATED_CONFIG.tier).toBe('federated');
expect(DEFAULT_FEDERATED_CONFIG.memory.type).toBe('pgvector');
});
it('DEFAULT_FEDERATED_CONFIG uses port 5433 (distinct from standalone 5432)', () => {
const url = (DEFAULT_FEDERATED_CONFIG.storage as { url: string }).url;
expect(url).toContain('5433');
});
it('DEFAULT_FEDERATED_CONFIG has enableVector=true on storage', () => {
const storage = DEFAULT_FEDERATED_CONFIG.storage as {
type: string;
url: string;
enableVector?: boolean;
};
expect(storage.enableVector).toBe(true);
});
});
describe('detectFromEnv — tier env-var routing', () => {
const originalEnv = process.env;
beforeEach(() => {
// Work on a fresh copy so individual tests can set/delete keys freely.
process.env = { ...originalEnv };
delete process.env['MOSAIC_STORAGE_TIER'];
delete process.env['DATABASE_URL'];
delete process.env['VALKEY_URL'];
});
afterEach(() => {
process.env = originalEnv;
});
it('no env vars → returns local config', () => {
const config = detectFromEnv();
expect(config.tier).toBe('local');
expect(config.storage.type).toBe('pglite');
expect(config.memory.type).toBe('keyword');
});
it('MOSAIC_STORAGE_TIER=federated alone → returns federated config with enableVector=true', () => {
process.env['MOSAIC_STORAGE_TIER'] = 'federated';
const config = detectFromEnv();
expect(config.tier).toBe('federated');
expect(config.memory.type).toBe('pgvector');
const storage = config.storage as { type: string; enableVector?: boolean };
expect(storage.enableVector).toBe(true);
});
it('MOSAIC_STORAGE_TIER=federated + DATABASE_URL → uses the URL and still has enableVector=true', () => {
process.env['MOSAIC_STORAGE_TIER'] = 'federated';
process.env['DATABASE_URL'] = 'postgresql://custom:pass@db.example.com:5432/mydb';
const config = detectFromEnv();
expect(config.tier).toBe('federated');
const storage = config.storage as { type: string; url: string; enableVector?: boolean };
expect(storage.url).toBe('postgresql://custom:pass@db.example.com:5432/mydb');
expect(storage.enableVector).toBe(true);
expect(config.memory.type).toBe('pgvector');
});
it('MOSAIC_STORAGE_TIER=standalone alone → returns standalone-shaped config (not local)', () => {
process.env['MOSAIC_STORAGE_TIER'] = 'standalone';
const config = detectFromEnv();
expect(config.tier).toBe('standalone');
expect(config.storage.type).toBe('postgres');
expect(config.memory.type).toBe('keyword');
});
});

View File

@@ -7,7 +7,7 @@ import type { QueueAdapterConfig as QueueConfig } from '@mosaicstack/queue';
/* Types */ /* Types */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
export type StorageTier = 'local' | 'team'; export type StorageTier = 'local' | 'standalone' | 'federated';
export interface MemoryConfigRef { export interface MemoryConfigRef {
type: 'pgvector' | 'sqlite-vec' | 'keyword'; type: 'pgvector' | 'sqlite-vec' | 'keyword';
@@ -31,10 +31,21 @@ export const DEFAULT_LOCAL_CONFIG: MosaicConfig = {
memory: { type: 'keyword' }, memory: { type: 'keyword' },
}; };
export const DEFAULT_TEAM_CONFIG: MosaicConfig = { export const DEFAULT_STANDALONE_CONFIG: MosaicConfig = {
tier: 'team', tier: 'standalone',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5432/mosaic' }, storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@localhost:5432/mosaic' },
queue: { type: 'bullmq' }, queue: { type: 'bullmq' },
memory: { type: 'keyword' },
};
export const DEFAULT_FEDERATED_CONFIG: MosaicConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@localhost:5433/mosaic',
enableVector: true,
},
queue: { type: 'bullmq' },
memory: { type: 'pgvector' }, memory: { type: 'pgvector' },
}; };
@@ -42,7 +53,7 @@ export const DEFAULT_TEAM_CONFIG: MosaicConfig = {
/* Validation */ /* Validation */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
const VALID_TIERS = new Set<string>(['local', 'team']); const VALID_TIERS = new Set<string>(['local', 'standalone', 'federated']);
const VALID_STORAGE_TYPES = new Set<string>(['postgres', 'pglite', 'files']); const VALID_STORAGE_TYPES = new Set<string>(['postgres', 'pglite', 'files']);
const VALID_QUEUE_TYPES = new Set<string>(['bullmq', 'local']); const VALID_QUEUE_TYPES = new Set<string>(['bullmq', 'local']);
const VALID_MEMORY_TYPES = new Set<string>(['pgvector', 'sqlite-vec', 'keyword']); const VALID_MEMORY_TYPES = new Set<string>(['pgvector', 'sqlite-vec', 'keyword']);
@@ -55,9 +66,19 @@ export function validateConfig(raw: unknown): MosaicConfig {
const obj = raw as Record<string, unknown>; const obj = raw as Record<string, unknown>;
// tier // tier
const tier = obj['tier']; let tier = obj['tier'];
// Deprecated alias: 'team' → 'standalone' (kept for backward-compat with 0.0.x installs)
if (tier === 'team') {
process.stderr.write(
'[mosaic] DEPRECATED: tier="team" is deprecated — use "standalone" instead. ' +
'Update your mosaic.config.json.\n',
);
tier = 'standalone';
}
if (typeof tier !== 'string' || !VALID_TIERS.has(tier)) { if (typeof tier !== 'string' || !VALID_TIERS.has(tier)) {
throw new Error(`Invalid tier "${String(tier)}" — expected "local" or "team"`); throw new Error(
`Invalid tier "${String(tier)}" — expected "local", "standalone", or "federated"`,
);
} }
// storage // storage
@@ -102,10 +123,52 @@ export function validateConfig(raw: unknown): MosaicConfig {
/* Loader */ /* Loader */
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
function detectFromEnv(): MosaicConfig { export function detectFromEnv(): MosaicConfig {
const tier = process.env['MOSAIC_STORAGE_TIER'];
if (tier === 'federated') {
if (process.env['DATABASE_URL']) {
return {
...DEFAULT_FEDERATED_CONFIG,
storage: {
type: 'postgres',
url: process.env['DATABASE_URL'],
enableVector: true,
},
queue: {
type: 'bullmq',
url: process.env['VALKEY_URL'],
},
};
}
// MOSAIC_STORAGE_TIER=federated without DATABASE_URL — use the default
// federated config (port 5433, enableVector: true, pgvector memory).
return DEFAULT_FEDERATED_CONFIG;
}
if (tier === 'standalone') {
if (process.env['DATABASE_URL']) {
return {
...DEFAULT_STANDALONE_CONFIG,
storage: {
type: 'postgres',
url: process.env['DATABASE_URL'],
},
queue: {
type: 'bullmq',
url: process.env['VALKEY_URL'],
},
};
}
// MOSAIC_STORAGE_TIER=standalone without DATABASE_URL — use the default
// standalone config instead of silently falling back to local.
return DEFAULT_STANDALONE_CONFIG;
}
// Legacy: DATABASE_URL set without MOSAIC_STORAGE_TIER — treat as standalone.
if (process.env['DATABASE_URL']) { if (process.env['DATABASE_URL']) {
return { return {
...DEFAULT_TEAM_CONFIG, ...DEFAULT_STANDALONE_CONFIG,
storage: { storage: {
type: 'postgres', type: 'postgres',
url: process.env['DATABASE_URL'], url: process.env['DATABASE_URL'],
@@ -116,6 +179,7 @@ function detectFromEnv(): MosaicConfig {
}, },
}; };
} }
return DEFAULT_LOCAL_CONFIG; return DEFAULT_LOCAL_CONFIG;
} }

View File

@@ -372,7 +372,11 @@ export const messages = pgTable(
// ─── pgvector custom type ─────────────────────────────────────────────────── // ─── pgvector custom type ───────────────────────────────────────────────────
const vector = customType<{ data: number[]; driverParam: string; config: { dimensions: number } }>({ export const vector = customType<{
data: number[];
driverParam: string;
config: { dimensions: number };
}>({
dataType(config) { dataType(config) {
return `vector(${config?.dimensions ?? 1536})`; return `vector(${config?.dimensions ?? 1536})`;
}, },

View File

@@ -216,8 +216,8 @@ describe('gatewayConfigStage', () => {
expect(daemonState.startCalled).toBe(0); expect(daemonState.startCalled).toBe(0);
}); });
it('honors MOSAIC_STORAGE_TIER=team in headless path', async () => { it('honors MOSAIC_STORAGE_TIER=standalone in headless path', async () => {
process.env['MOSAIC_STORAGE_TIER'] = 'team'; process.env['MOSAIC_STORAGE_TIER'] = 'standalone';
process.env['MOSAIC_DATABASE_URL'] = 'postgresql://test/db'; process.env['MOSAIC_DATABASE_URL'] = 'postgresql://test/db';
process.env['MOSAIC_VALKEY_URL'] = 'redis://test:6379'; process.env['MOSAIC_VALKEY_URL'] = 'redis://test:6379';
@@ -231,12 +231,75 @@ describe('gatewayConfigStage', () => {
}); });
expect(result.ready).toBe(true); expect(result.ready).toBe(true);
expect(state.gateway?.tier).toBe('team'); expect(state.gateway?.tier).toBe('standalone');
const envContents = readFileSync(daemonState.envFile, 'utf-8'); const envContents = readFileSync(daemonState.envFile, 'utf-8');
expect(envContents).toContain('DATABASE_URL=postgresql://test/db'); expect(envContents).toContain('DATABASE_URL=postgresql://test/db');
expect(envContents).toContain('VALKEY_URL=redis://test:6379'); expect(envContents).toContain('VALKEY_URL=redis://test:6379');
const mosaicConfig = JSON.parse(readFileSync(daemonState.mosaicConfigFile, 'utf-8')); const mosaicConfig = JSON.parse(readFileSync(daemonState.mosaicConfigFile, 'utf-8'));
expect(mosaicConfig.tier).toBe('team'); expect(mosaicConfig.tier).toBe('standalone');
});
it('accepts deprecated MOSAIC_STORAGE_TIER=team as alias for standalone', async () => {
process.env['MOSAIC_STORAGE_TIER'] = 'team';
process.env['MOSAIC_DATABASE_URL'] = 'postgresql://test/db';
process.env['MOSAIC_VALKEY_URL'] = 'redis://test:6379';
const p = buildPrompter();
const state = makeState('/home/user/.config/mosaic');
const result = await gatewayConfigStage(p, state, {
host: 'localhost',
defaultPort: 14242,
skipInstall: true,
});
// Deprecated alias 'team' maps to 'standalone'
expect(result.ready).toBe(true);
expect(state.gateway?.tier).toBe('standalone');
const mosaicConfig = JSON.parse(readFileSync(daemonState.mosaicConfigFile, 'utf-8'));
expect(mosaicConfig.tier).toBe('standalone');
});
it('honors MOSAIC_STORAGE_TIER=federated in headless path', async () => {
process.env['MOSAIC_STORAGE_TIER'] = 'federated';
process.env['MOSAIC_DATABASE_URL'] = 'postgresql://test/feddb';
process.env['MOSAIC_VALKEY_URL'] = 'redis://test:6379';
const p = buildPrompter();
const state = makeState('/home/user/.config/mosaic');
const result = await gatewayConfigStage(p, state, {
host: 'localhost',
defaultPort: 14242,
skipInstall: true,
});
expect(result.ready).toBe(true);
expect(state.gateway?.tier).toBe('federated');
const envContents = readFileSync(daemonState.envFile, 'utf-8');
expect(envContents).toContain('DATABASE_URL=postgresql://test/feddb');
const mosaicConfig = JSON.parse(readFileSync(daemonState.mosaicConfigFile, 'utf-8'));
expect(mosaicConfig.tier).toBe('federated');
expect(mosaicConfig.memory.type).toBe('pgvector');
});
it('rejects an unknown MOSAIC_STORAGE_TIER value in headless mode with a descriptive warning', async () => {
process.env['MOSAIC_STORAGE_TIER'] = 'federatd'; // deliberate typo
const warnFn = vi.fn();
const p = buildPrompter({ warn: warnFn });
const state = makeState('/home/user/.config/mosaic');
const result = await gatewayConfigStage(p, state, {
host: 'localhost',
defaultPort: 14242,
skipInstall: true,
});
// The stage surfaces validation errors as ready:false (warning is shown to the user).
expect(result.ready).toBe(false);
// The warning message must name all three valid values.
expect(warnFn).toHaveBeenCalledWith(expect.stringMatching(/local.*standalone.*federated/i));
}); });
it('regenerates config when portOverride differs from saved GATEWAY_PORT', async () => { it('regenerates config when portOverride differs from saved GATEWAY_PORT', async () => {

View File

@@ -84,10 +84,15 @@ async function promptTier(p: WizardPrompter): Promise<GatewayStorageTier> {
hint: 'embedded database, no dependencies', hint: 'embedded database, no dependencies',
}, },
{ {
value: 'team', value: 'standalone',
label: 'Team', label: 'Standalone',
hint: 'PostgreSQL + Valkey required', hint: 'PostgreSQL + Valkey required',
}, },
{
value: 'federated',
label: 'Federated',
hint: 'PostgreSQL + Valkey + pgvector, federation server+client',
},
], ],
}); });
return tier; return tier;
@@ -437,7 +442,21 @@ async function collectAndWriteConfig(
p.log('Headless mode detected — reading configuration from environment variables.'); p.log('Headless mode detected — reading configuration from environment variables.');
const storageTierEnv = process.env['MOSAIC_STORAGE_TIER'] ?? 'local'; const storageTierEnv = process.env['MOSAIC_STORAGE_TIER'] ?? 'local';
tier = storageTierEnv === 'team' ? 'team' : 'local'; if (storageTierEnv === 'team') {
// Deprecated alias — warn and treat as standalone
process.stderr.write(
'[mosaic] DEPRECATED: MOSAIC_STORAGE_TIER=team is deprecated — use "standalone" instead.\n',
);
tier = 'standalone';
} else if (storageTierEnv === 'standalone' || storageTierEnv === 'federated') {
tier = storageTierEnv;
} else if (storageTierEnv !== '' && storageTierEnv !== 'local') {
throw new GatewayConfigValidationError(
`Invalid MOSAIC_STORAGE_TIER="${storageTierEnv}" — expected "local", "standalone", or "federated" (deprecated alias "team" also accepted)`,
);
} else {
tier = 'local';
}
const portEnv = process.env['MOSAIC_GATEWAY_PORT']; const portEnv = process.env['MOSAIC_GATEWAY_PORT'];
port = portEnv ? parseInt(portEnv, 10) : opts.defaultPort; port = portEnv ? parseInt(portEnv, 10) : opts.defaultPort;
@@ -453,13 +472,13 @@ async function collectAndWriteConfig(
hostname = hostnameEnv; hostname = hostnameEnv;
corsOrigin = corsOverride ?? deriveCorsOrigin(hostnameEnv, 3000); corsOrigin = corsOverride ?? deriveCorsOrigin(hostnameEnv, 3000);
if (tier === 'team') { if (tier === 'standalone' || tier === 'federated') {
const missing: string[] = []; const missing: string[] = [];
if (!databaseUrl) missing.push('MOSAIC_DATABASE_URL'); if (!databaseUrl) missing.push('MOSAIC_DATABASE_URL');
if (!valkeyUrl) missing.push('MOSAIC_VALKEY_URL'); if (!valkeyUrl) missing.push('MOSAIC_VALKEY_URL');
if (missing.length > 0) { if (missing.length > 0) {
throw new GatewayConfigValidationError( throw new GatewayConfigValidationError(
'Headless install with tier=team requires env vars: ' + missing.join(', '), `Headless install with tier=${tier} requires env vars: ` + missing.join(', '),
); );
} }
} }
@@ -467,11 +486,15 @@ async function collectAndWriteConfig(
tier = await promptTier(p); tier = await promptTier(p);
port = await promptPort(p, opts.defaultPort); port = await promptPort(p, opts.defaultPort);
if (tier === 'team') { if (tier === 'standalone' || tier === 'federated') {
const defaultDbUrl =
tier === 'federated'
? 'postgresql://mosaic:mosaic@localhost:5433/mosaic'
: 'postgresql://mosaic:mosaic@localhost:5432/mosaic';
databaseUrl = await p.text({ databaseUrl = await p.text({
message: 'DATABASE_URL', message: 'DATABASE_URL',
initialValue: 'postgresql://mosaic:mosaic@localhost:5433/mosaic', initialValue: defaultDbUrl,
defaultValue: 'postgresql://mosaic:mosaic@localhost:5433/mosaic', defaultValue: defaultDbUrl,
}); });
valkeyUrl = await p.text({ valkeyUrl = await p.text({
message: 'VALKEY_URL', message: 'VALKEY_URL',
@@ -521,7 +544,7 @@ async function collectAndWriteConfig(
`OTEL_SERVICE_NAME=mosaic-gateway`, `OTEL_SERVICE_NAME=mosaic-gateway`,
]; ];
if (tier === 'team' && databaseUrl && valkeyUrl) { if ((tier === 'standalone' || tier === 'federated') && databaseUrl && valkeyUrl) {
envLines.push(`DATABASE_URL=${databaseUrl}`); envLines.push(`DATABASE_URL=${databaseUrl}`);
envLines.push(`VALKEY_URL=${valkeyUrl}`); envLines.push(`VALKEY_URL=${valkeyUrl}`);
} }
@@ -545,12 +568,19 @@ async function collectAndWriteConfig(
queue: { type: 'local', dataDir: join(opts.gatewayHome, 'queue') }, queue: { type: 'local', dataDir: join(opts.gatewayHome, 'queue') },
memory: { type: 'keyword' }, memory: { type: 'keyword' },
} }
: { : tier === 'federated'
tier: 'team', ? {
storage: { type: 'postgres', url: databaseUrl }, tier: 'federated',
queue: { type: 'bullmq', url: valkeyUrl }, storage: { type: 'postgres', url: databaseUrl },
memory: { type: 'pgvector' }, queue: { type: 'bullmq', url: valkeyUrl },
}; memory: { type: 'pgvector' },
}
: {
tier: 'standalone',
storage: { type: 'postgres', url: databaseUrl },
queue: { type: 'bullmq', url: valkeyUrl },
memory: { type: 'keyword' },
};
writeFileSync(opts.mosaicConfigFile, JSON.stringify(mosaicConfig, null, 2) + '\n', { writeFileSync(opts.mosaicConfigFile, JSON.stringify(mosaicConfig, null, 2) + '\n', {
mode: 0o600, mode: 0o600,

View File

@@ -58,7 +58,7 @@ export interface HooksState {
acceptedAt?: string; acceptedAt?: string;
} }
export type GatewayStorageTier = 'local' | 'team'; export type GatewayStorageTier = 'local' | 'standalone' | 'federated';
export interface GatewayAdminState { export interface GatewayAdminState {
name: string; name: string;

View File

@@ -0,0 +1,107 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import type { DbHandle } from '@mosaicstack/db';
// Mock @mosaicstack/db before importing the adapter
vi.mock('@mosaicstack/db', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = await importOriginal<Record<string, any>>();
return {
...actual,
createDb: vi.fn(),
runMigrations: vi.fn().mockResolvedValue(undefined),
};
});
import { createDb, runMigrations } from '@mosaicstack/db';
import { PostgresAdapter } from './postgres.js';
describe('PostgresAdapter — vector extension gating', () => {
let mockExecute: ReturnType<typeof vi.fn>;
let mockDb: { execute: ReturnType<typeof vi.fn> };
let mockHandle: Pick<DbHandle, 'close'> & { db: typeof mockDb };
beforeEach(() => {
vi.clearAllMocks();
mockExecute = vi.fn().mockResolvedValue(undefined);
mockDb = { execute: mockExecute };
mockHandle = { db: mockDb, close: vi.fn().mockResolvedValue(undefined) };
vi.mocked(createDb).mockReturnValue(mockHandle as unknown as DbHandle);
});
it('calls db.execute with CREATE EXTENSION IF NOT EXISTS vector when enableVector=true', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: true,
});
await adapter.migrate();
// Should have called execute
expect(mockExecute).toHaveBeenCalledTimes(1);
// Verify the SQL contains the extension creation statement.
// Prefer Drizzle's public toSQL() API; fall back to queryChunks if unavailable.
// NOTE: queryChunks is an undocumented Drizzle internal (drizzle-orm ^0.45.x).
// toSQL() was not present on the raw sql`` result in this version — if a future
// Drizzle upgrade adds it, remove the fallback path and delete this comment.
const sqlObj = mockExecute.mock.calls[0]![0] as {
toSQL?: () => { sql: string; params: unknown[] };
queryChunks?: Array<{ value: string[] }>;
};
const sqlText = sqlObj.toSQL
? sqlObj.toSQL().sql.toLowerCase()
: (sqlObj.queryChunks ?? [])
.flatMap((chunk) => chunk.value)
.join('')
.toLowerCase();
expect(sqlText).toContain('create extension if not exists vector');
});
it('does NOT call db.execute for extension when enableVector is false', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: false,
});
await adapter.migrate();
expect(mockExecute).not.toHaveBeenCalled();
expect(vi.mocked(runMigrations)).toHaveBeenCalledOnce();
});
it('does NOT call db.execute for extension when enableVector is unset', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
});
await adapter.migrate();
expect(mockExecute).not.toHaveBeenCalled();
expect(vi.mocked(runMigrations)).toHaveBeenCalledOnce();
});
it('calls runMigrations after the extension is created', async () => {
const callOrder: string[] = [];
mockExecute.mockImplementation(() => {
callOrder.push('execute');
return Promise.resolve(undefined);
});
vi.mocked(runMigrations).mockImplementation(() => {
callOrder.push('runMigrations');
return Promise.resolve();
});
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: true,
});
await adapter.migrate();
expect(callOrder).toEqual(['execute', 'runMigrations']);
});
});

View File

@@ -66,13 +66,19 @@ export class PostgresAdapter implements StorageAdapter {
private handle: DbHandle; private handle: DbHandle;
private db: Db; private db: Db;
private url: string; private url: string;
private enableVector: boolean;
constructor(config: Extract<StorageConfig, { type: 'postgres' }>) { constructor(config: Extract<StorageConfig, { type: 'postgres' }>) {
this.url = config.url; this.url = config.url;
this.enableVector = config.enableVector ?? false;
this.handle = createDb(config.url); this.handle = createDb(config.url);
this.db = this.handle.db; this.db = this.handle.db;
} }
private async ensureVectorExtension(): Promise<void> {
await this.db.execute(sql`CREATE EXTENSION IF NOT EXISTS vector`);
}
async create<T extends Record<string, unknown>>( async create<T extends Record<string, unknown>>(
collection: string, collection: string,
data: T, data: T,
@@ -149,6 +155,9 @@ export class PostgresAdapter implements StorageAdapter {
} }
async migrate(): Promise<void> { async migrate(): Promise<void> {
if (this.enableVector) {
await this.ensureVectorExtension();
}
await runMigrations(this.url); await runMigrations(this.url);
} }

View File

@@ -38,6 +38,6 @@ export interface StorageAdapter {
} }
export type StorageConfig = export type StorageConfig =
| { type: 'postgres'; url: string } | { type: 'postgres'; url: string; enableVector?: boolean }
| { type: 'pglite'; dataDir?: string } | { type: 'pglite'; dataDir?: string }
| { type: 'files'; dataDir: string; format?: 'json' | 'md' }; | { type: 'files'; dataDir: string; format?: 'json' | 'md' };

6
pnpm-lock.yaml generated
View File

@@ -152,12 +152,18 @@ importers:
fastify: fastify:
specifier: ^5.0.0 specifier: ^5.0.0
version: 5.8.2 version: 5.8.2
ioredis:
specifier: ^5.10.0
version: 5.10.0
node-cron: node-cron:
specifier: ^4.2.1 specifier: ^4.2.1
version: 4.2.1 version: 4.2.1
openai: openai:
specifier: ^6.32.0 specifier: ^6.32.0
version: 6.32.0(ws@8.20.0)(zod@4.3.6) version: 6.32.0(ws@8.20.0)(zod@4.3.6)
postgres:
specifier: ^3.4.8
version: 3.4.8
reflect-metadata: reflect-metadata:
specifier: ^0.2.0 specifier: ^0.2.0
version: 0.2.2 version: 0.2.2