chore: consolidate new foundation and archive v1 (#1495)

This commit is contained in:
2026-09-07 12:32:57 -05:00
3511 changed files with 727899 additions and 10 deletions
+44
View File
@@ -0,0 +1,44 @@
{
"name": "@mosaicstack/storage",
"version": "0.0.4",
"repository": {
"type": "git",
"url": "https://git.mosaicstack.dev/mosaicstack/stack.git",
"directory": "packages/storage"
},
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"scripts": {
"build": "tsc",
"lint": "eslint src",
"typecheck": "tsc --noEmit",
"test": "vitest run --passWithNoTests"
},
"dependencies": {
"@electric-sql/pglite": "^0.2.17",
"@mosaicstack/db": "workspace:^",
"@mosaicstack/types": "workspace:*",
"commander": "^13.0.0",
"ioredis": "^5.10.0",
"postgres": "^3.4.8"
},
"devDependencies": {
"drizzle-orm": "^0.45.1",
"typescript": "^5.8.0",
"vitest": "^2.0.0"
},
"publishConfig": {
"registry": "https://git.mosaicstack.dev/api/packages/mosaicstack/npm/",
"access": "public"
},
"files": [
"dist"
]
}
@@ -0,0 +1,202 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { PgliteAdapter } from './pglite.js';
describe('PgliteAdapter', () => {
let adapter: PgliteAdapter;
beforeEach(async () => {
// In-memory PGlite instance — no dataDir = memory mode
adapter = new PgliteAdapter({ type: 'pglite' });
await adapter.migrate();
});
afterEach(async () => {
await adapter.close();
});
describe('CRUD', () => {
it('creates and reads a record', async () => {
const created = await adapter.create('users', { name: 'Alice', email: '[email protected]' });
expect(created.id).toBeDefined();
expect(created.name).toBe('Alice');
const read = await adapter.read('users', created.id);
expect(read).not.toBeNull();
expect(read!.name).toBe('Alice');
expect(read!.email).toBe('[email protected]');
});
it('returns null for non-existent record', async () => {
const result = await adapter.read('users', 'does-not-exist');
expect(result).toBeNull();
});
it('updates a record', async () => {
const created = await adapter.create('users', { name: 'Alice' });
const updated = await adapter.update('users', created.id, { name: 'Bob' });
expect(updated).toBe(true);
const read = await adapter.read('users', created.id);
expect(read!.name).toBe('Bob');
});
it('update returns false for non-existent record', async () => {
const result = await adapter.update('users', 'does-not-exist', { name: 'X' });
expect(result).toBe(false);
});
it('deletes a record', async () => {
const created = await adapter.create('users', { name: 'Alice' });
const deleted = await adapter.delete('users', created.id);
expect(deleted).toBe(true);
const read = await adapter.read('users', created.id);
expect(read).toBeNull();
});
it('delete returns false for non-existent record', async () => {
const result = await adapter.delete('users', 'does-not-exist');
expect(result).toBe(false);
});
});
describe('find', () => {
it('finds records with filter', async () => {
await adapter.create('users', { name: 'Alice', role: 'admin' });
await adapter.create('users', { name: 'Bob', role: 'user' });
await adapter.create('users', { name: 'Charlie', role: 'admin' });
const admins = await adapter.find('users', { role: 'admin' });
expect(admins).toHaveLength(2);
expect(admins.map((u) => u.name).sort()).toEqual(['Alice', 'Charlie']);
});
it('finds all records without filter', async () => {
await adapter.create('users', { name: 'Alice' });
await adapter.create('users', { name: 'Bob' });
const all = await adapter.find('users');
expect(all).toHaveLength(2);
});
it('supports limit and offset', async () => {
for (let i = 0; i < 5; i++) {
await adapter.create('users', { name: `User${i.toString()}`, idx: i });
}
const page = await adapter.find('users', undefined, {
limit: 2,
offset: 1,
orderBy: 'created_at',
});
expect(page).toHaveLength(2);
});
it('findOne returns first match', async () => {
await adapter.create('users', { name: 'Alice', role: 'admin' });
await adapter.create('users', { name: 'Bob', role: 'user' });
const found = await adapter.findOne('users', { role: 'user' });
expect(found).not.toBeNull();
expect(found!.name).toBe('Bob');
});
it('findOne returns null when no match', async () => {
const result = await adapter.findOne('users', { role: 'nonexistent' });
expect(result).toBeNull();
});
});
describe('count', () => {
it('counts all records', async () => {
await adapter.create('users', { name: 'Alice' });
await adapter.create('users', { name: 'Bob' });
const total = await adapter.count('users');
expect(total).toBe(2);
});
it('counts with filter', async () => {
await adapter.create('users', { name: 'Alice', role: 'admin' });
await adapter.create('users', { name: 'Bob', role: 'user' });
await adapter.create('users', { name: 'Charlie', role: 'admin' });
const adminCount = await adapter.count('users', { role: 'admin' });
expect(adminCount).toBe(2);
});
it('returns 0 for empty collection', async () => {
const count = await adapter.count('users');
expect(count).toBe(0);
});
});
describe('transaction', () => {
it('commits on success', async () => {
await adapter.transaction(async (tx) => {
await tx.create('users', { name: 'Alice' });
await tx.create('users', { name: 'Bob' });
});
const count = await adapter.count('users');
expect(count).toBe(2);
});
it('rolls back on error', async () => {
await expect(
adapter.transaction(async (tx) => {
await tx.create('users', { name: 'Alice' });
throw new Error('rollback test');
}),
).rejects.toThrow('rollback test');
const count = await adapter.count('users');
expect(count).toBe(0);
});
});
describe('migrate', () => {
it('creates all tables', async () => {
// migrate() was already called in beforeEach — verify tables exist
const collections = [
'users',
'sessions',
'accounts',
'projects',
'missions',
'tasks',
'agents',
'conversations',
'messages',
'preferences',
'insights',
'skills',
'events',
'routing_rules',
'provider_credentials',
'agent_logs',
'teams',
'team_members',
'mission_tasks',
'tickets',
'summarization_jobs',
'appreciations',
'verifications',
];
for (const collection of collections) {
// Should not throw
const count = await adapter.count(collection);
expect(count).toBe(0);
}
});
it('is idempotent', async () => {
await adapter.migrate();
await adapter.migrate();
// Should not throw
const count = await adapter.count('users');
expect(count).toBe(0);
});
});
});
+290
View File
@@ -0,0 +1,290 @@
import { PGlite } from '@electric-sql/pglite';
import { randomUUID } from 'node:crypto';
import type { StorageAdapter, StorageConfig } from '../types.js';
/* eslint-disable @typescript-eslint/no-explicit-any */
const COLLECTIONS = [
'users',
'sessions',
'accounts',
'projects',
'missions',
'tasks',
'agents',
'conversations',
'messages',
'preferences',
'insights',
'skills',
'events',
'routing_rules',
'provider_credentials',
'agent_logs',
'teams',
'team_members',
'mission_tasks',
'tickets',
'summarization_jobs',
'appreciations',
'verifications',
] as const;
function buildFilterClause(filter?: Record<string, unknown>): {
clause: string;
params: unknown[];
} {
if (!filter || Object.keys(filter).length === 0) return { clause: '', params: [] };
const conditions: string[] = [];
const params: unknown[] = [];
let paramIdx = 1;
for (const [key, value] of Object.entries(filter)) {
if (key === 'id') {
conditions.push(`id = $${paramIdx.toString()}`);
params.push(value);
paramIdx++;
} else {
conditions.push(`data->>'${key}' = $${paramIdx.toString()}`);
params.push(typeof value === 'object' ? JSON.stringify(value) : value);
paramIdx++;
}
}
return { clause: ` WHERE ${conditions.join(' AND ')}`, params };
}
type PgClient = PGlite | { query: PGlite['query'] };
async function pgCreate<T extends Record<string, unknown>>(
pg: PgClient,
collection: string,
data: T,
): Promise<T & { id: string }> {
const id = (data as any).id ?? randomUUID();
const rest = Object.fromEntries(Object.entries(data).filter(([k]) => k !== 'id'));
await pg.query(`INSERT INTO ${collection} (id, data) VALUES ($1, $2::jsonb)`, [
id,
JSON.stringify(rest),
]);
return { ...data, id } as T & { id: string };
}
async function pgRead<T extends Record<string, unknown>>(
pg: PgClient,
collection: string,
id: string,
): Promise<T | null> {
const result = await pg.query<{ id: string; data: Record<string, unknown> }>(
`SELECT id, data FROM ${collection} WHERE id = $1`,
[id],
);
const row = result.rows[0];
if (!row) return null;
return { id: row.id, ...(row.data as object) } as unknown as T;
}
async function pgUpdate(
pg: PgClient,
collection: string,
id: string,
data: Record<string, unknown>,
): Promise<boolean> {
const existing = await pg.query<{ data: Record<string, unknown> }>(
`SELECT data FROM ${collection} WHERE id = $1`,
[id],
);
const row = existing.rows[0];
if (!row) return false;
const merged = { ...(row.data as object), ...data };
const result = await pg.query(
`UPDATE ${collection} SET data = $1::jsonb, updated_at = now() WHERE id = $2`,
[JSON.stringify(merged), id],
);
return (result.affectedRows ?? 0) > 0;
}
async function pgDelete(pg: PgClient, collection: string, id: string): Promise<boolean> {
const result = await pg.query(`DELETE FROM ${collection} WHERE id = $1`, [id]);
return (result.affectedRows ?? 0) > 0;
}
async function pgFind<T extends Record<string, unknown>>(
pg: PgClient,
collection: string,
filter?: Record<string, unknown>,
opts?: { limit?: number; offset?: number; orderBy?: string; order?: 'asc' | 'desc' },
): Promise<T[]> {
const { clause, params } = buildFilterClause(filter);
let paramIdx = params.length + 1;
let query = `SELECT id, data FROM ${collection}${clause}`;
if (opts?.orderBy) {
const dir = opts.order === 'desc' ? 'DESC' : 'ASC';
const col =
opts.orderBy === 'id'
? 'id'
: opts.orderBy === 'created_at' || opts.orderBy === 'updated_at'
? opts.orderBy
: `data->>'${opts.orderBy}'`;
query += ` ORDER BY ${col} ${dir}`;
}
if (opts?.limit !== undefined) {
query += ` LIMIT $${paramIdx.toString()}`;
params.push(opts.limit);
paramIdx++;
}
if (opts?.offset !== undefined) {
query += ` OFFSET $${paramIdx.toString()}`;
params.push(opts.offset);
paramIdx++;
}
const result = await pg.query<{ id: string; data: Record<string, unknown> }>(query, params);
return result.rows.map((row) => ({ id: row.id, ...(row.data as object) }) as unknown as T);
}
async function pgCount(
pg: PgClient,
collection: string,
filter?: Record<string, unknown>,
): Promise<number> {
const { clause, params } = buildFilterClause(filter);
const result = await pg.query<{ count: string }>(
`SELECT COUNT(*) as count FROM ${collection}${clause}`,
params,
);
return parseInt(result.rows[0]?.count ?? '0', 10);
}
export class PgliteAdapter implements StorageAdapter {
readonly name = 'pglite';
private pg: PGlite;
constructor(config: Extract<StorageConfig, { type: 'pglite' }>) {
this.pg = new PGlite(config.dataDir);
}
async create<T extends Record<string, unknown>>(
collection: string,
data: T,
): Promise<T & { id: string }> {
return pgCreate(this.pg, collection, data);
}
async read<T extends Record<string, unknown>>(collection: string, id: string): Promise<T | null> {
return pgRead(this.pg, collection, id);
}
async update(collection: string, id: string, data: Record<string, unknown>): Promise<boolean> {
return pgUpdate(this.pg, collection, id, data);
}
async delete(collection: string, id: string): Promise<boolean> {
return pgDelete(this.pg, collection, id);
}
async find<T extends Record<string, unknown>>(
collection: string,
filter?: Record<string, unknown>,
opts?: { limit?: number; offset?: number; orderBy?: string; order?: 'asc' | 'desc' },
): Promise<T[]> {
return pgFind(this.pg, collection, filter, opts);
}
async findOne<T extends Record<string, unknown>>(
collection: string,
filter: Record<string, unknown>,
): Promise<T | null> {
const results = await this.find<T>(collection, filter, { limit: 1 });
return results[0] ?? null;
}
async count(collection: string, filter?: Record<string, unknown>): Promise<number> {
return pgCount(this.pg, collection, filter);
}
async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
return this.pg.transaction(async (tx) => {
const txAdapter = new PgliteTxAdapter(tx as unknown as PgClient);
return fn(txAdapter);
});
}
async migrate(): Promise<void> {
for (const name of COLLECTIONS) {
await this.pg.query(`
CREATE TABLE IF NOT EXISTS ${name} (
id TEXT PRIMARY KEY,
data JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
`);
}
}
async close(): Promise<void> {
await this.pg.close();
}
}
/**
* Transaction wrapper that delegates to the PGlite transaction connection.
*/
class PgliteTxAdapter implements StorageAdapter {
readonly name = 'pglite';
private pg: PgClient;
constructor(pg: PgClient) {
this.pg = pg;
}
async create<T extends Record<string, unknown>>(
collection: string,
data: T,
): Promise<T & { id: string }> {
return pgCreate(this.pg, collection, data);
}
async read<T extends Record<string, unknown>>(collection: string, id: string): Promise<T | null> {
return pgRead(this.pg, collection, id);
}
async update(collection: string, id: string, data: Record<string, unknown>): Promise<boolean> {
return pgUpdate(this.pg, collection, id, data);
}
async delete(collection: string, id: string): Promise<boolean> {
return pgDelete(this.pg, collection, id);
}
async find<T extends Record<string, unknown>>(
collection: string,
filter?: Record<string, unknown>,
opts?: { limit?: number; offset?: number; orderBy?: string; order?: 'asc' | 'desc' },
): Promise<T[]> {
return pgFind(this.pg, collection, filter, opts);
}
async findOne<T extends Record<string, unknown>>(
collection: string,
filter: Record<string, unknown>,
): Promise<T | null> {
const results = await this.find<T>(collection, filter, { limit: 1 });
return results[0] ?? null;
}
async count(collection: string, filter?: Record<string, unknown>): Promise<number> {
return pgCount(this.pg, collection, filter);
}
async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
// Already inside a transaction — run directly
return fn(this);
}
async migrate(): Promise<void> {
// No-op inside transaction
}
async close(): Promise<void> {
// No-op inside transaction
}
}
@@ -0,0 +1,107 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import type { DbHandle } from '@mosaicstack/db';
// Mock @mosaicstack/db before importing the adapter
vi.mock('@mosaicstack/db', async (importOriginal) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const actual = await importOriginal<Record<string, any>>();
return {
...actual,
createDb: vi.fn(),
runMigrations: vi.fn().mockResolvedValue(undefined),
};
});
import { createDb, runMigrations } from '@mosaicstack/db';
import { PostgresAdapter } from './postgres.js';
describe('PostgresAdapter — vector extension gating', () => {
let mockExecute: ReturnType<typeof vi.fn>;
let mockDb: { execute: ReturnType<typeof vi.fn> };
let mockHandle: Pick<DbHandle, 'close'> & { db: typeof mockDb };
beforeEach(() => {
vi.clearAllMocks();
mockExecute = vi.fn().mockResolvedValue(undefined);
mockDb = { execute: mockExecute };
mockHandle = { db: mockDb, close: vi.fn().mockResolvedValue(undefined) };
vi.mocked(createDb).mockReturnValue(mockHandle as unknown as DbHandle);
});
it('calls db.execute with CREATE EXTENSION IF NOT EXISTS vector when enableVector=true', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: true,
});
await adapter.migrate();
// Should have called execute
expect(mockExecute).toHaveBeenCalledTimes(1);
// Verify the SQL contains the extension creation statement.
// Prefer Drizzle's public toSQL() API; fall back to queryChunks if unavailable.
// NOTE: queryChunks is an undocumented Drizzle internal (drizzle-orm ^0.45.x).
// toSQL() was not present on the raw sql`` result in this version — if a future
// Drizzle upgrade adds it, remove the fallback path and delete this comment.
const sqlObj = mockExecute.mock.calls[0]![0] as {
toSQL?: () => { sql: string; params: unknown[] };
queryChunks?: Array<{ value: string[] }>;
};
const sqlText = sqlObj.toSQL
? sqlObj.toSQL().sql.toLowerCase()
: (sqlObj.queryChunks ?? [])
.flatMap((chunk) => chunk.value)
.join('')
.toLowerCase();
expect(sqlText).toContain('create extension if not exists vector');
});
it('does NOT call db.execute for extension when enableVector is false', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: false,
});
await adapter.migrate();
expect(mockExecute).not.toHaveBeenCalled();
expect(vi.mocked(runMigrations)).toHaveBeenCalledOnce();
});
it('does NOT call db.execute for extension when enableVector is unset', async () => {
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
});
await adapter.migrate();
expect(mockExecute).not.toHaveBeenCalled();
expect(vi.mocked(runMigrations)).toHaveBeenCalledOnce();
});
it('calls runMigrations after the extension is created', async () => {
const callOrder: string[] = [];
mockExecute.mockImplementation(() => {
callOrder.push('execute');
return Promise.resolve(undefined);
});
vi.mocked(runMigrations).mockImplementation(() => {
callOrder.push('runMigrations');
return Promise.resolve();
});
const adapter = new PostgresAdapter({
type: 'postgres',
url: 'postgresql://test:test@localhost:5432/test',
enableVector: true,
});
await adapter.migrate();
expect(callOrder).toEqual(['execute', 'runMigrations']);
});
});
@@ -0,0 +1,261 @@
import {
createDb,
runMigrations,
eq,
and,
asc,
desc,
sql,
type Db,
type DbHandle,
} from '@mosaicstack/db';
import * as schema from '@mosaicstack/db';
import type { StorageAdapter, StorageConfig } from '../types.js';
/* eslint-disable @typescript-eslint/no-explicit-any */
/**
* Maps collection name → Drizzle table object.
* Typed as `any` because the generic StorageAdapter interface erases table
* types — all runtime values are still strongly-typed Drizzle table objects.
*/
const TABLE_MAP: Record<string, any> = {
users: schema.users,
sessions: schema.sessions,
accounts: schema.accounts,
verifications: schema.verifications,
teams: schema.teams,
team_members: schema.teamMembers,
projects: schema.projects,
missions: schema.missions,
tasks: schema.tasks,
mission_tasks: schema.missionTasks,
events: schema.events,
agents: schema.agents,
tickets: schema.tickets,
appreciations: schema.appreciations,
conversations: schema.conversations,
messages: schema.messages,
preferences: schema.preferences,
insights: schema.insights,
agent_logs: schema.agentLogs,
skills: schema.skills,
routing_rules: schema.routingRules,
provider_credentials: schema.providerCredentials,
summarization_jobs: schema.summarizationJobs,
};
function getTable(collection: string): any {
const table = TABLE_MAP[collection];
if (!table) throw new Error(`Unknown collection: ${collection}`);
return table;
}
function buildWhereClause(table: any, filter?: Record<string, unknown>) {
if (!filter || Object.keys(filter).length === 0) return undefined;
const conditions = Object.entries(filter).map(([key, value]) => {
const column = table[key];
if (!column) throw new Error(`Unknown column "${key}" on table`);
return eq(column, value);
});
return conditions.length === 1 ? conditions[0]! : and(...conditions);
}
export class PostgresAdapter implements StorageAdapter {
readonly name = 'postgres';
private handle: DbHandle;
private db: Db;
private url: string;
private enableVector: boolean;
constructor(config: Extract<StorageConfig, { type: 'postgres' }>) {
this.url = config.url;
this.enableVector = config.enableVector ?? false;
this.handle = createDb(config.url);
this.db = this.handle.db;
}
private async ensureVectorExtension(): Promise<void> {
await this.db.execute(sql`CREATE EXTENSION IF NOT EXISTS vector`);
}
async create<T extends Record<string, unknown>>(
collection: string,
data: T,
): Promise<T & { id: string }> {
const table = getTable(collection);
const [row] = await (this.db as any).insert(table).values(data).returning();
return row as T & { id: string };
}
async read<T extends Record<string, unknown>>(collection: string, id: string): Promise<T | null> {
const table = getTable(collection);
const [row] = await (this.db as any).select().from(table).where(eq(table.id, id));
return (row as T) ?? null;
}
async update(collection: string, id: string, data: Record<string, unknown>): Promise<boolean> {
const table = getTable(collection);
const result = await (this.db as any)
.update(table)
.set(data)
.where(eq(table.id, id))
.returning();
return result.length > 0;
}
async delete(collection: string, id: string): Promise<boolean> {
const table = getTable(collection);
const result = await (this.db as any).delete(table).where(eq(table.id, id)).returning();
return result.length > 0;
}
async find<T extends Record<string, unknown>>(
collection: string,
filter?: Record<string, unknown>,
opts?: { limit?: number; offset?: number; orderBy?: string; order?: 'asc' | 'desc' },
): Promise<T[]> {
const table = getTable(collection);
let query = (this.db as any).select().from(table);
const where = buildWhereClause(table, filter);
if (where) query = query.where(where);
if (opts?.orderBy) {
const col = table[opts.orderBy];
if (col) {
query = query.orderBy(opts.order === 'desc' ? desc(col) : asc(col));
}
}
if (opts?.limit) query = query.limit(opts.limit);
if (opts?.offset) query = query.offset(opts.offset);
return (await query) as T[];
}
async findOne<T extends Record<string, unknown>>(
collection: string,
filter: Record<string, unknown>,
): Promise<T | null> {
const results = await this.find<T>(collection, filter, { limit: 1 });
return results[0] ?? null;
}
async count(collection: string, filter?: Record<string, unknown>): Promise<number> {
const table = getTable(collection);
let query = (this.db as any).select({ count: sql<number>`count(*)::int` }).from(table);
const where = buildWhereClause(table, filter);
if (where) query = query.where(where);
const [row] = await query;
return (row as any)?.count ?? 0;
}
async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
return (this.db as any).transaction(async (drizzleTx: any) => {
const txAdapter = new PostgresTxAdapter(drizzleTx, this.url);
return fn(txAdapter);
});
}
async migrate(): Promise<void> {
if (this.enableVector) {
await this.ensureVectorExtension();
}
await runMigrations(this.url);
}
async close(): Promise<void> {
await this.handle.close();
}
}
/**
* Thin transaction wrapper — delegates to the Drizzle transaction object
* instead of the top-level db handle.
*/
class PostgresTxAdapter implements StorageAdapter {
readonly name = 'postgres';
private tx: any;
private url: string;
constructor(tx: any, url: string) {
this.tx = tx;
this.url = url;
}
async create<T extends Record<string, unknown>>(
collection: string,
data: T,
): Promise<T & { id: string }> {
const table = getTable(collection);
const [row] = await this.tx.insert(table).values(data).returning();
return row as T & { id: string };
}
async read<T extends Record<string, unknown>>(collection: string, id: string): Promise<T | null> {
const table = getTable(collection);
const [row] = await this.tx.select().from(table).where(eq(table.id, id));
return (row as T) ?? null;
}
async update(collection: string, id: string, data: Record<string, unknown>): Promise<boolean> {
const table = getTable(collection);
const result = await this.tx.update(table).set(data).where(eq(table.id, id)).returning();
return result.length > 0;
}
async delete(collection: string, id: string): Promise<boolean> {
const table = getTable(collection);
const result = await this.tx.delete(table).where(eq(table.id, id)).returning();
return result.length > 0;
}
async find<T extends Record<string, unknown>>(
collection: string,
filter?: Record<string, unknown>,
opts?: { limit?: number; offset?: number; orderBy?: string; order?: 'asc' | 'desc' },
): Promise<T[]> {
const table = getTable(collection);
let query = this.tx.select().from(table);
const where = buildWhereClause(table, filter);
if (where) query = query.where(where);
if (opts?.orderBy) {
const col = table[opts.orderBy];
if (col) {
query = query.orderBy(opts.order === 'desc' ? desc(col) : asc(col));
}
}
if (opts?.limit) query = query.limit(opts.limit);
if (opts?.offset) query = query.offset(opts.offset);
return (await query) as T[];
}
async findOne<T extends Record<string, unknown>>(
collection: string,
filter: Record<string, unknown>,
): Promise<T | null> {
const results = await this.find<T>(collection, filter, { limit: 1 });
return results[0] ?? null;
}
async count(collection: string, filter?: Record<string, unknown>): Promise<number> {
const table = getTable(collection);
let query = this.tx.select({ count: sql<number>`count(*)::int` }).from(table);
const where = buildWhereClause(table, filter);
if (where) query = query.where(where);
const [row] = await query;
return (row as any)?.count ?? 0;
}
async transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T> {
return this.tx.transaction(async (nestedTx: any) => {
const nested = new PostgresTxAdapter(nestedTx, this.url);
return fn(nested);
});
}
async migrate(): Promise<void> {
await runMigrations(this.url);
}
async close(): Promise<void> {
// No-op inside a transaction
}
}
+85
View File
@@ -0,0 +1,85 @@
import { describe, it, expect } from 'vitest';
import { Command } from 'commander';
import { registerStorageCommand } from './cli.js';
describe('registerStorageCommand', () => {
function buildProgram(): Command {
const program = new Command();
program.exitOverride(); // prevent process.exit in tests
registerStorageCommand(program);
return program;
}
it('registers a "storage" command on the parent', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage');
expect(storageCmd).toBeDefined();
});
it('registers "storage status" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const statusCmd = storageCmd.commands.find((c) => c.name() === 'status');
expect(statusCmd).toBeDefined();
});
it('registers "storage tier" subcommand group', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const tierCmd = storageCmd.commands.find((c) => c.name() === 'tier');
expect(tierCmd).toBeDefined();
});
it('registers "storage tier show" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const tierCmd = storageCmd.commands.find((c) => c.name() === 'tier')!;
const showCmd = tierCmd.commands.find((c) => c.name() === 'show');
expect(showCmd).toBeDefined();
});
it('registers "storage tier switch" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const tierCmd = storageCmd.commands.find((c) => c.name() === 'tier')!;
const switchCmd = tierCmd.commands.find((c) => c.name() === 'switch');
expect(switchCmd).toBeDefined();
});
it('registers "storage export" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const exportCmd = storageCmd.commands.find((c) => c.name() === 'export');
expect(exportCmd).toBeDefined();
});
it('registers "storage import" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const importCmd = storageCmd.commands.find((c) => c.name() === 'import');
expect(importCmd).toBeDefined();
});
it('registers "storage migrate" subcommand', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const migrateCmd = storageCmd.commands.find((c) => c.name() === 'migrate');
expect(migrateCmd).toBeDefined();
});
it('has all required subcommands in a single assertion', () => {
const program = buildProgram();
const storageCmd = program.commands.find((c) => c.name() === 'storage')!;
const topLevel = storageCmd.commands.map((c) => c.name());
expect(topLevel).toContain('status');
expect(topLevel).toContain('tier');
expect(topLevel).toContain('export');
expect(topLevel).toContain('import');
expect(topLevel).toContain('migrate');
const tierCmd = storageCmd.commands.find((c) => c.name() === 'tier')!;
const tierSubcmds = tierCmd.commands.map((c) => c.name());
expect(tierSubcmds).toContain('show');
expect(tierSubcmds).toContain('switch');
});
});
+455
View File
@@ -0,0 +1,455 @@
import type { Command } from 'commander';
import type { MigrationSource } from './migrate-tier.js';
import { redactErrMsg } from './redact-error.js';
/**
* Reads the DATABASE_URL environment variable and redacts the password portion.
*/
function redactedConnectionString(): string | null {
const url = process.env['DATABASE_URL'];
if (!url) return null;
try {
const parsed = new URL(url);
if (parsed.password) {
parsed.password = '***';
}
return parsed.toString();
} catch {
// Not a valid URL — redact anything that looks like :password@
return url.replace(/:([^@/]+)@/, ':***@');
}
}
/**
* Determine the active storage tier from the environment.
* Looks at DATABASE_URL; if absent or set to a pglite path, treats tier as pglite.
*/
function activeTier(): 'postgres' | 'pglite' {
const url = process.env['DATABASE_URL'];
if (url && url.startsWith('postgres')) return 'postgres';
return 'pglite';
}
/**
* Return a human-readable config source description.
*/
function configSource(): string {
if (process.env['DATABASE_URL']) return 'env:DATABASE_URL';
const pgliteDir = process.env['PGLITE_DATA_DIR'];
if (pgliteDir) return `env:PGLITE_DATA_DIR (${pgliteDir})`;
return 'default (no DATABASE_URL set)';
}
/**
* Register storage subcommands on an existing Commander program.
* Follows the registerQualityRails pattern — uses the caller's Command
* instance to avoid cross-package Commander version mismatches.
*/
export function registerStorageCommand(parent: Command): void {
const storage = parent
.command('storage')
.description('Inspect and manage Mosaic storage configuration');
// ── storage status ───────────────────────────────────────────────────────
storage
.command('status')
.description('Show the configured storage tier and whether the adapter is reachable')
.action(async () => {
const tier = activeTier();
const source = configSource();
const connStr = tier === 'postgres' ? redactedConnectionString() : null;
console.log(`[storage] tier: ${tier}`);
console.log(`[storage] config source: ${source}`);
if (tier === 'postgres' && connStr) {
console.log(`[storage] connection: ${connStr}`);
try {
const { createDb, sql } = await import('@mosaicstack/db');
const url = process.env['DATABASE_URL'] ?? '';
const handle = createDb(url);
await handle.db.execute(sql`SELECT 1`);
await handle.close();
console.log('[storage] reachable: yes');
} catch (err) {
console.log(
`[storage] reachable: no (${redactErrMsg(err instanceof Error ? err.message : String(err))})`,
);
}
} else {
const dataDir = process.env['PGLITE_DATA_DIR'] ?? ':memory:';
console.log(`[storage] data dir: ${dataDir}`);
console.log('[storage] reachable: pglite is always local — no network check needed');
}
});
// ── storage tier ─────────────────────────────────────────────────────────
const tier = storage.command('tier').description('Inspect or switch the storage tier');
tier
.command('show')
.description('Print the active storage tier and its config source')
.action(() => {
const activeTierValue = activeTier();
const source = configSource();
console.log(`[storage] active tier: ${activeTierValue}`);
console.log(`[storage] config source: ${source}`);
});
tier
.command('switch <tier>')
.description('Switch storage tier between pglite and postgres')
.action((newTier: string) => {
const validTiers = ['pglite', 'postgres'];
if (!validTiers.includes(newTier)) {
console.error(
`[storage] unknown tier: ${newTier}. Valid options: ${validTiers.join(', ')}`,
);
process.exitCode = 1;
return;
}
console.log(`[storage] tier switch requested: ${newTier}`);
console.log('');
console.log('Mosaic storage tier is controlled by environment variables.');
console.log('Automatic config-file mutation is not supported — set the variable manually.');
console.log('');
if (newTier === 'postgres') {
console.log('To switch to postgres:');
console.log(' 1. Set DATABASE_URL in your environment or .env file:');
console.log(' export DATABASE_URL="postgresql://user:pass@localhost:5432/mosaic"');
console.log(' 2. Run migrations:');
console.log(' pnpm --filter @mosaicstack/db db:migrate');
console.log(' 3. Restart the gateway.');
} else {
console.log('To switch to pglite:');
console.log(' 1. Unset DATABASE_URL (or set it to a pglite path):');
console.log(' unset DATABASE_URL');
console.log(' # optionally: export PGLITE_DATA_DIR=/path/to/pglite/data');
console.log(' 2. Restart the gateway.');
console.log(' Note: pglite uses an in-process database — no migrations needed.');
}
});
// ── storage export ───────────────────────────────────────────────────────
storage
.command('export <path>')
.description('Dump the active storage contents to a file')
.action((outputPath: string) => {
const currentTier = activeTier();
if (currentTier === 'postgres') {
const redacted = redactedConnectionString() ?? '<DATABASE_URL>';
console.log('[storage] export for postgres tier');
console.log('');
console.log('postgres export is not yet wired in the CLI — use pg_dump directly:');
console.log('');
console.log(` pg_dump "${redacted}" > ${outputPath}`);
console.log('');
console.log('Or with Docker:');
console.log(
` docker exec <postgres-container> pg_dump -U <user> <dbname> > ${outputPath}`,
);
process.exitCode = 0;
} else {
const dataDir = process.env['PGLITE_DATA_DIR'];
console.log('[storage] export for pglite tier');
console.log('');
console.log(
'pglite export is not yet wired in the CLI — copy the data directory directly:',
);
console.log('');
if (dataDir) {
console.log(` cp -r ${dataDir} ${outputPath}`);
} else {
console.log(
' PGLITE_DATA_DIR is not set; the database is in-memory and cannot be exported.',
);
console.log(' Set PGLITE_DATA_DIR to a persistent path before running export.');
}
process.exitCode = 0;
}
});
// ── storage import ───────────────────────────────────────────────────────
storage
.command('import <path>')
.description('Restore storage contents from a previously exported file')
.action((inputPath: string) => {
const currentTier = activeTier();
if (currentTier === 'postgres') {
const redacted = redactedConnectionString() ?? '<DATABASE_URL>';
console.log('[storage] import for postgres tier');
console.log('');
console.log('postgres import is not yet wired in the CLI — use psql directly:');
console.log('');
console.log(` psql "${redacted}" < ${inputPath}`);
process.exitCode = 0;
} else {
const dataDir = process.env['PGLITE_DATA_DIR'];
console.log('[storage] import for pglite tier');
console.log('');
console.log(
'pglite import is not yet wired in the CLI — restore the data directory directly:',
);
console.log('');
if (dataDir) {
console.log(` rm -rf ${dataDir} && cp -r ${inputPath} ${dataDir}`);
console.log(' Then restart the gateway.');
} else {
console.log(
' PGLITE_DATA_DIR is not set; set it to a persistent path before running import.',
);
}
process.exitCode = 0;
}
});
// ── storage migrate-tier ─────────────────────────────────────────────────
storage
.command('migrate-tier')
.description('Migrate data from tier: local/standalone → tier: federated (Postgres + pgvector)')
.requiredOption(
'--to <tier>',
'Target tier to migrate to (only "federated" is supported)',
'federated',
)
.requiredOption('--target-url <url>', 'Target federated Postgres connection string (required)')
.option(
'--source-config <path>',
'Path to mosaic.config.json (default: cwd/mosaic.config.json)',
)
.option('--dry-run', 'Print what would be migrated without writing anything')
.option('--yes', 'Skip interactive confirmation prompt (required for non-TTY environments)')
.option('--batch-size <n>', 'Rows per transaction batch', '1000')
.option('--allow-non-empty', 'Allow writing to a non-empty target (upsert — idempotent)')
.action(
async (opts: {
to: string;
targetUrl: string;
sourceConfig?: string;
dryRun?: boolean;
yes?: boolean;
batchSize?: string;
allowNonEmpty?: boolean;
}) => {
if (opts.to !== 'federated') {
console.error(
`[migrate-tier] --to "${opts.to}" is not supported. Only "federated" is allowed.`,
);
process.exitCode = 1;
return;
}
const batchSize = parseInt(opts.batchSize ?? '1000', 10);
if (isNaN(batchSize) || batchSize < 1) {
console.error('[migrate-tier] --batch-size must be a positive integer.');
process.exitCode = 1;
return;
}
// Redact target URL password for display.
function redactUrl(url: string): string {
try {
const parsed = new URL(url);
if (parsed.password) parsed.password = '***';
return parsed.toString();
} catch {
return url.replace(/:([^@/]+)@/, ':***@');
}
}
const redactedTarget = redactUrl(opts.targetUrl);
const isDryRun = opts.dryRun ?? false;
const allowNonEmpty = opts.allowNonEmpty ?? false;
// Determine source tier from environment.
const sourceTier = activeTier();
const sourceDesc = configSource();
console.log('');
console.log('[migrate-tier] ─────────────────────────────────────────');
console.log(`[migrate-tier] Source tier: ${sourceTier}`);
console.log(`[migrate-tier] Source: ${sourceDesc}`);
console.log(`[migrate-tier] Target tier: federated (Postgres + pgvector)`);
console.log(`[migrate-tier] Target: ${redactedTarget}`);
console.log(`[migrate-tier] Batch size: ${batchSize.toString()}`);
console.log(`[migrate-tier] Dry run: ${isDryRun.toString()}`);
console.log(`[migrate-tier] Allow non-empty: ${allowNonEmpty.toString()}`);
console.log('[migrate-tier] ─────────────────────────────────────────');
console.log('');
// Lazy-import core migration logic to keep the CLI thin.
const {
runMigrateTier,
PostgresMigrationTarget,
DrizzleMigrationSource,
getMigrationOrder,
} = await import('./migrate-tier.js');
// Build source adapter using Drizzle-backed DrizzleMigrationSource.
// Both local (PGlite) and standalone (Postgres) sources expose the same
// normalized Drizzle schema — this is where the actual domain data lives.
let sourceAdapter: MigrationSource;
if (sourceTier === 'pglite') {
const { createPgliteDb } = await import('@mosaicstack/db');
const pgliteDataDir = process.env['PGLITE_DATA_DIR'];
if (!pgliteDataDir) {
console.error(
'[migrate-tier] PGLITE_DATA_DIR is not set. ' +
'Cannot open PGlite source — set it to the data directory path.',
);
process.exitCode = 1;
return;
}
const handle = createPgliteDb(pgliteDataDir);
// Local/PGlite sources do not have pgvector registered — the embedding
// column is omitted from the insights SELECT and set to null on target.
sourceAdapter = new DrizzleMigrationSource(handle.db, /* sourceHasVector= */ false);
} else {
const { createDb } = await import('@mosaicstack/db');
const url = process.env['DATABASE_URL'];
if (!url) {
console.error('[migrate-tier] DATABASE_URL is not set for postgres source.');
process.exitCode = 1;
return;
}
const handle = createDb(url);
// Standalone Postgres may or may not have pgvector — assume it does not
// (it is a non-federated tier) so embedding is treated as null.
sourceAdapter = new DrizzleMigrationSource(handle.db, /* sourceHasVector= */ false);
}
// Print per-table row counts for the confirmation prompt.
const tablesToMigrate = getMigrationOrder();
const counts: Array<{ table: string; count: number }> = [];
for (const table of tablesToMigrate) {
const n = await sourceAdapter.count(table);
counts.push({ table, count: n });
}
console.log('[migrate-tier] Source row counts:');
for (const { table, count } of counts) {
console.log(` ${table}: ${count.toString()}`);
}
console.log(' sessions: SKIPPED (ephemeral)');
console.log(' verifications: SKIPPED (ephemeral)');
console.log(' admin_tokens: SKIPPED (environment-specific)');
console.log('');
// Interactive confirmation unless --yes or dry-run.
const isTTY = process.stdin.isTTY;
if (!isDryRun) {
if (!opts.yes && !isTTY) {
console.error(
'[migrate-tier] Not running in a TTY and --yes was not passed. ' +
'Pass --yes to confirm in headless environments.',
);
process.exitCode = 1;
await sourceAdapter.close();
return;
}
if (!opts.yes) {
const { createInterface } = await import('node:readline');
const rl = createInterface({ input: process.stdin, output: process.stdout });
const answer = await new Promise<string>((resolve) => {
rl.question(`This will WRITE to ${redactedTarget}. Continue? [y/N] `, (ans) => {
rl.close();
resolve(ans);
});
});
if (answer.trim().toLowerCase() !== 'y') {
console.log('[migrate-tier] Aborted.');
await sourceAdapter.close();
return;
}
}
}
const target = new PostgresMigrationTarget(opts.targetUrl);
try {
const result = await runMigrateTier(
sourceAdapter,
target,
{
targetUrl: opts.targetUrl,
dryRun: isDryRun,
allowNonEmpty,
batchSize,
onProgress: (msg) => console.log(msg),
},
/* sourceHasVector= */ sourceTier === 'postgres',
);
if (result.dryRun) {
console.log('[migrate-tier] Dry run complete. No data was written.');
} else {
console.log(
`[migrate-tier] Migration complete. ${result.totalRows.toString()} rows migrated.`,
);
}
} catch (err) {
console.error(
`[migrate-tier] ERROR: ${redactErrMsg(err instanceof Error ? err.message : String(err))}`,
);
process.exitCode = 1;
} finally {
await Promise.all([sourceAdapter.close(), target.close()]);
}
},
);
// ── storage migrate ──────────────────────────────────────────────────────
storage
.command('migrate')
.description(
'Run database migrations (thin wrapper — delegates to pnpm db:migrate or prints the command)',
)
.option('--run', 'Actually execute the migration command via shell')
.action(async (opts: { run?: boolean }) => {
const currentTier = activeTier();
if (currentTier === 'pglite') {
console.log('[storage] pglite tier detected');
console.log(
'pglite runs schema setup automatically on first connection via adapter.migrate().',
);
console.log('No separate migration step is required.');
return;
}
const migrateCmd = 'pnpm --filter @mosaicstack/db db:migrate';
console.log('[storage] postgres tier detected');
console.log(`Migration command: ${migrateCmd}`);
console.log('');
if (opts.run) {
console.log('Running migrations...');
const { execSync } = await import('node:child_process');
try {
execSync(migrateCmd, { stdio: 'inherit' });
console.log('[storage] migrations complete.');
} catch (err) {
console.error(
`[storage] migration failed: ${err instanceof Error ? err.message : String(err)}`,
);
process.exitCode = 1;
}
} else {
console.log('To run migrations, execute:');
console.log(` ${migrateCmd}`);
console.log('');
console.log('Or pass --run to have this command execute it for you.');
}
});
}
+18
View File
@@ -0,0 +1,18 @@
import type { StorageAdapter, StorageConfig } from './types.js';
type StorageType = StorageConfig['type'];
const registry = new Map<StorageType, (config: StorageConfig) => StorageAdapter>();
export function registerStorageAdapter(
type: StorageType,
factory: (config: StorageConfig) => StorageAdapter,
): void {
registry.set(type, factory);
}
export function createStorageAdapter(config: StorageConfig): StorageAdapter {
const factory = registry.get(config.type);
if (!factory) throw new Error(`No adapter registered for type: ${config.type}`);
return factory(config);
}
+39
View File
@@ -0,0 +1,39 @@
export type { StorageAdapter, StorageConfig } from './types.js';
export { TierDetectionError, detectAndAssertTier, probeServiceHealth } from './tier-detection.js';
export type { ServiceCheck, TierHealthReport } from './tier-detection.js';
export { createStorageAdapter, registerStorageAdapter } from './factory.js';
export { PostgresAdapter } from './adapters/postgres.js';
export { PgliteAdapter } from './adapters/pglite.js';
export { registerStorageCommand } from './cli.js';
export {
getMigrationOrder,
topoSort,
runMigrateTier,
checkTargetPreconditions,
normaliseSourceRow,
PostgresMigrationTarget,
DrizzleMigrationSource,
SKIP_TABLES,
MIGRATION_ORDER,
MigrationPreconditionError,
} from './migrate-tier.js';
export type {
MigrationSource,
MigrationTarget,
MigrateTierOptions,
MigrateTierResult,
TableMigrationResult,
} from './migrate-tier.js';
import { registerStorageAdapter } from './factory.js';
import { PostgresAdapter } from './adapters/postgres.js';
import { PgliteAdapter } from './adapters/pglite.js';
import type { StorageConfig } from './types.js';
registerStorageAdapter('postgres', (config: StorageConfig) => {
return new PostgresAdapter(config as Extract<StorageConfig, { type: 'postgres' }>);
});
registerStorageAdapter('pglite', (config: StorageConfig) => {
return new PgliteAdapter(config as Extract<StorageConfig, { type: 'pglite' }>);
});
@@ -0,0 +1,328 @@
/**
* FED-M1-08 — Integration test: PGlite → federated Postgres+pgvector migration.
*
* Prereq: docker compose -f docker-compose.federated.yml --profile federated up -d
* Run: FEDERATED_INTEGRATION=1 pnpm --filter @mosaicstack/storage test src/migrate-tier.integration.test.ts
*
* Skipped when FEDERATED_INTEGRATION !== '1'.
*
* Strategy: users.id (TEXT PK) uses the recognisable prefix `fed-m1-08-` for
* easy cleanup. UUID-PKed tables (teams, conversations, messages, team_members)
* use deterministic valid UUIDs in the `f0000xxx-…` namespace. Cleanup is
* explicit DELETE by id — no full-table truncation.
*/
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
users,
teams,
teamMembers,
conversations,
messages,
createPgliteDb,
runPgliteMigrations,
} from '@mosaicstack/db';
import postgres from 'postgres';
import { afterAll, describe, expect, it } from 'vitest';
import { DrizzleMigrationSource, PostgresMigrationTarget, runMigrateTier } from './migrate-tier.js';
/* ------------------------------------------------------------------ */
/* Constants */
/* ------------------------------------------------------------------ */
const run = process.env['FEDERATED_INTEGRATION'] === '1';
const FEDERATED_PG_URL = 'postgresql://mosaic:mosaic@localhost:5433/mosaic';
/**
* Deterministic IDs for the test's seed data.
*
* users.id is TEXT (any string) — we use a recognisable prefix for easy cleanup.
* All other tables use UUID primary keys — must be valid UUID v4 format.
* The 4th segment starts with '4' (version 4) and 5th starts with '8' (variant).
*/
const IDS = {
// text PK — can be any string
user1: 'fed-m1-08-user-1',
user2: 'fed-m1-08-user-2',
// UUID PKs — must be valid UUID format
team1: 'f0000001-0000-4000-8000-000000000001',
teamMember1: 'f0000002-0000-4000-8000-000000000001',
teamMember2: 'f0000002-0000-4000-8000-000000000002',
conv1: 'f0000003-0000-4000-8000-000000000001',
conv2: 'f0000003-0000-4000-8000-000000000002',
msg1: 'f0000004-0000-4000-8000-000000000001',
msg2: 'f0000004-0000-4000-8000-000000000002',
msg3: 'f0000004-0000-4000-8000-000000000003',
msg4: 'f0000004-0000-4000-8000-000000000004',
msg5: 'f0000004-0000-4000-8000-000000000005',
} as const;
/* ------------------------------------------------------------------ */
/* Shared handles for afterAll cleanup */
/* ------------------------------------------------------------------ */
let targetSql: ReturnType<typeof postgres> | undefined;
let pgliteDataDir: string | undefined;
afterAll(async () => {
if (targetSql) {
await cleanTarget(targetSql).catch(() => {});
await targetSql.end({ timeout: 5 }).catch(() => {});
}
if (pgliteDataDir) {
await fs.rm(pgliteDataDir, { recursive: true, force: true }).catch(() => {});
}
});
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
/** Delete all test-owned rows from target in safe FK order. */
async function cleanTarget(sql: ReturnType<typeof postgres>): Promise<void> {
// Reverse FK order: messages → conversations → team_members → teams → users
await sql.unsafe(`DELETE FROM messages WHERE id = ANY($1)`, [
[IDS.msg1, IDS.msg2, IDS.msg3, IDS.msg4, IDS.msg5],
] as never[]);
await sql.unsafe(`DELETE FROM conversations WHERE id = ANY($1)`, [
[IDS.conv1, IDS.conv2],
] as never[]);
await sql.unsafe(`DELETE FROM team_members WHERE id = ANY($1)`, [
[IDS.teamMember1, IDS.teamMember2],
] as never[]);
await sql.unsafe(`DELETE FROM teams WHERE id = $1`, [IDS.team1] as never[]);
await sql.unsafe(`DELETE FROM users WHERE id = ANY($1)`, [[IDS.user1, IDS.user2]] as never[]);
}
/* ------------------------------------------------------------------ */
/* Test suite */
/* ------------------------------------------------------------------ */
describe.skipIf(!run)('migrate-tier — PGlite → federated PG', () => {
it('seeds PGlite, runs migrate-tier, asserts row counts and sample rows on target', async () => {
/* ---- 1. Create a temp PGlite db ---------------------------------- */
pgliteDataDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fed-m1-08-'));
const handle = createPgliteDb(pgliteDataDir);
await runPgliteMigrations(handle);
/* ---- 2. Seed representative data --------------------------------- */
const now = new Date();
const db = handle.db;
// users (2 rows)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (db as any).insert(users).values([
{
id: IDS.user1,
name: 'Fed Test User One',
email: '[email protected]',
emailVerified: false,
role: 'member',
createdAt: now,
updatedAt: now,
},
{
id: IDS.user2,
name: 'Fed Test User Two',
email: '[email protected]',
emailVerified: false,
role: 'member',
createdAt: now,
updatedAt: now,
},
]);
// teams (1 row)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (db as any).insert(teams).values([
{
id: IDS.team1,
name: 'Fed M1-08 Team',
slug: 'fed-m1-08-team',
ownerId: IDS.user1,
managerId: IDS.user1,
createdAt: now,
updatedAt: now,
},
]);
// team_members (2 rows linking both users to the team)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (db as any).insert(teamMembers).values([
{
id: IDS.teamMember1,
teamId: IDS.team1,
userId: IDS.user1,
role: 'manager',
joinedAt: now,
},
{
id: IDS.teamMember2,
teamId: IDS.team1,
userId: IDS.user2,
role: 'member',
joinedAt: now,
},
]);
// conversations (2 rows)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (db as any).insert(conversations).values([
{
id: IDS.conv1,
title: 'Fed M1-08 Conversation Alpha',
userId: IDS.user1,
archived: false,
createdAt: now,
updatedAt: now,
},
{
id: IDS.conv2,
title: 'Fed M1-08 Conversation Beta',
userId: IDS.user2,
archived: false,
createdAt: now,
updatedAt: now,
},
]);
// messages (5 rows across both conversations)
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await (db as any).insert(messages).values([
{
id: IDS.msg1,
conversationId: IDS.conv1,
role: 'user',
content: 'Hello from conv1 msg1',
createdAt: now,
},
{
id: IDS.msg2,
conversationId: IDS.conv1,
role: 'assistant',
content: 'Reply in conv1 msg2',
createdAt: now,
},
{
id: IDS.msg3,
conversationId: IDS.conv1,
role: 'user',
content: 'Follow-up in conv1 msg3',
createdAt: now,
},
{
id: IDS.msg4,
conversationId: IDS.conv2,
role: 'user',
content: 'Hello from conv2 msg4',
createdAt: now,
},
{
id: IDS.msg5,
conversationId: IDS.conv2,
role: 'assistant',
content: 'Reply in conv2 msg5',
createdAt: now,
},
]);
/* ---- 3. Pre-clean the target so the test is repeatable ----------- */
targetSql = postgres(FEDERATED_PG_URL, {
max: 3,
connect_timeout: 10,
idle_timeout: 30,
});
await cleanTarget(targetSql);
/* ---- 4. Build source / target adapters and run migration --------- */
const source = new DrizzleMigrationSource(db, /* sourceHasVector= */ false);
const target = new PostgresMigrationTarget(FEDERATED_PG_URL);
try {
await runMigrateTier(
source,
target,
{
targetUrl: FEDERATED_PG_URL,
dryRun: false,
allowNonEmpty: true,
batchSize: 500,
onProgress: (_msg) => {
// Uncomment for debugging: console.log(_msg);
},
},
/* sourceHasVector= */ false,
);
} finally {
await target.close();
}
/* ---- 5. Assert: row counts in target match seed ------------------ */
const countUsers = await targetSql.unsafe<Array<{ n: string }>>(
`SELECT COUNT(*)::text AS n FROM users WHERE id = ANY($1)`,
[[IDS.user1, IDS.user2]] as never[],
);
expect(Number(countUsers[0]?.n)).toBe(2);
const countTeams = await targetSql.unsafe<Array<{ n: string }>>(
`SELECT COUNT(*)::text AS n FROM teams WHERE id = $1`,
[IDS.team1] as never[],
);
expect(Number(countTeams[0]?.n)).toBe(1);
const countTeamMembers = await targetSql.unsafe<Array<{ n: string }>>(
`SELECT COUNT(*)::text AS n FROM team_members WHERE id = ANY($1)`,
[[IDS.teamMember1, IDS.teamMember2]] as never[],
);
expect(Number(countTeamMembers[0]?.n)).toBe(2);
const countConvs = await targetSql.unsafe<Array<{ n: string }>>(
`SELECT COUNT(*)::text AS n FROM conversations WHERE id = ANY($1)`,
[[IDS.conv1, IDS.conv2]] as never[],
);
expect(Number(countConvs[0]?.n)).toBe(2);
const countMsgs = await targetSql.unsafe<Array<{ n: string }>>(
`SELECT COUNT(*)::text AS n FROM messages WHERE id = ANY($1)`,
[[IDS.msg1, IDS.msg2, IDS.msg3, IDS.msg4, IDS.msg5]] as never[],
);
expect(Number(countMsgs[0]?.n)).toBe(5);
/* ---- 6. Assert: sample row field values --------------------------- */
// User 1: check email and name
const userRows = await targetSql.unsafe<Array<{ id: string; email: string; name: string }>>(
`SELECT id, email, name FROM users WHERE id = $1`,
[IDS.user1] as never[],
);
expect(userRows[0]?.email).toBe('[email protected]');
expect(userRows[0]?.name).toBe('Fed Test User One');
// Conversation 1: check title and user_id
const convRows = await targetSql.unsafe<Array<{ id: string; title: string; user_id: string }>>(
`SELECT id, title, user_id FROM conversations WHERE id = $1`,
[IDS.conv1] as never[],
);
expect(convRows[0]?.title).toBe('Fed M1-08 Conversation Alpha');
expect(convRows[0]?.user_id).toBe(IDS.user1);
/* ---- 7. Cleanup: delete test rows from target -------------------- */
await cleanTarget(targetSql);
// Close PGlite
await handle.close();
}, 60_000);
});
@@ -0,0 +1,495 @@
/**
* migrate-tier.spec.ts — Unit tests for the migrate-tier core logic.
*
* These are pure unit tests — no real database connections.
* FED-M1-08 will add integration tests against real services.
*/
import { describe, it, expect, vi } from 'vitest';
import {
getMigrationOrder,
topoSort,
runMigrateTier,
checkTargetPreconditions,
normaliseSourceRow,
SKIP_TABLES,
MigrationPreconditionError,
type MigrationSource,
type MigrationTarget,
} from './migrate-tier.js';
/* ------------------------------------------------------------------ */
/* Mock factories */
/* ------------------------------------------------------------------ */
/**
* Build a mock MigrationSource backed by an in-memory table map.
* Implements the DrizzleMigrationSource-shaped contract:
* - readTable(tableName, opts?) returns paginated rows
* - count(tableName) returns row count
*
* The `sourceHasVector` flag controls whether the mock simulates the
* no-pgvector projection: when false and tableName is 'insights', rows
* are returned WITHOUT the 'embedding' field (matching DrizzleMigrationSource
* behaviour for local/PGlite sources).
*/
function makeMockSource(
data: Record<string, Record<string, unknown>[]>,
sourceHasVector = true,
): MigrationSource & {
readTableCalls: Array<{ table: string; opts?: { limit?: number; offset?: number } }>;
} {
const readTableCalls: Array<{ table: string; opts?: { limit?: number; offset?: number } }> = [];
return {
readTableCalls,
readTable: vi.fn(async (tableName: string, opts?: { limit?: number; offset?: number }) => {
readTableCalls.push({ table: tableName, opts });
let rows = data[tableName] ?? [];
// Simulate no-vector projection: omit 'embedding' from insights rows
// when sourceHasVector is false (matches DrizzleMigrationSource behaviour).
if (tableName === 'insights' && !sourceHasVector) {
rows = rows.map(({ embedding: _omit, ...rest }) => rest);
}
const offset = opts?.offset ?? 0;
const limit = opts?.limit ?? rows.length;
return rows.slice(offset, offset + limit);
}),
count: vi.fn(async (tableName: string) => (data[tableName] ?? []).length),
close: vi.fn(async () => undefined),
};
}
function makeMockTarget(opts?: {
hasPgvector?: boolean;
nonEmptyTable?: string;
}): MigrationTarget & { upsertCalls: Array<{ table: string; rows: Record<string, unknown>[] }> } {
const upsertCalls: Array<{ table: string; rows: Record<string, unknown>[] }> = [];
const storedCounts: Record<string, number> = {};
return {
upsertCalls,
upsertBatch: vi.fn(async (table: string, rows: Record<string, unknown>[]) => {
upsertCalls.push({ table, rows });
storedCounts[table] = (storedCounts[table] ?? 0) + rows.length;
}),
count: vi.fn(async (table: string) => {
if (opts?.nonEmptyTable === table) return 5;
return storedCounts[table] ?? 0;
}),
hasPgvector: vi.fn(async () => opts?.hasPgvector ?? true),
close: vi.fn(async () => undefined),
};
}
function noopProgress(): (msg: string) => void {
return () => undefined;
}
/* ------------------------------------------------------------------ */
/* 1. Topological ordering */
/* ------------------------------------------------------------------ */
describe('topoSort', () => {
it('returns empty array for empty input', () => {
expect(topoSort(new Map())).toEqual([]);
});
it('orders parents before children — linear chain', () => {
// users -> teams -> messages
const deps = new Map([
['users', []],
['teams', ['users']],
['messages', ['teams']],
]);
const order = topoSort(deps);
expect(order.indexOf('users')).toBeLessThan(order.indexOf('teams'));
expect(order.indexOf('teams')).toBeLessThan(order.indexOf('messages'));
});
it('orders parents before children — diamond graph', () => {
// a -> (b, c) -> d
const deps = new Map([
['a', []],
['b', ['a']],
['c', ['a']],
['d', ['b', 'c']],
]);
const order = topoSort(deps);
expect(order.indexOf('a')).toBeLessThan(order.indexOf('b'));
expect(order.indexOf('a')).toBeLessThan(order.indexOf('c'));
expect(order.indexOf('b')).toBeLessThan(order.indexOf('d'));
expect(order.indexOf('c')).toBeLessThan(order.indexOf('d'));
});
it('throws on cyclic dependencies', () => {
const deps = new Map([
['a', ['b']],
['b', ['a']],
]);
expect(() => topoSort(deps)).toThrow('Cycle detected');
});
});
/* ------------------------------------------------------------------ */
/* 2. getMigrationOrder — sessions / verifications excluded */
/* ------------------------------------------------------------------ */
describe('getMigrationOrder', () => {
it('does not include "sessions"', () => {
expect(getMigrationOrder()).not.toContain('sessions');
});
it('does not include "verifications"', () => {
expect(getMigrationOrder()).not.toContain('verifications');
});
it('does not include "admin_tokens"', () => {
expect(getMigrationOrder()).not.toContain('admin_tokens');
});
it('includes "users" before "teams"', () => {
const order = getMigrationOrder();
expect(order.indexOf('users')).toBeLessThan(order.indexOf('teams'));
});
it('includes "users" before "conversations"', () => {
const order = getMigrationOrder();
expect(order.indexOf('users')).toBeLessThan(order.indexOf('conversations'));
});
it('includes "conversations" before "messages"', () => {
const order = getMigrationOrder();
expect(order.indexOf('conversations')).toBeLessThan(order.indexOf('messages'));
});
it('includes "projects" before "agents"', () => {
const order = getMigrationOrder();
expect(order.indexOf('projects')).toBeLessThan(order.indexOf('agents'));
});
it('includes "agents" before "conversations"', () => {
const order = getMigrationOrder();
expect(order.indexOf('agents')).toBeLessThan(order.indexOf('conversations'));
});
it('includes "missions" before "mission_tasks"', () => {
const order = getMigrationOrder();
expect(order.indexOf('missions')).toBeLessThan(order.indexOf('mission_tasks'));
});
it('includes all expected tables', () => {
const order = getMigrationOrder();
const expected = [
'users',
'teams',
'accounts',
'projects',
'agents',
'conversations',
'messages',
'insights',
];
for (const t of expected) {
expect(order).toContain(t);
}
});
});
/* ------------------------------------------------------------------ */
/* 3. Dry-run makes no writes */
/* ------------------------------------------------------------------ */
describe('runMigrateTier — dry-run', () => {
it('makes no calls to upsertBatch', async () => {
const source = makeMockSource({
users: [{ id: 'u1', name: 'Alice', email: '[email protected]' }],
});
const target = makeMockTarget();
const result = await runMigrateTier(source, target, {
targetUrl: 'postgresql://localhost/test',
dryRun: true,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
});
expect(target.upsertCalls).toHaveLength(0);
expect(result.dryRun).toBe(true);
expect(result.totalRows).toBe(0);
});
it('does not call checkTargetPreconditions in dry-run', async () => {
// Even if hasPgvector is false, dry-run should not throw.
const source = makeMockSource({});
const target = makeMockTarget({ hasPgvector: false });
await expect(
runMigrateTier(source, target, {
targetUrl: 'postgresql://localhost/test',
dryRun: true,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
}),
).resolves.not.toThrow();
// hasPgvector should NOT have been called during dry run.
expect(target.hasPgvector).not.toHaveBeenCalled();
});
});
/* ------------------------------------------------------------------ */
/* 4. Idempotency */
/* ------------------------------------------------------------------ */
describe('runMigrateTier — idempotency', () => {
it('produces the same logical row count on second run (upsert semantics)', async () => {
const userData = [
{ id: 'u1', name: 'Alice', email: '[email protected]' },
{ id: 'u2', name: 'Bob', email: '[email protected]' },
];
const source = makeMockSource({ users: userData });
// First run target.
const target1 = makeMockTarget();
await runMigrateTier(source, target1, {
targetUrl: 'postgresql://localhost/test',
dryRun: false,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
});
const firstRunUpserts = target1.upsertCalls.filter((c) => c.table === 'users');
const firstRunRows = firstRunUpserts.reduce((acc, c) => acc + c.rows.length, 0);
// Second run — allowNonEmpty because first run already wrote rows.
const target2 = makeMockTarget();
await runMigrateTier(source, target2, {
targetUrl: 'postgresql://localhost/test',
dryRun: false,
allowNonEmpty: true,
batchSize: 100,
onProgress: noopProgress(),
});
const secondRunUpserts = target2.upsertCalls.filter((c) => c.table === 'users');
const secondRunRows = secondRunUpserts.reduce((acc, c) => acc + c.rows.length, 0);
// Both runs write the same number of rows (upsert — second run updates in place).
expect(firstRunRows).toBe(userData.length);
expect(secondRunRows).toBe(userData.length);
});
});
/* ------------------------------------------------------------------ */
/* 5. Empty-target precondition */
/* ------------------------------------------------------------------ */
describe('checkTargetPreconditions', () => {
it('throws when target table is non-empty and allowNonEmpty is false', async () => {
const target = makeMockTarget({ nonEmptyTable: 'users' });
await expect(checkTargetPreconditions(target, false, ['users'])).rejects.toThrow(
MigrationPreconditionError,
);
});
it('includes remediation hint in thrown error', async () => {
const target = makeMockTarget({ nonEmptyTable: 'users' });
await expect(checkTargetPreconditions(target, false, ['users'])).rejects.toMatchObject({
name: 'MigrationPreconditionError',
remediation: expect.stringContaining('--allow-non-empty'),
});
});
it('does NOT throw when allowNonEmpty is true', async () => {
const target = makeMockTarget({ nonEmptyTable: 'users' });
await expect(checkTargetPreconditions(target, true, ['users'])).resolves.not.toThrow();
});
it('throws when pgvector extension is missing', async () => {
const target = makeMockTarget({ hasPgvector: false });
await expect(checkTargetPreconditions(target, false, ['users'])).rejects.toMatchObject({
name: 'MigrationPreconditionError',
remediation: expect.stringContaining('pgvector'),
});
});
it('passes when target is empty and pgvector is present', async () => {
const target = makeMockTarget({ hasPgvector: true });
await expect(checkTargetPreconditions(target, false, ['users'])).resolves.not.toThrow();
});
});
/* ------------------------------------------------------------------ */
/* 6. Skipped tables documented */
/* ------------------------------------------------------------------ */
describe('SKIP_TABLES', () => {
it('includes "sessions"', () => {
expect(SKIP_TABLES.has('sessions')).toBe(true);
});
it('includes "verifications"', () => {
expect(SKIP_TABLES.has('verifications')).toBe(true);
});
it('includes "admin_tokens"', () => {
expect(SKIP_TABLES.has('admin_tokens')).toBe(true);
});
it('migration result includes skipped table entries', async () => {
const source = makeMockSource({});
const target = makeMockTarget();
const result = await runMigrateTier(source, target, {
targetUrl: 'postgresql://localhost/test',
dryRun: false,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
});
const skippedNames = result.tables.filter((t) => t.skipped).map((t) => t.table);
expect(skippedNames).toContain('sessions');
expect(skippedNames).toContain('verifications');
expect(skippedNames).toContain('admin_tokens');
});
});
/* ------------------------------------------------------------------ */
/* 7. Embedding NULL on migrate from non-pgvector source */
/* ------------------------------------------------------------------ */
describe('normaliseSourceRow — embedding handling', () => {
it('sets embedding to null when sourceHasVector is false and table is insights', () => {
const row: Record<string, unknown> = {
id: 'ins-1',
content: 'Some insight',
userId: 'u1',
};
const normalised = normaliseSourceRow('insights', row, false);
expect(normalised['embedding']).toBeNull();
});
it('preserves existing embedding when sourceHasVector is true', () => {
const embedding = [0.1, 0.2, 0.3];
const row: Record<string, unknown> = {
id: 'ins-1',
content: 'Some insight',
userId: 'u1',
embedding,
};
const normalised = normaliseSourceRow('insights', row, true);
expect(normalised['embedding']).toBe(embedding);
});
it('does not add embedding field to non-vector tables', () => {
const row: Record<string, unknown> = { id: 'u1', name: 'Alice' };
const normalised = normaliseSourceRow('users', row, false);
expect('embedding' in normalised).toBe(false);
});
it('passes through rows for non-vector tables unchanged', () => {
const row: Record<string, unknown> = { id: 'u1', name: 'Alice', email: '[email protected]' };
const normalised = normaliseSourceRow('users', row, false);
expect(normalised).toEqual(row);
});
});
/* ------------------------------------------------------------------ */
/* 8. End-to-end: correct order of upsert calls */
/* ------------------------------------------------------------------ */
describe('runMigrateTier — migration order', () => {
it('writes users before messages', async () => {
const source = makeMockSource({
users: [{ id: 'u1', name: 'Alice', email: '[email protected]' }],
messages: [{ id: 'm1', conversationId: 'c1', role: 'user', content: 'Hi' }],
});
const target = makeMockTarget();
await runMigrateTier(source, target, {
targetUrl: 'postgresql://localhost/test',
dryRun: false,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
});
const tableOrder = target.upsertCalls.map((c) => c.table);
const usersIdx = tableOrder.indexOf('users');
const messagesIdx = tableOrder.indexOf('messages');
// users must appear before messages in the upsert call sequence.
expect(usersIdx).toBeGreaterThanOrEqual(0);
expect(messagesIdx).toBeGreaterThanOrEqual(0);
expect(usersIdx).toBeLessThan(messagesIdx);
});
});
/* ------------------------------------------------------------------ */
/* 9. Embedding-null projection: no-pgvector source */
/* ------------------------------------------------------------------ */
describe('DrizzleMigrationSource embedding-null projection', () => {
it(
'when sourceHasVector is false, readTable for insights omits embedding column ' +
'and normaliseSourceRow sets it to null for the target insert',
async () => {
// Source has insights data but no vector — embedding omitted at read time.
const insightRowWithEmbedding = {
id: 'ins-1',
userId: 'u1',
content: 'Test insight',
embedding: [0.1, 0.2, 0.3], // present in raw data but omitted by source
source: 'agent',
category: 'general',
relevanceScore: 1.0,
};
// makeMockSource with sourceHasVector=false simulates DrizzleMigrationSource
// behaviour: the embedding field is stripped from the returned row.
const source = makeMockSource(
{
users: [{ id: 'u1', name: 'Alice', email: '[email protected]' }],
insights: [insightRowWithEmbedding],
},
/* sourceHasVector= */ false,
);
const target = makeMockTarget();
await runMigrateTier(
source,
target,
{
targetUrl: 'postgresql://localhost/test',
dryRun: false,
allowNonEmpty: false,
batchSize: 100,
onProgress: noopProgress(),
},
/* sourceHasVector= */ false,
);
// Assert: readTable was called for insights
const insightsRead = source.readTableCalls.find((c) => c.table === 'insights');
expect(insightsRead).toBeDefined();
// Assert: the upsert to insights has embedding === null (not the original vector)
const insightsUpsert = target.upsertCalls.find((c) => c.table === 'insights');
expect(insightsUpsert).toBeDefined();
const upsertedRow = insightsUpsert!.rows[0];
expect(upsertedRow).toBeDefined();
// embedding must be null — not the original [0.1, 0.2, 0.3]
expect(upsertedRow!['embedding']).toBeNull();
// Other fields must pass through unchanged
expect(upsertedRow!['id']).toBe('ins-1');
expect(upsertedRow!['content']).toBe('Test insight');
},
);
});
+799
View File
@@ -0,0 +1,799 @@
/**
* migrate-tier.ts — Core logic for `mosaic storage migrate-tier`.
*
* Migrates data from `tier: local` (PGlite, normalized Drizzle schema) or
* `tier: standalone` (Postgres without pgvector) → `tier: federated`
* (Postgres + pgvector).
*
* Source: DrizzleMigrationSource — reads from the NORMALIZED Drizzle/relational
* schema tables (not the flat `id TEXT + data JSONB` PgliteAdapter schema).
* Both local (PGlite) and standalone (Postgres) sources use the same Drizzle
* abstraction via createPgliteDb() or createDb() from @mosaicstack/db.
* Target: PostgresMigrationTarget — upserts via raw SQL into the same schema.
*
* Key design decisions:
* - Tables are migrated in topological (FK-dependency) order so that
* parent rows exist before child rows are inserted.
* - sessions + verifications are skipped — they are ephemeral / TTL'd.
* - adminTokens is skipped — token hashes are environment-specific
* and should be re-issued on the target.
* - insights.embedding is omitted from source SELECT when the source lacks
* pgvector (local/PGlite tier); target insert gets NULL for that column.
* insights.embedding is nullable per schema (no .notNull() constraint).
* - Each table's batch is wrapped in a transaction for atomicity.
* - Upsert semantics (ON CONFLICT DO UPDATE) make re-runs idempotent.
*
* TODO (FED-M1-08): Add integration tests against real PGlite → real PG.
*/
import postgres from 'postgres';
import * as schema from '@mosaicstack/db';
import { sql as drizzleSql } from '@mosaicstack/db';
import { redactErrMsg } from './redact-error.js';
/* ------------------------------------------------------------------ */
/* Types */
/* ------------------------------------------------------------------ */
export interface MigrationSource {
/**
* Return all rows from a table (normalized Drizzle schema rows).
* When sourceHasVector is false and the table has a vector column,
* the source MUST omit the vector column from the result and the
* caller will set it to null (see normaliseSourceRow).
*/
readTable(
tableName: string,
opts?: { limit?: number; offset?: number },
): Promise<Record<string, unknown>[]>;
/** Count rows in a table. */
count(tableName: string): Promise<number>;
/** Close the source connection. */
close(): Promise<void>;
}
export interface MigrationTarget {
/**
* Upsert a batch of rows into a table.
* Must use ON CONFLICT (id) DO UPDATE semantics.
*/
upsertBatch(tableName: string, rows: Record<string, unknown>[]): Promise<void>;
/**
* Count rows in a target table.
*/
count(tableName: string): Promise<number>;
/**
* Check whether pgvector extension is installed.
*/
hasPgvector(): Promise<boolean>;
/** Close the target connection. */
close(): Promise<void>;
/**
* Attempt to acquire a session-level Postgres advisory lock for migrate-tier.
* Returns true if the lock was acquired, false if another process holds it.
* Targets that do not support advisory locks (e.g. test mocks) may omit this
* by not implementing the method — the caller skips locking gracefully.
*/
tryAcquireAdvisoryLock?(): Promise<boolean>;
/**
* Release the session-level advisory lock acquired by tryAcquireAdvisoryLock.
* Must be called in a finally block.
*/
releaseAdvisoryLock?(): Promise<void>;
}
export interface MigrateTierOptions {
/** Target postgres connection URL. */
targetUrl: string;
/** Whether to skip all writes (dry-run). */
dryRun: boolean;
/** Skip the non-empty target guard. */
allowNonEmpty: boolean;
/** Rows per transaction batch. */
batchSize: number;
/** Called with progress messages. */
onProgress: (msg: string) => void;
}
export interface TableMigrationResult {
table: string;
rowsMigrated: number;
skipped: boolean;
skipReason?: string;
}
export interface MigrateTierResult {
tables: TableMigrationResult[];
totalRows: number;
dryRun: boolean;
}
/* ------------------------------------------------------------------ */
/* Schema: FK-aware topological table order */
/* ------------------------------------------------------------------ */
/**
* SKIP_TABLES: ephemeral or environment-specific tables not worth migrating.
*
* WHY these tables are skipped:
* - sessions: TTL'd auth sessions — they are invalid in the new environment
* and would immediately expire or fail JWT verification anyway.
* - verifications: one-time tokens (email verify, password-reset links, etc.)
* — they have already expired or been consumed; re-sending is
* the correct action on the new environment.
* - admin_tokens: hashed tokens bound to the old environment's secret keys —
* the hash is environment-specific and must be re-issued on
* the target.
*
* WHY these tables are NOT skipped (intentionally migrated):
* - accounts (OAuth tokens): durable credentials bound to the user's identity,
* not to the deployment environment. OAuth tokens survive environment changes
* and should follow the user to the federated tier.
* - provider_credentials (AI provider keys): durable, user-owned API keys for
* AI providers (e.g. OpenAI, Anthropic). These are bound to the user, not
* the server, and must be preserved so AI features work immediately after
* migration.
*
* OPERATOR NOTE: If migrating to a shared or multi-tenant federated tier, review
* whether `accounts` and `provider_credentials` should be wiped post-migration
* to prevent unintended cross-tenant credential exposure.
*/
export const SKIP_TABLES = new Set(['sessions', 'verifications', 'admin_tokens']);
/**
* Topologically ordered table list (parents before children).
*
* Derived from FK references in packages/db/src/schema.ts:
*
* users (no FKs)
* teams → users
* team_members → teams, users
* accounts → users
* projects → users, teams
* agents → projects, users
* missions → projects, users
* tasks → projects, missions
* mission_tasks → missions, tasks, users
* conversations → users, projects, agents
* messages → conversations
* preferences → users
* insights → users [has embedding vector column]
* agent_logs → users
* skills → users (installedBy, nullable)
* routing_rules → users (userId, nullable)
* provider_credentials → users
* appreciations (no FKs)
* events (no FKs)
* tickets (no FKs)
* summarization_jobs (no FKs)
*
* Skipped (not in this list):
* sessions → users (ephemeral)
* verifications (no FKs, ephemeral)
* admin_tokens → users (environment-specific)
*/
export const MIGRATION_ORDER: string[] = [
'users',
'teams',
'team_members',
'accounts',
'projects',
'agents',
'missions',
'tasks',
'mission_tasks',
'conversations',
'messages',
'preferences',
'insights',
'agent_logs',
'skills',
'routing_rules',
'provider_credentials',
'appreciations',
'events',
'tickets',
'summarization_jobs',
];
/** Tables that carry a vector embedding column on the target. */
const VECTOR_TABLES = new Set(['insights']);
/* ------------------------------------------------------------------ */
/* Utility: derive topological order from an adjacency list */
/* ------------------------------------------------------------------ */
/**
* Given an adjacency list (table → list of tables it depends on),
* return a valid topological ordering (Kahn's algorithm).
*
* Exposed for unit testing.
*/
export function topoSort(deps: Map<string, string[]>): string[] {
const nodes = [...deps.keys()];
const inDegree = new Map<string, number>();
const adjReverse = new Map<string, string[]>();
for (const node of nodes) {
if (!inDegree.has(node)) inDegree.set(node, 0);
if (!adjReverse.has(node)) adjReverse.set(node, []);
for (const dep of deps.get(node) ?? []) {
inDegree.set(node, (inDegree.get(node) ?? 0) + 1);
if (!adjReverse.has(dep)) adjReverse.set(dep, []);
adjReverse.get(dep)!.push(node);
}
}
// Start with nodes that have no dependencies.
const queue: string[] = [];
for (const [node, deg] of inDegree) {
if (deg === 0) queue.push(node);
}
const result: string[] = [];
while (queue.length > 0) {
const node = queue.shift()!;
result.push(node);
for (const dependent of adjReverse.get(node) ?? []) {
const newDeg = (inDegree.get(dependent) ?? 0) - 1;
inDegree.set(dependent, newDeg);
if (newDeg === 0) queue.push(dependent);
}
}
if (result.length !== nodes.length) {
throw new Error('Cycle detected in FK dependency graph');
}
return result;
}
/**
* Return the migration table order, excluding SKIP_TABLES.
* Uses the pre-computed MIGRATION_ORDER (verified against schema.ts).
*/
export function getMigrationOrder(): string[] {
return MIGRATION_ORDER.filter((t) => !SKIP_TABLES.has(t));
}
/* ------------------------------------------------------------------ */
/* TABLE_OBJECTS: migration table name → Drizzle table object */
/* ------------------------------------------------------------------ */
/**
* Maps MIGRATION_ORDER table names to their corresponding Drizzle table
* objects from the normalized schema. Used by DrizzleMigrationSource to
* execute typed `db.select().from(table)` queries.
*
* Keyed by snake_case table name (matching MIGRATION_ORDER + SKIP_TABLES).
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const TABLE_OBJECTS: Record<string, any> = {
users: schema.users,
teams: schema.teams,
team_members: schema.teamMembers,
accounts: schema.accounts,
projects: schema.projects,
agents: schema.agents,
missions: schema.missions,
tasks: schema.tasks,
mission_tasks: schema.missionTasks,
conversations: schema.conversations,
messages: schema.messages,
preferences: schema.preferences,
insights: schema.insights,
agent_logs: schema.agentLogs,
skills: schema.skills,
routing_rules: schema.routingRules,
provider_credentials: schema.providerCredentials,
appreciations: schema.appreciations,
events: schema.events,
tickets: schema.tickets,
summarization_jobs: schema.summarizationJobs,
// Skipped tables — included so count() works for preflight but never passed
// to upsertBatch.
sessions: schema.sessions,
verifications: schema.verifications,
admin_tokens: schema.adminTokens,
};
/* ------------------------------------------------------------------ */
/* DrizzleMigrationSource */
/* ------------------------------------------------------------------ */
/**
* MigrationSource backed by a Drizzle DB handle (works with both
* PostgresJsDatabase and PgliteDatabase — they share the same Drizzle
* query API for schema-defined tables).
*
* For the `insights` table (the only vector-column table), when the source
* lacks pgvector (local/PGlite tier), the `embedding` column is excluded
* from the SELECT projection via a raw `db.execute()` query that lists
* only non-vector columns. This prevents a type-registration error from
* PGlite, which does not know the `vector` type. The caller (runMigrateTier
* via normaliseSourceRow) will set embedding to null on the resulting rows.
*
* Column projection is opt-in: pass `sourceHasVector: false` to activate it.
*/
export class DrizzleMigrationSource implements MigrationSource {
constructor(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private readonly db: any,
private readonly sourceHasVector: boolean = true,
) {}
/**
* Columns of the insights table that do NOT include the vector embedding.
* Used for the no-pgvector projection path.
*/
private static readonly INSIGHTS_COLUMNS_NO_VECTOR = [
'id',
'user_id',
'content',
'source',
'category',
'relevance_score',
'metadata',
'created_at',
'updated_at',
'decayed_at',
] as const;
async readTable(
tableName: string,
opts?: { limit?: number; offset?: number },
): Promise<Record<string, unknown>[]> {
const table = TABLE_OBJECTS[tableName];
if (!table) throw new Error(`DrizzleMigrationSource: unknown table "${tableName}"`);
// For vector tables when source lacks pgvector: use column-allowlist raw query
// to avoid type-registration errors.
if (VECTOR_TABLES.has(tableName) && !this.sourceHasVector) {
const cols = DrizzleMigrationSource.INSIGHTS_COLUMNS_NO_VECTOR.map((c) => `"${c}"`).join(
', ',
);
let sql = `SELECT ${cols} FROM "${tableName}"`;
const params: unknown[] = [];
if (opts?.limit !== undefined) {
params.push(opts.limit);
sql += ` LIMIT $${params.length.toString()}`;
}
if (opts?.offset !== undefined) {
params.push(opts.offset);
sql += ` OFFSET $${params.length.toString()}`;
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const result = await (this.db as any).execute(
// drizzle-orm/pglite and drizzle-orm/postgres-js both accept a raw
// SQL template; use the tagged-template sql helper from drizzle-orm.
// Since we need dynamic params, we use db.execute with a raw string
// via the PGlite/postgres.js driver directly.
{ sql, params, typings: [] },
);
// drizzle execute returns { rows: unknown[][] } for PGlite driver,
// or a RowList for postgres.js. Normalise both shapes.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const raw = result as any;
if (Array.isArray(raw)) {
// postgres.js shape: array of row objects
return raw as Record<string, unknown>[];
}
if (raw?.rows && Array.isArray(raw.rows)) {
// PGlite shape: { rows: unknown[][] } OR { rows: Record<string,unknown>[] }
const rows = raw.rows as unknown[];
if (rows.length === 0) return [];
if (Array.isArray(rows[0])) {
// Columnar: convert to objects using fields array if available
const fields: string[] =
(raw.fields as Array<{ name: string }> | undefined)?.map((f) => f.name) ??
DrizzleMigrationSource.INSIGHTS_COLUMNS_NO_VECTOR.slice();
return (rows as unknown[][]).map((row) => {
const obj: Record<string, unknown> = {};
for (let i = 0; i < fields.length; i++) {
obj[fields[i]!] = row[i];
}
return obj;
});
}
return rows as Record<string, unknown>[];
}
return [];
}
// Standard Drizzle select for all other tables (and vector tables when
// the source has pgvector registered).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let query = (this.db as any).select().from(table);
if (opts?.limit !== undefined) query = query.limit(opts.limit);
if (opts?.offset !== undefined) query = query.offset(opts.offset);
return (await query) as Record<string, unknown>[];
}
async count(tableName: string): Promise<number> {
const table = TABLE_OBJECTS[tableName];
if (!table) throw new Error(`DrizzleMigrationSource: unknown table "${tableName}"`);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const [row] = await (this.db as any)
.select({ n: drizzleSql<number>`COUNT(*)::int` })
.from(table);
return (row as { n: number } | undefined)?.n ?? 0;
}
async close(): Promise<void> {
// Lifecycle managed externally — caller closes the db handle.
}
}
/* ------------------------------------------------------------------ */
/* Real postgres target adapter */
/* ------------------------------------------------------------------ */
/**
* Live implementation of MigrationTarget backed by a real Postgres connection.
* Used by the CLI; mocked in tests.
*/
export class PostgresMigrationTarget implements MigrationTarget {
private sql: ReturnType<typeof postgres>;
constructor(url: string) {
this.sql = postgres(url, {
max: 5,
connect_timeout: 10,
idle_timeout: 30,
});
}
async upsertBatch(tableName: string, rows: Record<string, unknown>[]): Promise<void> {
if (rows.length === 0) return;
// Collect all column names from the batch (union of all row keys).
const colSet = new Set<string>();
for (const row of rows) {
for (const k of Object.keys(row)) colSet.add(k);
}
const cols = [...colSet];
if (!cols.includes('id')) {
throw new Error(`Table ${tableName}: rows missing 'id' column`);
}
// Build VALUES list — use postgres tagged-template helpers for safety.
// postgres.js supports bulk inserts via array of objects.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
await this.sql.begin(async (tx: any) => {
// Insert in chunks to avoid enormous single queries.
for (let i = 0; i < rows.length; i += 500) {
const chunk = rows.slice(i, i + 500);
// Normalise rows: fill missing columns with null.
const normalised = chunk.map((row) => {
const out: Record<string, unknown> = {};
for (const col of cols) {
out[col] = row[col] ?? null;
}
return out;
});
const colList = cols.map((c) => `"${c}"`).join(', ');
const updateList = cols
.filter((c) => c !== 'id')
.map((c) => `"${c}" = EXCLUDED."${c}"`)
.join(', ');
// Build values placeholders
const valuePlaceholders = normalised
.map((_, ri) => `(${cols.map((_, ci) => `$${ri * cols.length + ci + 1}`).join(', ')})`)
.join(', ');
const flatValues = normalised.flatMap((row) => cols.map((c) => row[c] ?? null));
const query = `
INSERT INTO "${tableName}" (${colList})
VALUES ${valuePlaceholders}
ON CONFLICT (id) DO UPDATE SET ${updateList}
`;
await tx.unsafe(query, flatValues as never[]);
}
});
}
async count(tableName: string): Promise<number> {
const rows = await this.sql.unsafe(`SELECT COUNT(*)::int AS n FROM "${tableName}"`);
const row = rows[0] as unknown as { n: number } | undefined;
return row?.n ?? 0;
}
async hasPgvector(): Promise<boolean> {
const rows = await this.sql`
SELECT 1 FROM pg_extension WHERE extname = 'vector'
`;
return rows.length > 0;
}
/**
* Attempt to acquire a non-blocking session-level Postgres advisory lock
* keyed by hashtext('mosaic-migrate-tier'). Returns true if acquired,
* false if another session already holds the lock.
*
* The lock is session-scoped: it is automatically released when the
* connection closes, and also explicitly released via releaseAdvisoryLock().
*/
async tryAcquireAdvisoryLock(): Promise<boolean> {
const rows = await this.sql`
SELECT pg_try_advisory_lock(hashtext('mosaic-migrate-tier')) AS acquired
`;
const row = rows[0] as { acquired: boolean } | undefined;
return row?.acquired ?? false;
}
/**
* Release the session-level advisory lock previously acquired by
* tryAcquireAdvisoryLock(). Safe to call even if the lock was not held
* (pg_advisory_unlock returns false but does not throw).
*/
async releaseAdvisoryLock(): Promise<void> {
await this.sql`
SELECT pg_advisory_unlock(hashtext('mosaic-migrate-tier'))
`;
}
async close(): Promise<void> {
await this.sql.end();
}
}
/* ------------------------------------------------------------------ */
/* Source-row normalisation */
/* ------------------------------------------------------------------ */
/**
* Convert a camelCase key to snake_case.
* e.g. "userId" → "user_id", "emailVerified" → "email_verified".
* Keys that are already snake_case (no uppercase letters) are returned as-is.
*/
function toSnakeCase(key: string): string {
return key.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
}
/**
* Drizzle returns rows as camelCase TypeScript objects (e.g. `userId`, not
* `user_id`). The PostgresMigrationTarget upserts via raw SQL and uses the
* column names as given. We must convert camelCase keys → snake_case before
* building the INSERT statement so column names match the PG schema.
*
* Exception: the `insights` no-vector path already returns snake_case keys
* from its raw SQL projection — toSnakeCase() is idempotent for already-
* snake_case keys so this conversion is safe in all paths.
*
* For vector tables (insights), if `embedding` is absent from the source row
* (because DrizzleMigrationSource omitted it in the no-vector projection), we
* explicitly set it to null so the target ON CONFLICT UPDATE doesn't error.
*
* NOTE: insights.embedding is defined as `vector('embedding', { dimensions:
* 1536 })` with no `.notNull()` in schema.ts — it accepts NULL.
*/
export function normaliseSourceRow(
tableName: string,
row: Record<string, unknown>,
sourceHasVector: boolean,
): Record<string, unknown> {
// Convert all camelCase keys to snake_case for raw-SQL target compatibility.
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(row)) {
out[toSnakeCase(k)] = v;
}
if (VECTOR_TABLES.has(tableName) && !sourceHasVector) {
// Source cannot have embeddings — explicitly null them so ON CONFLICT
// UPDATE doesn't try to write undefined.
out['embedding'] = null;
}
return out;
}
/* ------------------------------------------------------------------ */
/* Precondition checks */
/* ------------------------------------------------------------------ */
export class MigrationPreconditionError extends Error {
constructor(
message: string,
public readonly remediation: string,
) {
super(message);
this.name = 'MigrationPreconditionError';
}
}
/**
* Verify target preconditions before writing any data.
*
* Checks:
* 1. pgvector extension installed.
* 2. User-data tables are empty (unless --allow-non-empty).
*/
export async function checkTargetPreconditions(
target: MigrationTarget,
allowNonEmpty: boolean,
tablesToMigrate: string[],
): Promise<void> {
const hasVector = await target.hasPgvector();
if (!hasVector) {
throw new MigrationPreconditionError(
'Target Postgres does not have the pgvector extension installed.',
'Run: CREATE EXTENSION IF NOT EXISTS vector; — or use the pgvector/pgvector:pg17 Docker image.',
);
}
if (!allowNonEmpty) {
// Check the first non-empty user-data table.
for (const table of tablesToMigrate) {
const n = await target.count(table);
if (n > 0) {
throw new MigrationPreconditionError(
`Target table "${table}" already contains ${n.toString()} rows.`,
'Pass --allow-non-empty to overwrite existing data (upsert semantics), ' +
'or point to an empty target database.',
);
}
}
}
}
/* ------------------------------------------------------------------ */
/* Core migration runner */
/* ------------------------------------------------------------------ */
/**
* Run the tier migration.
*
* @param source Adapter for reading source rows.
* @param target Adapter for writing rows to target.
* @param opts Migration options.
* @param sourceHasVector True if the source tier supports vector columns.
*/
export async function runMigrateTier(
source: MigrationSource,
target: MigrationTarget,
opts: MigrateTierOptions,
sourceHasVector = false,
): Promise<MigrateTierResult> {
const { dryRun, allowNonEmpty, batchSize, onProgress } = opts;
const tablesToMigrate = getMigrationOrder();
// Preflight: gather row counts from source.
onProgress('[migrate-tier] Gathering source row counts...');
const sourceCounts = new Map<string, number>();
for (const table of tablesToMigrate) {
const n = await source.count(table);
sourceCounts.set(table, n);
}
// Log preflight summary.
onProgress('[migrate-tier] Tables to migrate:');
for (const table of tablesToMigrate) {
const n = sourceCounts.get(table) ?? 0;
onProgress(` ${table}: ${n.toString()} rows`);
}
for (const skipped of SKIP_TABLES) {
onProgress(` ${skipped}: SKIPPED (ephemeral/environment-specific)`);
}
// Vector column notice.
if (!sourceHasVector) {
onProgress(
'[migrate-tier] NOTE: Source tier has no pgvector support. ' +
'insights.embedding will be NULL on all migrated rows.',
);
}
if (dryRun) {
onProgress('[migrate-tier] DRY RUN — no writes will be made.');
const tables: TableMigrationResult[] = tablesToMigrate.map((t) => ({
table: t,
rowsMigrated: 0,
skipped: false,
}));
for (const skipped of SKIP_TABLES) {
tables.push({ table: skipped, rowsMigrated: 0, skipped: true, skipReason: 'ephemeral' });
}
return { tables, totalRows: 0, dryRun: true };
}
// Acquire a Postgres advisory lock on the target BEFORE checking preconditions
// so that two concurrent invocations cannot both pass the non-empty guard and
// race each other. Use non-blocking pg_try_advisory_lock so we fail fast
// instead of deadlocking.
//
// Targets that don't implement tryAcquireAdvisoryLock (e.g. test mocks) skip
// this step — the optional chaining guard handles that case.
const lockAcquired = target.tryAcquireAdvisoryLock ? await target.tryAcquireAdvisoryLock() : true; // mocks / test doubles — no locking needed
if (!lockAcquired) {
throw new Error(
'Another migrate-tier process is already running against this target. ' +
'Wait for it to complete or check for stuck locks via ' +
"SELECT * FROM pg_locks WHERE locktype='advisory'.",
);
}
try {
// Check preconditions before writing.
await checkTargetPreconditions(target, allowNonEmpty, tablesToMigrate);
const results: TableMigrationResult[] = [];
let totalRows = 0;
for (const table of tablesToMigrate) {
const sourceCount = sourceCounts.get(table) ?? 0;
if (sourceCount === 0) {
onProgress(`[migrate-tier] ${table}: 0 rows — skipping.`);
results.push({ table, rowsMigrated: 0, skipped: false });
continue;
}
onProgress(`[migrate-tier] ${table}: migrating ${sourceCount.toString()} rows...`);
let offset = 0;
let tableTotal = 0;
let lastSuccessfulId: string | undefined;
try {
while (offset < sourceCount) {
const rows = await source.readTable(table, { limit: batchSize, offset });
if (rows.length === 0) break;
const normalised = rows.map((row) => normaliseSourceRow(table, row, sourceHasVector));
await target.upsertBatch(table, normalised);
lastSuccessfulId = rows[rows.length - 1]?.['id'] as string | undefined;
tableTotal += rows.length;
offset += rows.length;
onProgress(
`[migrate-tier] ${table}: ${tableTotal.toString()}/${sourceCount.toString()} rows written`,
);
}
} catch (err) {
const errMsg = redactErrMsg(err instanceof Error ? err.message : String(err));
throw new Error(
`[migrate-tier] Failed on table "${table}" after ${tableTotal.toString()} rows ` +
`(last id: ${lastSuccessfulId ?? 'none'}). Error: ${errMsg}\n` +
`Remediation: Re-run with --allow-non-empty to resume (upsert is idempotent).`,
);
}
results.push({ table, rowsMigrated: tableTotal, skipped: false });
totalRows += tableTotal;
onProgress(`[migrate-tier] ${table}: done (${tableTotal.toString()} rows).`);
}
// Add skipped table records.
for (const skipped of SKIP_TABLES) {
results.push({
table: skipped,
rowsMigrated: 0,
skipped: true,
skipReason: 'ephemeral or environment-specific — re-issue on target',
});
}
onProgress(`[migrate-tier] Complete. ${totalRows.toString()} total rows migrated.`);
return { tables: results, totalRows, dryRun: false };
} finally {
// Release the advisory lock regardless of success or failure.
if (target.releaseAdvisoryLock) {
await target.releaseAdvisoryLock();
}
}
}
@@ -0,0 +1,62 @@
import { describe, it, expect } from 'vitest';
import { redactErrMsg } from './redact-error.js';
describe('redactErrMsg', () => {
it('redacts user:password from a postgres:// URL embedded in an error message', () => {
const msg = 'connect ECONNREFUSED postgres://admin:[email protected]:5432/mosaic';
expect(redactErrMsg(msg)).toBe(
'connect ECONNREFUSED postgres://***@db.example.com:5432/mosaic',
);
});
it('redacts user:password from a postgresql:// URL', () => {
const msg = 'connection failed: postgresql://myuser:mypass@localhost:5432/testdb';
expect(redactErrMsg(msg)).toBe('connection failed: postgresql://***@localhost:5432/testdb');
});
it('handles URLs with no password (user only) — still redacts userinfo', () => {
const msg = 'error postgres://justuser@host:5432/db';
expect(redactErrMsg(msg)).toBe('error postgres://***@host:5432/db');
});
it('returns the original message unchanged when no connection URL is present', () => {
const msg = 'connection timed out after 5 seconds';
expect(redactErrMsg(msg)).toBe('connection timed out after 5 seconds');
});
it('is case-insensitive for the scheme (scheme is normalized to lowercase in output)', () => {
// The regex replacement uses a lowercase literal, so the matched scheme is
// replaced with the lowercase form regardless of the original casing.
const msg = 'POSTGRES://admin:pass@host:5432/db';
expect(redactErrMsg(msg)).toBe('postgres://***@host:5432/db');
});
it('redacts multiple URLs in a single message', () => {
const msg = 'src postgres://u:p@host1/db1 dst postgresql://v:q@host2/db2';
expect(redactErrMsg(msg)).toBe('src postgres://***@host1/db1 dst postgresql://***@host2/db2');
});
it('does not alter a message with a postgres URL that has no userinfo', () => {
// No userinfo component — pattern does not match, message unchanged.
const msg = 'error at postgres://host:5432/db';
expect(redactErrMsg(msg)).toBe('error at postgres://host:5432/db');
});
it('redacts user:password from a redis:// URL', () => {
const msg = 'connect ECONNREFUSED redis://user:pass@host:6379';
expect(redactErrMsg(msg)).toBe('connect ECONNREFUSED redis://***@host:6379');
});
it('redacts user:password from a rediss:// URL (TLS)', () => {
const msg = 'connect ECONNREFUSED rediss://user:pass@host:6379';
expect(redactErrMsg(msg)).toBe('connect ECONNREFUSED rediss://***@host:6379');
});
it('redacts both a postgres URL and a redis URL in the same message', () => {
const msg =
'primary postgres://admin:s3cr3t@db:5432/mosaic cache redis://cacheuser:cachepass@cache:6379';
expect(redactErrMsg(msg)).toBe(
'primary postgres://***@db:5432/mosaic cache redis://***@cache:6379',
);
});
});
+39
View File
@@ -0,0 +1,39 @@
/**
* redact-error.ts — Internal credential-scrubbing helper.
*
* The `postgres` npm package can embed the full DSN (including the password)
* in connection-failure error messages. This module provides a single helper
* that strips the user:password portion from any such message before it is
* re-thrown, logged, or surfaced in a structured health report.
*
* This file is intentionally NOT re-exported from the package index — it is
* an internal utility for use within packages/storage/src only.
*/
/**
* Redacts credentials from error messages that may include connection URLs.
* The `postgres` npm package can embed the full DSN in connection-failure
* messages, and ioredis can embed `redis://` / `rediss://` URLs similarly.
* This helper strips the user:password portion before display.
*
* Handles `postgres://`, `postgresql://`, `redis://`, and `rediss://`
* schemes (case-insensitive). Everything between `://` and `@` (the userinfo
* component) is replaced with `***` so that the host, port, and database name
* remain visible for diagnostics while the secret is never written to logs or
* CI output.
*
* @example
* redactErrMsg('connect ECONNREFUSED postgres://admin:s3cr3t@db:5432/mosaic')
* // → 'connect ECONNREFUSED postgres://***@db:5432/mosaic'
*
* redactErrMsg('connect ECONNREFUSED redis://user:pass@cache:6379')
* // → 'connect ECONNREFUSED redis://***@cache:6379'
*/
const CREDENTIAL_URL_RE = /(postgres(?:ql)?|rediss?):\/\/[^@\s]*@/gi;
export function redactErrMsg(msg: string): string {
return msg.replace(
CREDENTIAL_URL_RE,
(_match, scheme: string) => `${scheme.toLowerCase()}://***@`,
);
}
@@ -0,0 +1,546 @@
/**
* Unit tests for tier-detection.ts.
*
* All external I/O (postgres, ioredis) is mocked — no live services required.
*
* Note on hoisting: vi.mock() factories are hoisted above all imports by vitest.
* Variables referenced inside factory callbacks must be declared via vi.hoisted()
* so they are available at hoist time.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
/* ------------------------------------------------------------------ */
/* Hoist shared mock state so factories can reference it */
/* ------------------------------------------------------------------ */
const mocks = vi.hoisted(() => {
const mockSqlFn = vi.fn();
const mockEnd = vi.fn().mockResolvedValue(undefined);
const mockPostgresConstructor = vi.fn(() => {
const sql = mockSqlFn as ReturnType<typeof mockSqlFn>;
(sql as unknown as Record<string, unknown>)['end'] = mockEnd;
return sql;
});
const mockRedisConnect = vi.fn().mockResolvedValue(undefined);
const mockRedisPing = vi.fn().mockResolvedValue('PONG');
const mockRedisDisconnect = vi.fn();
const MockRedis = vi.fn().mockImplementation(() => ({
connect: mockRedisConnect,
ping: mockRedisPing,
disconnect: mockRedisDisconnect,
}));
return {
mockSqlFn,
mockEnd,
mockPostgresConstructor,
mockRedisConnect,
mockRedisPing,
mockRedisDisconnect,
MockRedis,
};
});
/* ------------------------------------------------------------------ */
/* Module mocks (registered at hoist time) */
/* ------------------------------------------------------------------ */
vi.mock('postgres', () => ({
default: mocks.mockPostgresConstructor,
}));
vi.mock('ioredis', () => ({
Redis: mocks.MockRedis,
}));
/* ------------------------------------------------------------------ */
/* Import SUT after mocks are registered */
/* ------------------------------------------------------------------ */
import { detectAndAssertTier, probeServiceHealth, TierDetectionError } from './tier-detection.js';
import type { TierConfig } from './tier-detection.js';
/* ------------------------------------------------------------------ */
/* Config fixtures */
/* ------------------------------------------------------------------ */
const LOCAL_CONFIG: TierConfig = {
tier: 'local',
storage: { type: 'pglite', dataDir: '.mosaic/pglite' },
queue: { type: 'local' },
};
const STANDALONE_CONFIG: TierConfig = {
tier: 'standalone',
storage: { type: 'postgres', url: 'postgresql://mosaic:mosaic@db-host:5432/mosaic' },
queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
};
const FEDERATED_CONFIG: TierConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
enableVector: true,
},
queue: { type: 'bullmq', url: 'redis://valkey-host:6380' },
};
/* ------------------------------------------------------------------ */
/* Tests */
/* ------------------------------------------------------------------ */
describe('detectAndAssertTier', () => {
beforeEach(() => {
vi.clearAllMocks();
// Default: all probes succeed.
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockEnd.mockResolvedValue(undefined);
mocks.mockRedisConnect.mockResolvedValue(undefined);
mocks.mockRedisPing.mockResolvedValue('PONG');
// Re-wire constructor to return a fresh sql-like object each time.
mocks.mockPostgresConstructor.mockImplementation(() => {
const sql = mocks.mockSqlFn as ReturnType<typeof mocks.mockSqlFn>;
(sql as unknown as Record<string, unknown>)['end'] = mocks.mockEnd;
return sql;
});
mocks.MockRedis.mockImplementation(() => ({
connect: mocks.mockRedisConnect,
ping: mocks.mockRedisPing,
disconnect: mocks.mockRedisDisconnect,
}));
});
/* ---------------------------------------------------------------- */
/* 1. local — no-op */
/* ---------------------------------------------------------------- */
it('resolves immediately for tier=local without touching postgres or ioredis', async () => {
await expect(detectAndAssertTier(LOCAL_CONFIG)).resolves.toBeUndefined();
expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 2. standalone — happy path */
/* ---------------------------------------------------------------- */
it('resolves for tier=standalone when postgres and valkey are reachable', async () => {
await expect(detectAndAssertTier(STANDALONE_CONFIG)).resolves.toBeUndefined();
// Postgres was probed (SELECT 1 only — no pgvector check).
expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(1);
expect(mocks.mockSqlFn).toHaveBeenCalledTimes(1);
// Valkey was probed.
expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
expect(mocks.mockRedisPing).toHaveBeenCalledTimes(1);
});
/* ---------------------------------------------------------------- */
/* 3. standalone — postgres unreachable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=postgres when postgres query rejects', async () => {
mocks.mockSqlFn.mockRejectedValueOnce(new Error('connection refused'));
const promise = detectAndAssertTier(STANDALONE_CONFIG);
await expect(promise).rejects.toBeInstanceOf(TierDetectionError);
// Confirm no valkey probe happened (fail fast on first error).
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
it('sets service=postgres on the error when postgres fails', async () => {
mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
try {
await detectAndAssertTier(STANDALONE_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('postgres');
expect(typed.remediation).toContain('docker compose');
}
});
/* ---------------------------------------------------------------- */
/* 4. standalone — valkey unreachable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=valkey when ping fails', async () => {
// Postgres probe succeeds; valkey connect fails.
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockRedisConnect.mockRejectedValue(new Error('ECONNREFUSED'));
try {
await detectAndAssertTier(STANDALONE_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('valkey');
expect(typed.message).toContain('valkey');
expect(typed.remediation).toContain('valkey-federated');
}
});
/* ---------------------------------------------------------------- */
/* 5. federated — happy path */
/* ---------------------------------------------------------------- */
it('resolves for tier=federated when all three checks pass', async () => {
// SELECT 1 and CREATE EXTENSION both succeed.
mocks.mockSqlFn.mockResolvedValue([]);
await expect(detectAndAssertTier(FEDERATED_CONFIG)).resolves.toBeUndefined();
// postgres probe (SELECT 1) + pgvector probe (CREATE EXTENSION) = 2 postgres constructors.
expect(mocks.mockPostgresConstructor).toHaveBeenCalledTimes(2);
expect(mocks.mockSqlFn).toHaveBeenCalledTimes(2);
// Valkey probed once.
expect(mocks.MockRedis).toHaveBeenCalledTimes(1);
});
/* ---------------------------------------------------------------- */
/* 6. federated — pgvector not installable */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=pgvector when CREATE EXTENSION fails', async () => {
// SELECT 1 succeeds (first call), CREATE EXTENSION fails (second call).
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1
.mockRejectedValueOnce(new Error('extension "vector" is not available'));
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
expect(typed.message).toContain('pgvector');
expect(typed.remediation).toContain('pgvector/pgvector');
}
});
/* ---------------------------------------------------------------- */
/* 7. probeValkey honors connectTimeout and lazyConnect */
/* ---------------------------------------------------------------- */
it('constructs the ioredis Redis client with connectTimeout: 5000', async () => {
await detectAndAssertTier(STANDALONE_CONFIG);
expect(mocks.MockRedis).toHaveBeenCalledOnce();
expect(mocks.MockRedis).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({ connectTimeout: 5000, lazyConnect: true }),
);
});
/* ---------------------------------------------------------------- */
/* 8. probePgvector — library-not-installed remediation */
/* ---------------------------------------------------------------- */
it('includes pgvector/pgvector:pg17 in remediation when pgvector library is missing', async () => {
// SELECT 1 succeeds; CREATE EXTENSION fails with the canonical library-missing message.
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
.mockRejectedValueOnce(new Error('extension "vector" is not available')); // probePgvector
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
expect(typed.remediation).toContain('pgvector/pgvector:pg17');
}
});
/* ---------------------------------------------------------------- */
/* 9. probePgvector — permission / other error remediation */
/* ---------------------------------------------------------------- */
it('mentions CREATE permission or superuser in remediation for a generic pgvector error', async () => {
// SELECT 1 succeeds; CREATE EXTENSION fails with a permission error.
mocks.mockSqlFn
.mockResolvedValueOnce([]) // SELECT 1 (probePostgres)
.mockRejectedValueOnce(new Error('permission denied to create extension'));
try {
await detectAndAssertTier(FEDERATED_CONFIG);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('pgvector');
// Must NOT point to the image fix — that's only for the library-missing case.
expect(typed.remediation).not.toContain('pgvector/pgvector:pg17');
// Must mention permissions or superuser.
expect(typed.remediation).toMatch(/CREATE|superuser/i);
}
});
/* ---------------------------------------------------------------- */
/* 10. federated tier rejects non-bullmq queue.type */
/* ---------------------------------------------------------------- */
it('throws TierDetectionError with service=config for federated tier with queue.type !== bullmq', async () => {
const badConfig: TierConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
enableVector: true,
},
queue: { type: 'local' },
};
try {
await detectAndAssertTier(badConfig);
expect.fail('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(TierDetectionError);
const typed = err as TierDetectionError;
expect(typed.service).toBe('config');
expect(typed.remediation).toContain('bullmq');
}
// No network probes should have been attempted.
expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 11. Error fields populated */
/* ---------------------------------------------------------------- */
it('populates host, port, and remediation on a thrown TierDetectionError', async () => {
mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
let caught: TierDetectionError | undefined;
try {
await detectAndAssertTier(STANDALONE_CONFIG);
} catch (err) {
caught = err as TierDetectionError;
}
expect(caught).toBeInstanceOf(TierDetectionError);
expect(caught!.service).toBe('postgres');
// Host and port are extracted from the Postgres URL in STANDALONE_CONFIG.
expect(caught!.host).toBe('db-host');
expect(caught!.port).toBe(5432);
expect(caught!.remediation).toMatch(/docker compose/i);
expect(caught!.message).toContain('db-host:5432');
});
});
/* ------------------------------------------------------------------ */
/* probeServiceHealth tests */
/* ------------------------------------------------------------------ */
describe('probeServiceHealth', () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockEnd.mockResolvedValue(undefined);
mocks.mockRedisConnect.mockResolvedValue(undefined);
mocks.mockRedisPing.mockResolvedValue('PONG');
mocks.mockPostgresConstructor.mockImplementation(() => {
const sql = mocks.mockSqlFn as ReturnType<typeof mocks.mockSqlFn>;
(sql as unknown as Record<string, unknown>)['end'] = mocks.mockEnd;
return sql;
});
mocks.MockRedis.mockImplementation(() => ({
connect: mocks.mockRedisConnect,
ping: mocks.mockRedisPing,
disconnect: mocks.mockRedisDisconnect,
}));
});
/* ---------------------------------------------------------------- */
/* 12. local tier — all skipped, green */
/* ---------------------------------------------------------------- */
it('returns all services as skipped and overall green for local tier', async () => {
const report = await probeServiceHealth(LOCAL_CONFIG);
expect(report.tier).toBe('local');
expect(report.overall).toBe('green');
expect(report.services).toHaveLength(3);
for (const svc of report.services) {
expect(svc.status).toBe('skipped');
}
expect(mocks.mockPostgresConstructor).not.toHaveBeenCalled();
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 13. postgres fails, valkey ok → red */
/* ---------------------------------------------------------------- */
it('returns red overall with postgres fail and valkey ok for standalone when postgres fails', async () => {
mocks.mockSqlFn.mockRejectedValue(new Error('connection refused'));
const report = await probeServiceHealth(STANDALONE_CONFIG);
expect(report.overall).toBe('red');
const pgCheck = report.services.find((s) => s.name === 'postgres');
expect(pgCheck?.status).toBe('fail');
expect(pgCheck?.error).toBeDefined();
expect(pgCheck?.error?.remediation).toContain('docker compose');
const vkCheck = report.services.find((s) => s.name === 'valkey');
expect(vkCheck?.status).toBe('ok');
});
/* ---------------------------------------------------------------- */
/* 14. federated all green → 3 services ok */
/* ---------------------------------------------------------------- */
it('returns green overall with all 3 services ok for federated when all pass', async () => {
mocks.mockSqlFn.mockResolvedValue([]);
const report = await probeServiceHealth(FEDERATED_CONFIG);
expect(report.tier).toBe('federated');
expect(report.overall).toBe('green');
expect(report.services).toHaveLength(3);
for (const svc of report.services) {
expect(svc.status).toBe('ok');
}
});
/* ---------------------------------------------------------------- */
/* 15. durationMs is a non-negative number for every service check */
/* ---------------------------------------------------------------- */
it('sets durationMs as a non-negative number for every service check', async () => {
mocks.mockSqlFn.mockResolvedValue([]);
const report = await probeServiceHealth(FEDERATED_CONFIG);
for (const svc of report.services) {
expect(typeof svc.durationMs).toBe('number');
expect(svc.durationMs).toBeGreaterThanOrEqual(0);
}
});
it('sets durationMs >= 0 even for skipped services (local tier)', async () => {
const report = await probeServiceHealth(LOCAL_CONFIG);
for (const svc of report.services) {
expect(typeof svc.durationMs).toBe('number');
expect(svc.durationMs).toBeGreaterThanOrEqual(0);
}
});
/* ---------------------------------------------------------------- */
/* 16. configPath is passed through to the report */
/* ---------------------------------------------------------------- */
it('includes configPath in the report when provided', async () => {
const report = await probeServiceHealth(LOCAL_CONFIG, '/etc/mosaic/mosaic.config.json');
expect(report.configPath).toBe('/etc/mosaic/mosaic.config.json');
});
/* ---------------------------------------------------------------- */
/* 17. standalone — valkey fails, postgres ok → red */
/* ---------------------------------------------------------------- */
it('returns red with valkey fail and postgres ok for standalone when valkey fails', async () => {
mocks.mockSqlFn.mockResolvedValue([]);
mocks.mockRedisConnect.mockRejectedValue(new Error('ECONNREFUSED'));
const report = await probeServiceHealth(STANDALONE_CONFIG);
expect(report.overall).toBe('red');
const pgCheck = report.services.find((s) => s.name === 'postgres');
expect(pgCheck?.status).toBe('ok');
const vkCheck = report.services.find((s) => s.name === 'valkey');
expect(vkCheck?.status).toBe('fail');
expect(vkCheck?.error).toBeDefined();
});
/* ---------------------------------------------------------------- */
/* 18. federated — pgvector fails → red with remediation */
/* ---------------------------------------------------------------- */
it('returns red with pgvector fail for federated when pgvector probe fails', async () => {
mocks.mockSqlFn
.mockResolvedValueOnce([]) // postgres SELECT 1
.mockRejectedValueOnce(new Error('extension "vector" is not available')); // pgvector
const report = await probeServiceHealth(FEDERATED_CONFIG);
expect(report.overall).toBe('red');
const pvCheck = report.services.find((s) => s.name === 'pgvector');
expect(pvCheck?.status).toBe('fail');
expect(pvCheck?.error?.remediation).toContain('pgvector/pgvector:pg17');
});
/* ---------------------------------------------------------------- */
/* 19. federated — non-bullmq queue → red config error, no network */
/* ---------------------------------------------------------------- */
it('returns red overall with config error when federated tier has non-bullmq queue (no network call)', async () => {
const federatedBadQueueConfig: TierConfig = {
tier: 'federated',
storage: {
type: 'postgres',
url: 'postgresql://mosaic:mosaic@db-host:5433/mosaic',
enableVector: true,
},
queue: { type: 'local' },
};
const report = await probeServiceHealth(federatedBadQueueConfig);
expect(report.overall).toBe('red');
const valkey = report.services.find((s) => s.name === 'valkey');
expect(valkey?.status).toBe('fail');
expect(valkey?.error?.remediation).toMatch(/bullmq/i);
// Critically: no network call was made — MockRedis constructor must NOT have been called.
expect(mocks.MockRedis).not.toHaveBeenCalled();
});
/* ---------------------------------------------------------------- */
/* 20. durationMs actually measures real elapsed time */
/* ---------------------------------------------------------------- */
it('measures real elapsed time for service probes', async () => {
const DELAY_MS = 25;
// Make the postgres mock introduce a real wall-clock delay.
mocks.mockSqlFn.mockImplementation(
() =>
new Promise((resolve) =>
setTimeout(() => {
resolve([]);
}, DELAY_MS),
),
);
const report = await probeServiceHealth(STANDALONE_CONFIG);
const pgCheck = report.services.find((s) => s.name === 'postgres');
expect(pgCheck).toBeDefined();
// Must be >= 20ms (small slack for jitter). Would be 0 if timer were stubbed.
expect(pgCheck!.durationMs).toBeGreaterThanOrEqual(20);
});
});
+559
View File
@@ -0,0 +1,559 @@
/**
* Tier Detection — pre-flight service reachability probes.
*
* Lifted from apps/gateway/src/bootstrap/tier-detector.ts so both the gateway
* and the mosaic CLI can share the same probe logic without duplicating code or
* creating circular workspace dependencies.
*
* Library choices:
* - Postgres: `postgres` npm package (already a dep via @mosaicstack/db / drizzle-orm).
* - Valkey: `ioredis` (compatible with Valkey; same URL convention used by bullmq).
*/
import postgres from 'postgres';
import { Redis } from 'ioredis';
import { redactErrMsg } from './redact-error.js';
/* ------------------------------------------------------------------ */
/* Local structural type — avoids circular dependency */
/* ------------------------------------------------------------------ */
/**
* Minimal structural shape required for tier detection.
* Mirrors the relevant fields of MosaicConfig (from @mosaicstack/config) without
* creating a dependency cycle (config depends on storage for StorageConfig).
* Any object that satisfies MosaicConfig also satisfies this type.
*/
export interface TierConfig {
tier: 'local' | 'standalone' | 'federated';
storage:
| { type: 'pglite'; dataDir?: string }
| { type: 'postgres'; url: string; enableVector?: boolean }
| { type: 'files'; dataDir: string; format?: 'json' | 'md' };
queue: { type: string; url?: string };
}
/* ------------------------------------------------------------------ */
/* Public types */
/* ------------------------------------------------------------------ */
export interface ServiceCheck {
name: 'postgres' | 'valkey' | 'pgvector';
status: 'ok' | 'fail' | 'skipped';
host?: string;
port?: number;
durationMs: number;
error?: { message: string; remediation: string };
}
export interface TierHealthReport {
tier: 'local' | 'standalone' | 'federated';
configPath?: string;
overall: 'green' | 'yellow' | 'red';
services: ServiceCheck[];
}
/* ------------------------------------------------------------------ */
/* Structured error type */
/* ------------------------------------------------------------------ */
export class TierDetectionError extends Error {
public readonly service: 'postgres' | 'valkey' | 'pgvector' | 'config';
public readonly host: string;
public readonly port: number;
public readonly remediation: string;
constructor(opts: {
service: 'postgres' | 'valkey' | 'pgvector' | 'config';
host: string;
port: number;
remediation: string;
cause?: unknown;
}) {
const message =
`[tier-detector] ${opts.service} unreachable or unusable at ` +
`${opts.host}:${opts.port}${opts.remediation}`;
super(message, { cause: opts.cause });
this.name = 'TierDetectionError';
this.service = opts.service;
this.host = opts.host;
this.port = opts.port;
this.remediation = opts.remediation;
}
}
/* ------------------------------------------------------------------ */
/* URL helpers */
/* ------------------------------------------------------------------ */
/** Extract host and port from a URL string, returning safe fallbacks on parse failure. */
function parseHostPort(url: string, defaultPort: number): { host: string; port: number } {
try {
const parsed = new URL(url);
const host = parsed.hostname || 'unknown';
const port = parsed.port ? parseInt(parsed.port, 10) : defaultPort;
return { host, port };
} catch {
return { host: 'unknown', port: defaultPort };
}
}
/* ------------------------------------------------------------------ */
/* Internal probe results */
/* ------------------------------------------------------------------ */
interface ProbeResult {
host: string;
port: number;
durationMs: number;
error?: { message: string; remediation: string };
}
/* ------------------------------------------------------------------ */
/* Postgres probe */
/* ------------------------------------------------------------------ */
async function probePostgres(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 5432);
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
// Run a trivial query to confirm connectivity.
await sql`SELECT 1`;
} catch (cause) {
throw new TierDetectionError({
service: 'postgres',
host,
port,
remediation:
'Start Postgres: `docker compose -f docker-compose.federated.yml --profile federated up -d postgres-federated`',
cause,
});
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {
// Ignore cleanup errors — we already have what we need.
});
}
}
}
async function probePostgresMeasured(url: string): Promise<ProbeResult> {
const { host, port } = parseHostPort(url, 5432);
const start = Date.now();
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
await sql`SELECT 1`;
return { host, port, durationMs: Date.now() - start };
} catch (cause) {
return {
host,
port,
durationMs: Date.now() - start,
error: {
message: redactErrMsg(cause instanceof Error ? cause.message : String(cause)),
remediation:
'Start Postgres: `docker compose -f docker-compose.federated.yml --profile federated up -d postgres-federated`',
},
};
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {});
}
}
}
/* ------------------------------------------------------------------ */
/* pgvector probe */
/* ------------------------------------------------------------------ */
async function probePgvector(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 5432);
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
// This succeeds whether the extension is already installed or freshly created.
// It errors only if the pgvector shared library is missing from the Postgres binary.
await sql`CREATE EXTENSION IF NOT EXISTS vector`;
} catch (cause) {
const causeMsg = cause instanceof Error ? cause.message.toLowerCase() : '';
const isLibraryMissing = causeMsg.includes('extension "vector" is not available');
const remediation = isLibraryMissing
? 'Use the `pgvector/pgvector:pg17` image, not the stock `postgres:17` image. See `docker-compose.federated.yml`.'
: 'The database role lacks permission to CREATE EXTENSION. Grant `CREATE` on the database, or run as a superuser.';
throw new TierDetectionError({
service: 'pgvector',
host,
port,
remediation,
cause,
});
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {
// Ignore cleanup errors.
});
}
}
}
async function probePgvectorMeasured(url: string): Promise<ProbeResult> {
const { host, port } = parseHostPort(url, 5432);
const start = Date.now();
let sql: ReturnType<typeof postgres> | undefined;
try {
sql = postgres(url, {
max: 1,
connect_timeout: 5,
idle_timeout: 5,
});
await sql`CREATE EXTENSION IF NOT EXISTS vector`;
return { host, port, durationMs: Date.now() - start };
} catch (cause) {
const causeMsg = cause instanceof Error ? cause.message.toLowerCase() : '';
const isLibraryMissing = causeMsg.includes('extension "vector" is not available');
const remediation = isLibraryMissing
? 'Use the `pgvector/pgvector:pg17` image, not the stock `postgres:17` image. See `docker-compose.federated.yml`.'
: 'The database role lacks permission to CREATE EXTENSION. Grant `CREATE` on the database, or run as a superuser.';
return {
host,
port,
durationMs: Date.now() - start,
error: {
message: redactErrMsg(cause instanceof Error ? cause.message : String(cause)),
remediation,
},
};
} finally {
if (sql) {
await sql.end({ timeout: 2 }).catch(() => {});
}
}
}
/* ------------------------------------------------------------------ */
/* Valkey probe */
/* ------------------------------------------------------------------ */
const DEFAULT_VALKEY_URL = 'redis://localhost:6380';
async function probeValkey(url: string): Promise<void> {
const { host, port } = parseHostPort(url, 6380);
const client = new Redis(url, {
enableReadyCheck: false,
maxRetriesPerRequest: 0,
retryStrategy: () => null, // no retries — fail fast
lazyConnect: true,
connectTimeout: 5000, // fail-fast: 5-second hard cap on connection attempt
});
try {
await client.connect();
const pong = await client.ping();
if (pong !== 'PONG') {
throw new Error(`Unexpected PING response: ${pong}`);
}
} catch (cause) {
throw new TierDetectionError({
service: 'valkey',
host,
port,
remediation:
'Start Valkey: `docker compose -f docker-compose.federated.yml --profile federated up -d valkey-federated`',
cause,
});
} finally {
client.disconnect();
}
}
async function probeValkeyMeasured(url: string): Promise<ProbeResult> {
const { host, port } = parseHostPort(url, 6380);
const start = Date.now();
const client = new Redis(url, {
enableReadyCheck: false,
maxRetriesPerRequest: 0,
retryStrategy: () => null,
lazyConnect: true,
connectTimeout: 5000,
});
try {
await client.connect();
const pong = await client.ping();
if (pong !== 'PONG') {
throw new Error(`Unexpected PING response: ${pong}`);
}
return { host, port, durationMs: Date.now() - start };
} catch (cause) {
return {
host,
port,
durationMs: Date.now() - start,
error: {
message: redactErrMsg(cause instanceof Error ? cause.message : String(cause)),
remediation:
'Start Valkey: `docker compose -f docker-compose.federated.yml --profile federated up -d valkey-federated`',
},
};
} finally {
client.disconnect();
}
}
/* ------------------------------------------------------------------ */
/* Public entry points */
/* ------------------------------------------------------------------ */
/**
* Assert that all services required by `config.tier` are reachable.
*
* - `local` — no-op (PGlite is in-process; no external services).
* - `standalone` — assert Postgres + Valkey (if queue.type === 'bullmq').
* - `federated` — assert Postgres + Valkey + pgvector installability.
*
* Throws `TierDetectionError` on the first failure with host:port and
* a remediation hint.
*/
export async function detectAndAssertTier(config: TierConfig): Promise<void> {
if (config.tier === 'local') {
// PGlite runs in-process — nothing to probe.
return;
}
const pgUrl =
config.storage.type === 'postgres' ? config.storage.url : 'postgresql://localhost:5432/mosaic';
const valkeyUrl =
config.queue.type === 'bullmq' ? (config.queue.url ?? DEFAULT_VALKEY_URL) : null;
if (config.tier === 'standalone') {
await probePostgres(pgUrl);
if (valkeyUrl) {
await probeValkey(valkeyUrl);
}
return;
}
// tier === 'federated'
// Reject misconfigured queue upfront — federated requires bullmq + a Valkey URL.
if (config.queue.type !== 'bullmq') {
throw new TierDetectionError({
service: 'config',
host: 'localhost',
port: 0,
remediation:
"Federated tier requires queue.type === 'bullmq'. " +
"Set queue: { type: 'bullmq', url: 'redis://...' } in your mosaic.config.json.",
});
}
const federatedValkeyUrl = config.queue.url ?? DEFAULT_VALKEY_URL;
await probePostgres(pgUrl);
await probeValkey(federatedValkeyUrl);
await probePgvector(pgUrl);
}
/**
* Non-throwing variant for `mosaic gateway doctor`.
*
* Probes ALL required services even if some fail, returning a structured report.
* Services not required for the current tier are reported as `skipped`.
*
* Overall status:
* - `green` — all required services OK
* - `yellow` — all required services OK, but a non-critical check failed
* (currently unused — reserved for future optional probes)
* - `red` — at least one required service failed
*/
export async function probeServiceHealth(
config: TierConfig,
configPath?: string,
): Promise<TierHealthReport> {
const tier = config.tier;
// local tier: PGlite is in-process, no external services needed.
if (tier === 'local') {
return {
tier,
configPath,
overall: 'green',
services: [
{ name: 'postgres', status: 'skipped', durationMs: 0 },
{ name: 'valkey', status: 'skipped', durationMs: 0 },
{ name: 'pgvector', status: 'skipped', durationMs: 0 },
],
};
}
const pgUrl =
config.storage.type === 'postgres' ? config.storage.url : 'postgresql://localhost:5432/mosaic';
const valkeyUrl =
config.queue.type === 'bullmq' ? (config.queue.url ?? DEFAULT_VALKEY_URL) : null;
const services: ServiceCheck[] = [];
let hasFailure = false;
if (tier === 'standalone') {
// Postgres — required
const pgResult = await probePostgresMeasured(pgUrl);
if (pgResult.error) {
hasFailure = true;
services.push({
name: 'postgres',
status: 'fail',
host: pgResult.host,
port: pgResult.port,
durationMs: pgResult.durationMs,
error: pgResult.error,
});
} else {
services.push({
name: 'postgres',
status: 'ok',
host: pgResult.host,
port: pgResult.port,
durationMs: pgResult.durationMs,
});
}
// Valkey — required if bullmq
if (valkeyUrl) {
const vkResult = await probeValkeyMeasured(valkeyUrl);
if (vkResult.error) {
hasFailure = true;
services.push({
name: 'valkey',
status: 'fail',
host: vkResult.host,
port: vkResult.port,
durationMs: vkResult.durationMs,
error: vkResult.error,
});
} else {
services.push({
name: 'valkey',
status: 'ok',
host: vkResult.host,
port: vkResult.port,
durationMs: vkResult.durationMs,
});
}
} else {
services.push({ name: 'valkey', status: 'skipped', durationMs: 0 });
}
// pgvector — not required for standalone
services.push({ name: 'pgvector', status: 'skipped', durationMs: 0 });
return {
tier,
configPath,
overall: hasFailure ? 'red' : 'green',
services,
};
}
// tier === 'federated'
// Postgres — required
const pgResult = await probePostgresMeasured(pgUrl);
if (pgResult.error) {
hasFailure = true;
services.push({
name: 'postgres',
status: 'fail',
host: pgResult.host,
port: pgResult.port,
durationMs: pgResult.durationMs,
error: pgResult.error,
});
} else {
services.push({
name: 'postgres',
status: 'ok',
host: pgResult.host,
port: pgResult.port,
durationMs: pgResult.durationMs,
});
}
// Valkey — required for federated (queue.type must be bullmq)
if (config.queue.type !== 'bullmq') {
hasFailure = true;
services.push({
name: 'valkey',
status: 'fail',
host: 'localhost',
port: 0,
durationMs: 0,
error: {
message: "Federated tier requires queue.type === 'bullmq'",
remediation:
"Set queue: { type: 'bullmq', url: 'redis://...' } in your mosaic.config.json.",
},
});
} else {
const federatedValkeyUrl = config.queue.url ?? DEFAULT_VALKEY_URL;
const vkResult = await probeValkeyMeasured(federatedValkeyUrl);
if (vkResult.error) {
hasFailure = true;
services.push({
name: 'valkey',
status: 'fail',
host: vkResult.host,
port: vkResult.port,
durationMs: vkResult.durationMs,
error: vkResult.error,
});
} else {
services.push({
name: 'valkey',
status: 'ok',
host: vkResult.host,
port: vkResult.port,
durationMs: vkResult.durationMs,
});
}
}
// pgvector — required for federated
const pvResult = await probePgvectorMeasured(pgUrl);
if (pvResult.error) {
hasFailure = true;
services.push({
name: 'pgvector',
status: 'fail',
host: pvResult.host,
port: pvResult.port,
durationMs: pvResult.durationMs,
error: pvResult.error,
});
} else {
services.push({
name: 'pgvector',
status: 'ok',
host: pvResult.host,
port: pvResult.port,
durationMs: pvResult.durationMs,
});
}
return {
tier,
configPath,
overall: hasFailure ? 'red' : 'green',
services,
};
}
+43
View File
@@ -0,0 +1,43 @@
export interface StorageAdapter {
readonly name: string;
create<T extends Record<string, unknown>>(
collection: string,
data: T,
): Promise<T & { id: string }>;
read<T extends Record<string, unknown>>(collection: string, id: string): Promise<T | null>;
update(collection: string, id: string, data: Record<string, unknown>): Promise<boolean>;
delete(collection: string, id: string): Promise<boolean>;
find<T extends Record<string, unknown>>(
collection: string,
filter?: Record<string, unknown>,
opts?: {
limit?: number;
offset?: number;
orderBy?: string;
order?: 'asc' | 'desc';
},
): Promise<T[]>;
findOne<T extends Record<string, unknown>>(
collection: string,
filter: Record<string, unknown>,
): Promise<T | null>;
count(collection: string, filter?: Record<string, unknown>): Promise<number>;
transaction<T>(fn: (tx: StorageAdapter) => Promise<T>): Promise<T>;
migrate(): Promise<void>;
close(): Promise<void>;
}
export type StorageConfig =
| { type: 'postgres'; url: string; enableVector?: boolean }
| { type: 'pglite'; dataDir?: string }
| { type: 'files'; dataDir: string; format?: 'json' | 'md' };
+9
View File
@@ -0,0 +1,9 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src"
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}
+8
View File
@@ -0,0 +1,8 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
environment: 'node',
},
});