test(storage): integration test for migrate-tier (FED-M1-08) + camelCase column fix (#477)

packages/storage/package.json

@@ -6,6 +6,7 @@
     "url": "https://git.mosaicstack.dev/mosaicstack/stack.git",
     "directory": "packages/storage"
   },
+  "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
@@ -29,6 +30,7 @@
     "postgres": "^3.4.8"
   },
   "devDependencies": {
+    "drizzle-orm": "^0.45.1",
     "typescript": "^5.8.0",
     "vitest": "^2.0.0"
   },

packages/storage/src/migrate-tier.integration.test.ts

@@ -0,0 +1,324 @@
+/**
+ * FED-M1-08 — Integration test: PGlite → federated Postgres+pgvector migration.
+ *
+ * Prereq: docker compose -f docker-compose.federated.yml --profile federated up -d
+ * Run:    FEDERATED_INTEGRATION=1 pnpm --filter @mosaicstack/storage test src/migrate-tier.integration.test.ts
+ *
+ * Skipped when FEDERATED_INTEGRATION !== '1'.
+ *
+ * Strategy: users.id (TEXT PK) uses the recognisable prefix `fed-m1-08-` for
+ * easy cleanup. UUID-PKed tables (teams, conversations, messages, team_members)
+ * use deterministic valid UUIDs in the `f0000xxx-…` namespace. Cleanup is
+ * explicit DELETE by id — no full-table truncation.
+ */
+
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { users, teams, teamMembers, conversations, messages } from '@mosaicstack/db';
+import { createPgliteDbWithVector, runPgliteMigrations } from './test-utils/pglite-with-vector.js';
+
+import postgres from 'postgres';
+import { afterAll, describe, expect, it } from 'vitest';
+
+import { DrizzleMigrationSource, PostgresMigrationTarget, runMigrateTier } from './migrate-tier.js';
+
+/* ------------------------------------------------------------------ */
+/*  Constants                                                           */
+/* ------------------------------------------------------------------ */
+
+const run = process.env['FEDERATED_INTEGRATION'] === '1';
+
+const FEDERATED_PG_URL = 'postgresql://mosaic:mosaic@localhost:5433/mosaic';
+
+/**
+ * Deterministic IDs for the test's seed data.
+ *
+ * users.id is TEXT (any string) — we use a recognisable prefix for easy cleanup.
+ * All other tables use UUID primary keys — must be valid UUID v4 format.
+ * The 4th segment starts with '4' (version 4) and 5th starts with '8' (variant).
+ */
+const IDS = {
+  // text PK — can be any string
+  user1: 'fed-m1-08-user-1',
+  user2: 'fed-m1-08-user-2',
+  // UUID PKs — must be valid UUID format
+  team1: 'f0000001-0000-4000-8000-000000000001',
+  teamMember1: 'f0000002-0000-4000-8000-000000000001',
+  teamMember2: 'f0000002-0000-4000-8000-000000000002',
+  conv1: 'f0000003-0000-4000-8000-000000000001',
+  conv2: 'f0000003-0000-4000-8000-000000000002',
+  msg1: 'f0000004-0000-4000-8000-000000000001',
+  msg2: 'f0000004-0000-4000-8000-000000000002',
+  msg3: 'f0000004-0000-4000-8000-000000000003',
+  msg4: 'f0000004-0000-4000-8000-000000000004',
+  msg5: 'f0000004-0000-4000-8000-000000000005',
+} as const;
+
+/* ------------------------------------------------------------------ */
+/*  Shared handles for afterAll cleanup                                 */
+/* ------------------------------------------------------------------ */
+
+let targetSql: ReturnType<typeof postgres> | undefined;
+let pgliteDataDir: string | undefined;
+
+afterAll(async () => {
+  if (targetSql) {
+    await cleanTarget(targetSql).catch(() => {});
+    await targetSql.end({ timeout: 5 }).catch(() => {});
+  }
+  if (pgliteDataDir) {
+    await fs.rm(pgliteDataDir, { recursive: true, force: true }).catch(() => {});
+  }
+});
+
+/* ------------------------------------------------------------------ */
+/*  Helpers                                                             */
+/* ------------------------------------------------------------------ */
+
+/** Delete all test-owned rows from target in safe FK order. */
+async function cleanTarget(sql: ReturnType<typeof postgres>): Promise<void> {
+  // Reverse FK order: messages → conversations → team_members → teams → users
+  await sql.unsafe(`DELETE FROM messages WHERE id = ANY($1)`, [
+    [IDS.msg1, IDS.msg2, IDS.msg3, IDS.msg4, IDS.msg5],
+  ] as never[]);
+  await sql.unsafe(`DELETE FROM conversations WHERE id = ANY($1)`, [
+    [IDS.conv1, IDS.conv2],
+  ] as never[]);
+  await sql.unsafe(`DELETE FROM team_members WHERE id = ANY($1)`, [
+    [IDS.teamMember1, IDS.teamMember2],
+  ] as never[]);
+  await sql.unsafe(`DELETE FROM teams WHERE id = $1`, [IDS.team1] as never[]);
+  await sql.unsafe(`DELETE FROM users WHERE id = ANY($1)`, [[IDS.user1, IDS.user2]] as never[]);
+}
+
+/* ------------------------------------------------------------------ */
+/*  Test suite                                                          */
+/* ------------------------------------------------------------------ */
+
+describe.skipIf(!run)('migrate-tier — PGlite → federated PG', () => {
+  it('seeds PGlite, runs migrate-tier, asserts row counts and sample rows on target', async () => {
+    /* ---- 1. Create a temp PGlite db ---------------------------------- */
+
+    pgliteDataDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fed-m1-08-'));
+    const handle = createPgliteDbWithVector(pgliteDataDir);
+
+    // Run Drizzle migrations against PGlite.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await runPgliteMigrations(handle.db as any);
+
+    /* ---- 2. Seed representative data --------------------------------- */
+
+    const now = new Date();
+    const db = handle.db;
+
+    // users (2 rows)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await (db as any).insert(users).values([
+      {
+        id: IDS.user1,
+        name: 'Fed Test User One',
+        email: 'fed-m1-08-user1@test.invalid',
+        emailVerified: false,
+        role: 'member',
+        createdAt: now,
+        updatedAt: now,
+      },
+      {
+        id: IDS.user2,
+        name: 'Fed Test User Two',
+        email: 'fed-m1-08-user2@test.invalid',
+        emailVerified: false,
+        role: 'member',
+        createdAt: now,
+        updatedAt: now,
+      },
+    ]);
+
+    // teams (1 row)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await (db as any).insert(teams).values([
+      {
+        id: IDS.team1,
+        name: 'Fed M1-08 Team',
+        slug: 'fed-m1-08-team',
+        ownerId: IDS.user1,
+        managerId: IDS.user1,
+        createdAt: now,
+        updatedAt: now,
+      },
+    ]);
+
+    // team_members (2 rows linking both users to the team)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await (db as any).insert(teamMembers).values([
+      {
+        id: IDS.teamMember1,
+        teamId: IDS.team1,
+        userId: IDS.user1,
+        role: 'manager',
+        joinedAt: now,
+      },
+      {
+        id: IDS.teamMember2,
+        teamId: IDS.team1,
+        userId: IDS.user2,
+        role: 'member',
+        joinedAt: now,
+      },
+    ]);
+
+    // conversations (2 rows)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await (db as any).insert(conversations).values([
+      {
+        id: IDS.conv1,
+        title: 'Fed M1-08 Conversation Alpha',
+        userId: IDS.user1,
+        archived: false,
+        createdAt: now,
+        updatedAt: now,
+      },
+      {
+        id: IDS.conv2,
+        title: 'Fed M1-08 Conversation Beta',
+        userId: IDS.user2,
+        archived: false,
+        createdAt: now,
+        updatedAt: now,
+      },
+    ]);
+
+    // messages (5 rows across both conversations)
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    await (db as any).insert(messages).values([
+      {
+        id: IDS.msg1,
+        conversationId: IDS.conv1,
+        role: 'user',
+        content: 'Hello from conv1 msg1',
+        createdAt: now,
+      },
+      {
+        id: IDS.msg2,
+        conversationId: IDS.conv1,
+        role: 'assistant',
+        content: 'Reply in conv1 msg2',
+        createdAt: now,
+      },
+      {
+        id: IDS.msg3,
+        conversationId: IDS.conv1,
+        role: 'user',
+        content: 'Follow-up in conv1 msg3',
+        createdAt: now,
+      },
+      {
+        id: IDS.msg4,
+        conversationId: IDS.conv2,
+        role: 'user',
+        content: 'Hello from conv2 msg4',
+        createdAt: now,
+      },
+      {
+        id: IDS.msg5,
+        conversationId: IDS.conv2,
+        role: 'assistant',
+        content: 'Reply in conv2 msg5',
+        createdAt: now,
+      },
+    ]);
+
+    /* ---- 3. Pre-clean the target so the test is repeatable ----------- */
+
+    targetSql = postgres(FEDERATED_PG_URL, {
+      max: 3,
+      connect_timeout: 10,
+      idle_timeout: 30,
+    });
+
+    await cleanTarget(targetSql);
+
+    /* ---- 4. Build source / target adapters and run migration --------- */
+
+    const source = new DrizzleMigrationSource(db, /* sourceHasVector= */ false);
+    const target = new PostgresMigrationTarget(FEDERATED_PG_URL);
+
+    try {
+      await runMigrateTier(
+        source,
+        target,
+        {
+          targetUrl: FEDERATED_PG_URL,
+          dryRun: false,
+          allowNonEmpty: true,
+          batchSize: 500,
+          onProgress: (_msg) => {
+            // Uncomment for debugging: console.log(_msg);
+          },
+        },
+        /* sourceHasVector= */ false,
+      );
+    } finally {
+      await target.close();
+    }
+
+    /* ---- 5. Assert: row counts in target match seed ------------------ */
+
+    const countUsers = await targetSql.unsafe<Array<{ n: string }>>(
+      `SELECT COUNT(*)::text AS n FROM users WHERE id = ANY($1)`,
+      [[IDS.user1, IDS.user2]] as never[],
+    );
+    expect(Number(countUsers[0]?.n)).toBe(2);
+
+    const countTeams = await targetSql.unsafe<Array<{ n: string }>>(
+      `SELECT COUNT(*)::text AS n FROM teams WHERE id = $1`,
+      [IDS.team1] as never[],
+    );
+    expect(Number(countTeams[0]?.n)).toBe(1);
+
+    const countTeamMembers = await targetSql.unsafe<Array<{ n: string }>>(
+      `SELECT COUNT(*)::text AS n FROM team_members WHERE id = ANY($1)`,
+      [[IDS.teamMember1, IDS.teamMember2]] as never[],
+    );
+    expect(Number(countTeamMembers[0]?.n)).toBe(2);
+
+    const countConvs = await targetSql.unsafe<Array<{ n: string }>>(
+      `SELECT COUNT(*)::text AS n FROM conversations WHERE id = ANY($1)`,
+      [[IDS.conv1, IDS.conv2]] as never[],
+    );
+    expect(Number(countConvs[0]?.n)).toBe(2);
+
+    const countMsgs = await targetSql.unsafe<Array<{ n: string }>>(
+      `SELECT COUNT(*)::text AS n FROM messages WHERE id = ANY($1)`,
+      [[IDS.msg1, IDS.msg2, IDS.msg3, IDS.msg4, IDS.msg5]] as never[],
+    );
+    expect(Number(countMsgs[0]?.n)).toBe(5);
+
+    /* ---- 6. Assert: sample row field values --------------------------- */
+
+    // User 1: check email and name
+    const userRows = await targetSql.unsafe<Array<{ id: string; email: string; name: string }>>(
+      `SELECT id, email, name FROM users WHERE id = $1`,
+      [IDS.user1] as never[],
+    );
+    expect(userRows[0]?.email).toBe('fed-m1-08-user1@test.invalid');
+    expect(userRows[0]?.name).toBe('Fed Test User One');
+
+    // Conversation 1: check title and user_id
+    const convRows = await targetSql.unsafe<Array<{ id: string; title: string; user_id: string }>>(
+      `SELECT id, title, user_id FROM conversations WHERE id = $1`,
+      [IDS.conv1] as never[],
+    );
+    expect(convRows[0]?.title).toBe('Fed M1-08 Conversation Alpha');
+    expect(convRows[0]?.user_id).toBe(IDS.user1);
+
+    /* ---- 7. Cleanup: delete test rows from target -------------------- */
+
+    await cleanTarget(targetSql);
+
+    // Close PGlite
+    await handle.close();
+  }, 60_000);
+});

packages/storage/src/migrate-tier.ts

@@ -491,11 +491,24 @@ export class PostgresMigrationTarget implements MigrationTarget {
 /*  Source-row normalisation                                           */
 /* ------------------------------------------------------------------ */
 
+/**
+ * Convert a camelCase key to snake_case.
+ * e.g. "userId" → "user_id", "emailVerified" → "email_verified".
+ * Keys that are already snake_case (no uppercase letters) are returned as-is.
+ */
+function toSnakeCase(key: string): string {
+  return key.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`);
+}
+
 /**
  * Drizzle returns rows as camelCase TypeScript objects (e.g. `userId`, not
  * `user_id`). The PostgresMigrationTarget upserts via raw SQL and uses the
- * column names as given — the `insights` no-vector path uses snake_case column
- * aliases in the SELECT, so those rows already arrive as snake_case.
+ * column names as given.  We must convert camelCase keys → snake_case before
+ * building the INSERT statement so column names match the PG schema.
+ *
+ * Exception: the `insights` no-vector path already returns snake_case keys
+ * from its raw SQL projection — toSnakeCase() is idempotent for already-
+ * snake_case keys so this conversion is safe in all paths.
  *
  * For vector tables (insights), if `embedding` is absent from the source row
  * (because DrizzleMigrationSource omitted it in the no-vector projection), we
@@ -509,7 +522,11 @@ export function normaliseSourceRow(
   row: Record<string, unknown>,
   sourceHasVector: boolean,
 ): Record<string, unknown> {
-  const out = { ...row };
+  // Convert all camelCase keys to snake_case for raw-SQL target compatibility.
+  const out: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(row)) {
+    out[toSnakeCase(k)] = v;
+  }
 
   if (VECTOR_TABLES.has(tableName) && !sourceHasVector) {
     // Source cannot have embeddings — explicitly null them so ON CONFLICT

packages/storage/src/test-utils/pglite-with-vector.ts

@@ -0,0 +1,52 @@
+/**
+ * Test-only helpers for creating a PGlite database with the pgvector extension
+ * and running Drizzle migrations against it.
+ *
+ * These are intentionally NOT exported from @mosaicstack/db to avoid pulling
+ * the WASM vector bundle into the public API surface.
+ */
+
+import { createRequire } from 'node:module';
+import { dirname, resolve } from 'node:path';
+
+import { PGlite } from '@electric-sql/pglite';
+import { vector } from '@electric-sql/pglite/vector';
+import { drizzle } from 'drizzle-orm/pglite';
+import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator';
+import type { PgliteDatabase } from 'drizzle-orm/pglite';
+import * as schema from '@mosaicstack/db';
+import type { DbHandle } from '@mosaicstack/db';
+
+/**
+ * Create a PGlite DB handle with the pgvector extension loaded.
+ * Required for running Drizzle migrations that include `CREATE EXTENSION vector`.
+ */
+export function createPgliteDbWithVector(dataDir: string): DbHandle {
+  const client = new PGlite(dataDir, { extensions: { vector } });
+  const db = drizzle(client, { schema });
+  return {
+    db: db as unknown as DbHandle['db'],
+    close: async () => {
+      await client.close();
+    },
+  };
+}
+
+/**
+ * Run Drizzle migrations against an already-open PGlite database handle.
+ * Resolves the migrations folder from @mosaicstack/db's installed location.
+ *
+ * @param db  A PgliteDatabase instance (from drizzle-orm/pglite).
+ */
+export async function runPgliteMigrations(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  db: PgliteDatabase<any>,
+): Promise<void> {
+  // Resolve @mosaicstack/db package root to locate its drizzle migrations folder.
+  const _require = createRequire(import.meta.url);
+  const dbPkgMain = _require.resolve('@mosaicstack/db');
+  // dbPkgMain → …/packages/db/dist/index.js  →  dirname = dist/
+  // go up one level from dist/ to find the sibling drizzle/ folder
+  const migrationsFolder = resolve(dirname(dbPkgMain), '../drizzle');
+  await migratePglite(db, { migrationsFolder });
+}