From 943a797a99a1446983c154c04e3cbda42fdad0d4 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Fri, 13 Mar 2026 08:46:40 -0500 Subject: [PATCH] =?UTF-8?q?feat(P4-001):=20@mosaic/memory=20=E2=80=94=20pr?= =?UTF-8?q?eference=20+=20insight=20stores?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add memory tables to DB schema (preferences, insights with pgvector embedding column, agent_logs, skills, summarization_jobs). Implement PreferencesRepo (CRUD + upsert) and InsightsRepo (CRUD + semantic search + relevance decay). Define VectorStore and EmbeddingProvider interfaces for future provider abstraction. Wire MemoryModule into gateway with REST endpoints at /api/memory/*. Co-Authored-By: Claude Opus 4.6 --- apps/gateway/package.json | 1 + apps/gateway/src/app.module.ts | 2 + apps/gateway/src/memory/memory.controller.ts | 108 +++++++++++++ apps/gateway/src/memory/memory.dto.ts | 19 +++ apps/gateway/src/memory/memory.module.ts | 20 +++ apps/gateway/src/memory/memory.tokens.ts | 1 + packages/db/src/index.ts | 16 +- packages/db/src/schema.ts | 162 ++++++++++++++++++- packages/memory/package.json | 5 +- packages/memory/src/index.ts | 16 +- packages/memory/src/insights.ts | 89 ++++++++++ packages/memory/src/memory.ts | 15 ++ packages/memory/src/preferences.ts | 59 +++++++ packages/memory/src/vector-store.ts | 39 +++++ pnpm-lock.yaml | 9 ++ 15 files changed, 557 insertions(+), 4 deletions(-) create mode 100644 apps/gateway/src/memory/memory.controller.ts create mode 100644 apps/gateway/src/memory/memory.dto.ts create mode 100644 apps/gateway/src/memory/memory.module.ts create mode 100644 apps/gateway/src/memory/memory.tokens.ts create mode 100644 packages/memory/src/insights.ts create mode 100644 packages/memory/src/memory.ts create mode 100644 packages/memory/src/preferences.ts create mode 100644 packages/memory/src/vector-store.ts diff --git a/apps/gateway/package.json b/apps/gateway/package.json index ff696d2..76da96a 100644 --- a/apps/gateway/package.json +++ b/apps/gateway/package.json @@ -18,6 +18,7 @@ "@mosaic/brain": "workspace:^", "@mosaic/coord": "workspace:^", "@mosaic/db": "workspace:^", + "@mosaic/memory": "workspace:^", "@mosaic/types": "workspace:^", "@nestjs/common": "^11.0.0", "@nestjs/core": "^11.0.0", diff --git a/apps/gateway/src/app.module.ts b/apps/gateway/src/app.module.ts index 694123b..eb0a9cf 100644 --- a/apps/gateway/src/app.module.ts +++ b/apps/gateway/src/app.module.ts @@ -10,6 +10,7 @@ import { ProjectsModule } from './projects/projects.module.js'; import { MissionsModule } from './missions/missions.module.js'; import { TasksModule } from './tasks/tasks.module.js'; import { CoordModule } from './coord/coord.module.js'; +import { MemoryModule } from './memory/memory.module.js'; @Module({ imports: [ @@ -23,6 +24,7 @@ import { CoordModule } from './coord/coord.module.js'; MissionsModule, TasksModule, CoordModule, + MemoryModule, ], controllers: [HealthController], }) diff --git a/apps/gateway/src/memory/memory.controller.ts b/apps/gateway/src/memory/memory.controller.ts new file mode 100644 index 0000000..85c2f06 --- /dev/null +++ b/apps/gateway/src/memory/memory.controller.ts @@ -0,0 +1,108 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Inject, + NotFoundException, + Param, + Post, + Query, + UseGuards, +} from '@nestjs/common'; +import type { Memory } from '@mosaic/memory'; +import { MEMORY } from './memory.tokens.js'; +import { AuthGuard } from '../auth/auth.guard.js'; +import type { UpsertPreferenceDto, CreateInsightDto, SearchMemoryDto } from './memory.dto.js'; + +@Controller('api/memory') +@UseGuards(AuthGuard) +export class MemoryController { + constructor(@Inject(MEMORY) private readonly memory: Memory) {} + + // ─── Preferences ──────────────────────────────────────────────────── + + @Get('preferences') + async listPreferences(@Query('userId') userId: string, @Query('category') category?: string) { + if (category) { + return this.memory.preferences.findByUserAndCategory( + userId, + category as Parameters[1], + ); + } + return this.memory.preferences.findByUser(userId); + } + + @Get('preferences/:key') + async getPreference(@Query('userId') userId: string, @Param('key') key: string) { + const pref = await this.memory.preferences.findByUserAndKey(userId, key); + if (!pref) throw new NotFoundException('Preference not found'); + return pref; + } + + @Post('preferences') + async upsertPreference(@Query('userId') userId: string, @Body() dto: UpsertPreferenceDto) { + return this.memory.preferences.upsert({ + userId, + key: dto.key, + value: dto.value, + category: dto.category, + source: dto.source, + }); + } + + @Delete('preferences/:key') + @HttpCode(HttpStatus.NO_CONTENT) + async removePreference(@Query('userId') userId: string, @Param('key') key: string) { + const deleted = await this.memory.preferences.remove(userId, key); + if (!deleted) throw new NotFoundException('Preference not found'); + } + + // ─── Insights ─────────────────────────────────────────────────────── + + @Get('insights') + async listInsights(@Query('userId') userId: string, @Query('limit') limit?: string) { + return this.memory.insights.findByUser(userId, limit ? Number(limit) : undefined); + } + + @Get('insights/:id') + async getInsight(@Param('id') id: string) { + const insight = await this.memory.insights.findById(id); + if (!insight) throw new NotFoundException('Insight not found'); + return insight; + } + + @Post('insights') + async createInsight(@Query('userId') userId: string, @Body() dto: CreateInsightDto) { + return this.memory.insights.create({ + userId, + content: dto.content, + source: dto.source, + category: dto.category, + metadata: dto.metadata, + }); + } + + @Delete('insights/:id') + @HttpCode(HttpStatus.NO_CONTENT) + async removeInsight(@Param('id') id: string) { + const deleted = await this.memory.insights.remove(id); + if (!deleted) throw new NotFoundException('Insight not found'); + } + + // ─── Search ───────────────────────────────────────────────────────── + + @Post('search') + async searchMemory(@Query('userId') userId: string, @Body() dto: SearchMemoryDto) { + // Search requires an embedding provider to be configured. + // For now, return empty results if no embedding is available. + // P4-002 will implement the full embedding + search pipeline. + return { + query: dto.query, + results: [], + message: 'Semantic search requires embedding provider (P4-002)', + }; + } +} diff --git a/apps/gateway/src/memory/memory.dto.ts b/apps/gateway/src/memory/memory.dto.ts new file mode 100644 index 0000000..5182ee2 --- /dev/null +++ b/apps/gateway/src/memory/memory.dto.ts @@ -0,0 +1,19 @@ +export interface UpsertPreferenceDto { + key: string; + value: unknown; + category?: 'communication' | 'coding' | 'workflow' | 'appearance' | 'general'; + source?: string; +} + +export interface CreateInsightDto { + content: string; + source?: 'agent' | 'user' | 'summarization' | 'system'; + category?: 'decision' | 'learning' | 'preference' | 'fact' | 'pattern' | 'general'; + metadata?: Record; +} + +export interface SearchMemoryDto { + query: string; + limit?: number; + maxDistance?: number; +} diff --git a/apps/gateway/src/memory/memory.module.ts b/apps/gateway/src/memory/memory.module.ts new file mode 100644 index 0000000..10705c6 --- /dev/null +++ b/apps/gateway/src/memory/memory.module.ts @@ -0,0 +1,20 @@ +import { Global, Module } from '@nestjs/common'; +import { createMemory, type Memory } from '@mosaic/memory'; +import type { Db } from '@mosaic/db'; +import { DB } from '../database/database.module.js'; +import { MEMORY } from './memory.tokens.js'; +import { MemoryController } from './memory.controller.js'; + +@Global() +@Module({ + providers: [ + { + provide: MEMORY, + useFactory: (db: Db): Memory => createMemory(db), + inject: [DB], + }, + ], + controllers: [MemoryController], + exports: [MEMORY], +}) +export class MemoryModule {} diff --git a/apps/gateway/src/memory/memory.tokens.ts b/apps/gateway/src/memory/memory.tokens.ts new file mode 100644 index 0000000..53baa78 --- /dev/null +++ b/apps/gateway/src/memory/memory.tokens.ts @@ -0,0 +1 @@ +export const MEMORY = 'MEMORY'; diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 4e8e998..f14fb70 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,4 +1,18 @@ export { createDb, type Db, type DbHandle } from './client.js'; export { runMigrations } from './migrate.js'; export * from './schema.js'; -export { eq, and, or, desc, asc, sql, inArray, isNull, isNotNull } from 'drizzle-orm'; +export { + eq, + and, + or, + desc, + asc, + sql, + inArray, + isNull, + isNotNull, + gt, + lt, + gte, + lte, +} from 'drizzle-orm'; diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 0ee28d0..a5c0993 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -3,7 +3,18 @@ * drizzle-kit reads this file directly (avoids CJS/ESM extension issues). */ -import { pgTable, text, timestamp, boolean, uuid, jsonb, index } from 'drizzle-orm/pg-core'; +import { + pgTable, + text, + timestamp, + boolean, + uuid, + jsonb, + index, + real, + integer, + customType, +} from 'drizzle-orm/pg-core'; // ─── Auth (BetterAuth-compatible) ──────────────────────────────────────────── @@ -211,3 +222,152 @@ export const messages = pgTable( }, (t) => [index('messages_conversation_id_idx').on(t.conversationId)], ); + +// ─── pgvector custom type ─────────────────────────────────────────────────── + +const vector = customType<{ data: number[]; driverParam: string; config: { dimensions: number } }>({ + dataType(config) { + return `vector(${config?.dimensions ?? 1536})`; + }, + fromDriver(value: unknown): number[] { + const str = value as string; + return str + .slice(1, -1) + .split(',') + .map((v) => Number(v)); + }, + toDriver(value: number[]): string { + return `[${value.join(',')}]`; + }, +}); + +// ─── Memory ───────────────────────────────────────────────────────────────── + +export const preferences = pgTable( + 'preferences', + { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id') + .notNull() + .references(() => users.id, { onDelete: 'cascade' }), + key: text('key').notNull(), + value: jsonb('value').notNull(), + category: text('category', { + enum: ['communication', 'coding', 'workflow', 'appearance', 'general'], + }) + .notNull() + .default('general'), + source: text('source'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [ + index('preferences_user_id_idx').on(t.userId), + index('preferences_user_key_idx').on(t.userId, t.key), + ], +); + +export const insights = pgTable( + 'insights', + { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id') + .notNull() + .references(() => users.id, { onDelete: 'cascade' }), + content: text('content').notNull(), + embedding: vector('embedding', { dimensions: 1536 }), + source: text('source', { + enum: ['agent', 'user', 'summarization', 'system'], + }) + .notNull() + .default('agent'), + category: text('category', { + enum: ['decision', 'learning', 'preference', 'fact', 'pattern', 'general'], + }) + .notNull() + .default('general'), + relevanceScore: real('relevance_score').notNull().default(1.0), + metadata: jsonb('metadata'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + decayedAt: timestamp('decayed_at', { withTimezone: true }), + }, + (t) => [ + index('insights_user_id_idx').on(t.userId), + index('insights_category_idx').on(t.category), + index('insights_relevance_idx').on(t.relevanceScore), + ], +); + +// ─── Agent Logs ───────────────────────────────────────────────────────────── + +export const agentLogs = pgTable( + 'agent_logs', + { + id: uuid('id').primaryKey().defaultRandom(), + sessionId: text('session_id').notNull(), + userId: text('user_id').references(() => users.id, { onDelete: 'set null' }), + level: text('level', { enum: ['debug', 'info', 'warn', 'error'] }) + .notNull() + .default('info'), + category: text('category', { + enum: ['decision', 'tool_use', 'learning', 'error', 'general'], + }) + .notNull() + .default('general'), + content: text('content').notNull(), + metadata: jsonb('metadata'), + tier: text('tier', { enum: ['hot', 'warm', 'cold'] }) + .notNull() + .default('hot'), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + summarizedAt: timestamp('summarized_at', { withTimezone: true }), + archivedAt: timestamp('archived_at', { withTimezone: true }), + }, + (t) => [ + index('agent_logs_session_id_idx').on(t.sessionId), + index('agent_logs_user_id_idx').on(t.userId), + index('agent_logs_tier_idx').on(t.tier), + index('agent_logs_created_at_idx').on(t.createdAt), + ], +); + +// ─── Skills ───────────────────────────────────────────────────────────────── + +export const skills = pgTable( + 'skills', + { + id: uuid('id').primaryKey().defaultRandom(), + name: text('name').notNull().unique(), + description: text('description'), + version: text('version'), + source: text('source', { enum: ['builtin', 'community', 'custom'] }) + .notNull() + .default('custom'), + config: jsonb('config'), + enabled: boolean('enabled').notNull().default(true), + installedBy: text('installed_by').references(() => users.id, { onDelete: 'set null' }), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [index('skills_enabled_idx').on(t.enabled)], +); + +// ─── Summarization Jobs ───────────────────────────────────────────────────── + +export const summarizationJobs = pgTable( + 'summarization_jobs', + { + id: uuid('id').primaryKey().defaultRandom(), + status: text('status', { enum: ['pending', 'running', 'completed', 'failed'] }) + .notNull() + .default('pending'), + logsProcessed: integer('logs_processed').notNull().default(0), + insightsCreated: integer('insights_created').notNull().default(0), + errorMessage: text('error_message'), + startedAt: timestamp('started_at', { withTimezone: true }), + completedAt: timestamp('completed_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [index('summarization_jobs_status_idx').on(t.status)], +); diff --git a/packages/memory/package.json b/packages/memory/package.json index cce07cd..f669d56 100644 --- a/packages/memory/package.json +++ b/packages/memory/package.json @@ -1,6 +1,7 @@ { "name": "@mosaic/memory", "version": "0.0.0", + "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", "exports": { @@ -16,7 +17,9 @@ "test": "vitest run --passWithNoTests" }, "dependencies": { - "@mosaic/types": "workspace:*" + "@mosaic/db": "workspace:*", + "@mosaic/types": "workspace:*", + "drizzle-orm": "^0.45.1" }, "devDependencies": { "typescript": "^5.8.0", diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index 0c18d5d..ebe925c 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -1 +1,15 @@ -export const VERSION = '0.0.0'; +export { createMemory, type Memory } from './memory.js'; +export { + createPreferencesRepo, + type PreferencesRepo, + type Preference, + type NewPreference, +} from './preferences.js'; +export { + createInsightsRepo, + type InsightsRepo, + type Insight, + type NewInsight, + type SearchResult, +} from './insights.js'; +export type { VectorStore, VectorSearchResult, EmbeddingProvider } from './vector-store.js'; diff --git a/packages/memory/src/insights.ts b/packages/memory/src/insights.ts new file mode 100644 index 0000000..d844caa --- /dev/null +++ b/packages/memory/src/insights.ts @@ -0,0 +1,89 @@ +import { eq, and, desc, sql, lt, type Db, insights } from '@mosaic/db'; + +export type Insight = typeof insights.$inferSelect; +export type NewInsight = typeof insights.$inferInsert; + +export interface SearchResult { + insight: Insight; + distance: number; +} + +export function createInsightsRepo(db: Db) { + return { + async findByUser(userId: string, limit = 50): Promise { + return db + .select() + .from(insights) + .where(eq(insights.userId, userId)) + .orderBy(desc(insights.createdAt)) + .limit(limit); + }, + + async findById(id: string): Promise { + const rows = await db.select().from(insights).where(eq(insights.id, id)); + return rows[0]; + }, + + async create(data: NewInsight): Promise { + const rows = await db.insert(insights).values(data).returning(); + return rows[0]!; + }, + + async update(id: string, data: Partial): Promise { + const rows = await db + .update(insights) + .set({ ...data, updatedAt: new Date() }) + .where(eq(insights.id, id)) + .returning(); + return rows[0]; + }, + + async remove(id: string): Promise { + const rows = await db.delete(insights).where(eq(insights.id, id)).returning(); + return rows.length > 0; + }, + + /** + * Semantic search using pgvector cosine distance. + * Requires the vector extension and an embedding for the query. + */ + async searchByEmbedding( + userId: string, + queryEmbedding: number[], + limit = 10, + maxDistance = 0.8, + ): Promise { + const embeddingStr = `[${queryEmbedding.join(',')}]`; + const rows = await db.execute(sql` + SELECT *, + (embedding <=> ${embeddingStr}::vector) AS distance + FROM insights + WHERE user_id = ${userId} + AND embedding IS NOT NULL + AND (embedding <=> ${embeddingStr}::vector) < ${maxDistance} + ORDER BY distance ASC + LIMIT ${limit} + `); + + return rows as unknown as SearchResult[]; + }, + + /** + * Decay relevance scores for old insights that haven't been accessed recently. + */ + async decayOldInsights(olderThan: Date, decayFactor = 0.95): Promise { + const result = await db + .update(insights) + .set({ + relevanceScore: sql`${insights.relevanceScore} * ${decayFactor}`, + decayedAt: new Date(), + updatedAt: new Date(), + }) + .where(and(lt(insights.updatedAt, olderThan), sql`${insights.relevanceScore} > 0.1`)) + .returning(); + return result.length; + }, + }; +} + +export type InsightsRepo = ReturnType; diff --git a/packages/memory/src/memory.ts b/packages/memory/src/memory.ts new file mode 100644 index 0000000..ef46aca --- /dev/null +++ b/packages/memory/src/memory.ts @@ -0,0 +1,15 @@ +import type { Db } from '@mosaic/db'; +import { createPreferencesRepo, type PreferencesRepo } from './preferences.js'; +import { createInsightsRepo, type InsightsRepo } from './insights.js'; + +export interface Memory { + preferences: PreferencesRepo; + insights: InsightsRepo; +} + +export function createMemory(db: Db): Memory { + return { + preferences: createPreferencesRepo(db), + insights: createInsightsRepo(db), + }; +} diff --git a/packages/memory/src/preferences.ts b/packages/memory/src/preferences.ts new file mode 100644 index 0000000..9b3a811 --- /dev/null +++ b/packages/memory/src/preferences.ts @@ -0,0 +1,59 @@ +import { eq, and, type Db, preferences } from '@mosaic/db'; + +export type Preference = typeof preferences.$inferSelect; +export type NewPreference = typeof preferences.$inferInsert; + +export function createPreferencesRepo(db: Db) { + return { + async findByUser(userId: string): Promise { + return db.select().from(preferences).where(eq(preferences.userId, userId)); + }, + + async findByUserAndKey(userId: string, key: string): Promise { + const rows = await db + .select() + .from(preferences) + .where(and(eq(preferences.userId, userId), eq(preferences.key, key))); + return rows[0]; + }, + + async findByUserAndCategory( + userId: string, + category: Preference['category'], + ): Promise { + return db + .select() + .from(preferences) + .where(and(eq(preferences.userId, userId), eq(preferences.category, category))); + }, + + async upsert(data: NewPreference): Promise { + const existing = await db + .select() + .from(preferences) + .where(and(eq(preferences.userId, data.userId), eq(preferences.key, data.key))); + + if (existing[0]) { + const rows = await db + .update(preferences) + .set({ value: data.value, category: data.category, updatedAt: new Date() }) + .where(eq(preferences.id, existing[0].id)) + .returning(); + return rows[0]!; + } + + const rows = await db.insert(preferences).values(data).returning(); + return rows[0]!; + }, + + async remove(userId: string, key: string): Promise { + const rows = await db + .delete(preferences) + .where(and(eq(preferences.userId, userId), eq(preferences.key, key))) + .returning(); + return rows.length > 0; + }, + }; +} + +export type PreferencesRepo = ReturnType; diff --git a/packages/memory/src/vector-store.ts b/packages/memory/src/vector-store.ts new file mode 100644 index 0000000..f268d49 --- /dev/null +++ b/packages/memory/src/vector-store.ts @@ -0,0 +1,39 @@ +/** + * VectorStore interface — abstraction over pgvector that allows future + * swap to Qdrant, Pinecone, etc. + */ +export interface VectorStore { + /** Store an embedding with an associated document ID. */ + store(documentId: string, embedding: number[], metadata?: Record): Promise; + + /** Search for similar embeddings, returning document IDs and distances. */ + search( + queryEmbedding: number[], + limit?: number, + filter?: Record, + ): Promise; + + /** Delete an embedding by document ID. */ + remove(documentId: string): Promise; +} + +export interface VectorSearchResult { + documentId: string; + distance: number; + metadata?: Record; +} + +/** + * EmbeddingProvider interface — generates embeddings from text. + * Implemented by the gateway using the configured LLM provider. + */ +export interface EmbeddingProvider { + /** Generate an embedding vector for the given text. */ + embed(text: string): Promise; + + /** Generate embeddings for multiple texts in batch. */ + embedBatch(texts: string[]): Promise; + + /** The dimensionality of the embeddings this provider generates. */ + dimensions: number; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b51a464..5e36928 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: '@mosaic/db': specifier: workspace:^ version: link:../../packages/db + '@mosaic/memory': + specifier: workspace:^ + version: link:../../packages/memory '@mosaic/types': specifier: workspace:^ version: link:../../packages/types @@ -331,9 +334,15 @@ importers: packages/memory: dependencies: + '@mosaic/db': + specifier: workspace:* + version: link:../db '@mosaic/types': specifier: workspace:* version: link:../types + drizzle-orm: + specifier: ^0.45.1 + version: 0.45.1(@opentelemetry/api@1.9.0)(@types/pg@8.15.6)(kysely@0.28.11)(postgres@3.4.8) devDependencies: typescript: specifier: ^5.8.0