feat: Add semantic search with pgvector (closes #68, #69, #70) #119

Merged
jason.woltje merged 2 commits from feature/semantic-search into develop 2026-01-30 21:20:32 +00:00
14 changed files with 1408 additions and 5 deletions

View File

@@ -88,6 +88,14 @@ JWT_EXPIRATION=24h
OLLAMA_ENDPOINT=http://ollama:11434
OLLAMA_PORT=11434
# ======================
# OpenAI API (For Semantic Search)
# ======================
# OPTIONAL: Semantic search requires an OpenAI API key
# Get your API key from: https://platform.openai.com/api-keys
# If not configured, semantic search endpoints will return an error
# OPENAI_API_KEY=sk-...
# ======================
# Application Environment
# ======================

View File

@@ -48,6 +48,7 @@
"marked-gfm-heading-id": "^4.1.3",
"marked-highlight": "^2.2.3",
"ollama": "^0.6.3",
"openai": "^6.17.0",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sanitize-html": "^2.17.0",

View File

@@ -0,0 +1,8 @@
-- Add HNSW index for fast vector similarity search on knowledge_embeddings table
-- Using cosine distance operator for semantic similarity
-- Parameters: m=16 (max connections per layer), ef_construction=64 (build quality)
CREATE INDEX IF NOT EXISTS knowledge_embeddings_embedding_idx
ON knowledge_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);

View File

@@ -12,6 +12,7 @@ import {
DefaultValuePipe,
} from "@nestjs/common";
import type { AuthUser } from "@mosaic/shared";
import { EntryStatus } from "@prisma/client";
import { KnowledgeService } from "./knowledge.service";
import { CreateEntryDto, UpdateEntryDto, EntryQueryDto, RestoreVersionDto } from "./dto";
import { AuthGuard } from "../auth/guards/auth.guard";
@@ -192,6 +193,38 @@ export class KnowledgeController {
}
}
/**
* Controller for knowledge embeddings endpoints
*/
@Controller("knowledge/embeddings")
@UseGuards(AuthGuard, WorkspaceGuard, PermissionGuard)
export class KnowledgeEmbeddingsController {
constructor(private readonly knowledgeService: KnowledgeService) {}
/**
* POST /api/knowledge/embeddings/batch
* Batch generate embeddings for all entries in the workspace
* Useful for populating embeddings for existing entries
* Requires: ADMIN role or higher
*/
@Post("batch")
@RequirePermission(Permission.WORKSPACE_ADMIN)
async batchGenerate(
@Workspace() workspaceId: string,
@Body() body: { status?: string }
) {
const status = body.status as EntryStatus | undefined;
const result = await this.knowledgeService.batchGenerateEmbeddings(
workspaceId,
status
);
return {
message: `Generated ${result.success} embeddings out of ${result.total} entries`,
...result,
};
}
}
/**
* Controller for knowledge cache endpoints
*/

View File

@@ -2,7 +2,11 @@ import { Module } from "@nestjs/common";
import { PrismaModule } from "../prisma/prisma.module";
import { AuthModule } from "../auth/auth.module";
import { KnowledgeService } from "./knowledge.service";
import { KnowledgeController, KnowledgeCacheController } from "./knowledge.controller";
import {
KnowledgeController,
KnowledgeCacheController,
KnowledgeEmbeddingsController,
} from "./knowledge.controller";
import { SearchController } from "./search.controller";
import { KnowledgeStatsController } from "./stats.controller";
import {
@@ -12,6 +16,7 @@ import {
GraphService,
StatsService,
KnowledgeCacheService,
EmbeddingService,
} from "./services";
@Module({
@@ -19,6 +24,7 @@ import {
controllers: [
KnowledgeController,
KnowledgeCacheController,
KnowledgeEmbeddingsController,
SearchController,
KnowledgeStatsController,
],
@@ -30,7 +36,8 @@ import {
GraphService,
StatsService,
KnowledgeCacheService,
EmbeddingService,
],
exports: [KnowledgeService, LinkResolutionService, SearchService],
exports: [KnowledgeService, LinkResolutionService, SearchService, EmbeddingService],
})
export class KnowledgeModule {}

View File

@@ -18,6 +18,7 @@ import type {
import { renderMarkdown } from "./utils/markdown";
import { LinkSyncService } from "./services/link-sync.service";
import { KnowledgeCacheService } from "./services/cache.service";
import { EmbeddingService } from "./services/embedding.service";
/**
* Service for managing knowledge entries
@@ -27,7 +28,8 @@ export class KnowledgeService {
constructor(
private readonly prisma: PrismaService,
private readonly linkSync: LinkSyncService,
private readonly cache: KnowledgeCacheService
private readonly cache: KnowledgeCacheService,
private readonly embedding: EmbeddingService
) {}
@@ -250,6 +252,13 @@ export class KnowledgeService {
// Sync wiki links after entry creation
await this.linkSync.syncLinks(workspaceId, result.id, createDto.content);
// Generate and store embedding asynchronously (don't block the response)
this.generateEntryEmbedding(result.id, result.title, result.content).catch(
(error) => {
console.error(`Failed to generate embedding for entry ${result.id}:`, error);
}
);
// Invalidate search and graph caches (new entry affects search results)
await this.cache.invalidateSearches(workspaceId);
await this.cache.invalidateGraphs(workspaceId);
@@ -408,6 +417,15 @@ export class KnowledgeService {
await this.linkSync.syncLinks(workspaceId, result.id, result.content);
}
// Regenerate embedding if content or title changed (async, don't block response)
if (updateDto.content !== undefined || updateDto.title !== undefined) {
this.generateEntryEmbedding(result.id, result.title, result.content).catch(
(error) => {
console.error(`Failed to generate embedding for entry ${result.id}:`, error);
}
);
}
// Invalidate caches
// Invalidate old slug cache if slug changed
if (newSlug !== slug) {
@@ -863,4 +881,64 @@ export class KnowledgeService {
)
);
}
/**
* Generate and store embedding for a knowledge entry
* Private helper method called asynchronously after entry create/update
*/
private async generateEntryEmbedding(
entryId: string,
title: string,
content: string
): Promise<void> {
const combinedContent = this.embedding.prepareContentForEmbedding(
title,
content
);
await this.embedding.generateAndStoreEmbedding(entryId, combinedContent);
}
/**
* Batch generate embeddings for all entries in a workspace
* Useful for populating embeddings for existing entries
*
* @param workspaceId - The workspace ID
* @param status - Optional status filter (default: not ARCHIVED)
* @returns Number of embeddings successfully generated
*/
async batchGenerateEmbeddings(
workspaceId: string,
status?: EntryStatus
): Promise<{ total: number; success: number }> {
const where: Prisma.KnowledgeEntryWhereInput = {
workspaceId,
status: status || { not: EntryStatus.ARCHIVED },
};
const entries = await this.prisma.knowledgeEntry.findMany({
where,
select: {
id: true,
title: true,
content: true,
},
});
const entriesForEmbedding = entries.map((entry) => ({
id: entry.id,
content: this.embedding.prepareContentForEmbedding(
entry.title,
entry.content
),
}));
const successCount = await this.embedding.batchGenerateEmbeddings(
entriesForEmbedding
);
return {
total: entries.length,
success: successCount,
};
}
}

View File

@@ -1,9 +1,10 @@
import { Controller, Get, Query, UseGuards } from "@nestjs/common";
import { Controller, Get, Post, Body, Query, UseGuards } from "@nestjs/common";
import { SearchService, PaginatedSearchResults } from "./services/search.service";
import { SearchQueryDto, TagSearchDto, RecentEntriesDto } from "./dto";
import { AuthGuard } from "../auth/guards/auth.guard";
import { WorkspaceGuard, PermissionGuard } from "../common/guards";
import { Workspace, Permission, RequirePermission } from "../common/decorators";
import { EntryStatus } from "@prisma/client";
import type {
PaginatedEntries,
KnowledgeEntryWithTags,
@@ -97,4 +98,55 @@ export class SearchController {
count: entries.length,
};
}
/**
* POST /api/knowledge/search/semantic
* Semantic search using vector similarity
* Requires: Any workspace member, OpenAI API key configured
*
* @body query - The search query string (required)
* @body status - Filter by entry status (optional)
* @query page - Page number (default: 1)
* @query limit - Results per page (default: 20, max: 100)
*/
@Post("semantic")
@RequirePermission(Permission.WORKSPACE_ANY)
async semanticSearch(
@Workspace() workspaceId: string,
@Body() body: { query: string; status?: EntryStatus },
@Query("page") page?: number,
@Query("limit") limit?: number
): Promise<PaginatedSearchResults> {
return this.searchService.semanticSearch(body.query, workspaceId, {
status: body.status,
page,
limit,
});
}
/**
* POST /api/knowledge/search/hybrid
* Hybrid search combining vector similarity and full-text search
* Uses Reciprocal Rank Fusion to merge results
* Requires: Any workspace member
*
* @body query - The search query string (required)
* @body status - Filter by entry status (optional)
* @query page - Page number (default: 1)
* @query limit - Results per page (default: 20, max: 100)
*/
@Post("hybrid")
@RequirePermission(Permission.WORKSPACE_ANY)
async hybridSearch(
@Workspace() workspaceId: string,
@Body() body: { query: string; status?: EntryStatus },
@Query("page") page?: number,
@Query("limit") limit?: number
): Promise<PaginatedSearchResults> {
return this.searchService.hybridSearch(body.query, workspaceId, {
status: body.status,
page,
limit,
});
}
}

View File

@@ -0,0 +1,115 @@
import { describe, it, expect, beforeEach, vi } from "vitest";
import { EmbeddingService } from "./embedding.service";
import { PrismaService } from "../../prisma/prisma.service";
describe("EmbeddingService", () => {
let service: EmbeddingService;
let prismaService: PrismaService;
beforeEach(() => {
prismaService = {
$executeRaw: vi.fn(),
knowledgeEmbedding: {
deleteMany: vi.fn(),
},
} as unknown as PrismaService;
service = new EmbeddingService(prismaService);
});
describe("isConfigured", () => {
it("should return false when OPENAI_API_KEY is not set", () => {
const originalEnv = process.env["OPENAI_API_KEY"];
delete process.env["OPENAI_API_KEY"];
expect(service.isConfigured()).toBe(false);
if (originalEnv) {
process.env["OPENAI_API_KEY"] = originalEnv;
}
});
it("should return true when OPENAI_API_KEY is set", () => {
const originalEnv = process.env["OPENAI_API_KEY"];
process.env["OPENAI_API_KEY"] = "test-key";
expect(service.isConfigured()).toBe(true);
if (originalEnv) {
process.env["OPENAI_API_KEY"] = originalEnv;
} else {
delete process.env["OPENAI_API_KEY"];
}
});
});
describe("prepareContentForEmbedding", () => {
it("should combine title and content with title weighting", () => {
const title = "Test Title";
const content = "Test content goes here";
const result = service.prepareContentForEmbedding(title, content);
expect(result).toContain(title);
expect(result).toContain(content);
// Title should appear twice for weighting
expect(result.split(title).length - 1).toBe(2);
});
it("should handle empty content", () => {
const title = "Test Title";
const content = "";
const result = service.prepareContentForEmbedding(title, content);
expect(result).toBe(`${title}\n\n${title}`);
});
});
describe("generateAndStoreEmbedding", () => {
it("should skip generation when not configured", async () => {
const originalEnv = process.env["OPENAI_API_KEY"];
delete process.env["OPENAI_API_KEY"];
await service.generateAndStoreEmbedding("test-id", "test content");
expect(prismaService.$executeRaw).not.toHaveBeenCalled();
if (originalEnv) {
process.env["OPENAI_API_KEY"] = originalEnv;
}
});
});
describe("deleteEmbedding", () => {
it("should delete embedding for entry", async () => {
const entryId = "test-entry-id";
await service.deleteEmbedding(entryId);
expect(prismaService.knowledgeEmbedding.deleteMany).toHaveBeenCalledWith({
where: { entryId },
});
});
});
describe("batchGenerateEmbeddings", () => {
it("should return 0 when not configured", async () => {
const originalEnv = process.env["OPENAI_API_KEY"];
delete process.env["OPENAI_API_KEY"];
const entries = [
{ id: "1", content: "content 1" },
{ id: "2", content: "content 2" },
];
const result = await service.batchGenerateEmbeddings(entries);
expect(result).toBe(0);
if (originalEnv) {
process.env["OPENAI_API_KEY"] = originalEnv;
}
});
});
});

View File

@@ -0,0 +1,190 @@
import { Injectable, Logger } from "@nestjs/common";
import OpenAI from "openai";
import { PrismaService } from "../../prisma/prisma.service";
import { EMBEDDING_DIMENSION } from "@mosaic/shared";
/**
* Options for generating embeddings
*/
export interface EmbeddingOptions {
/**
* Model to use for embedding generation
* @default "text-embedding-3-small"
*/
model?: string;
}
/**
* Service for generating and managing embeddings using OpenAI API
*/
@Injectable()
export class EmbeddingService {
private readonly logger = new Logger(EmbeddingService.name);
private readonly openai: OpenAI;
private readonly defaultModel = "text-embedding-3-small";
constructor(private readonly prisma: PrismaService) {
const apiKey = process.env["OPENAI_API_KEY"];
if (!apiKey) {
this.logger.warn("OPENAI_API_KEY not configured - embedding generation will be disabled");
}
this.openai = new OpenAI({
apiKey: apiKey || "dummy-key", // Provide dummy key to allow instantiation
});
}
/**
* Check if the service is properly configured
*/
isConfigured(): boolean {
return !!process.env["OPENAI_API_KEY"];
}
/**
* Generate an embedding vector for the given text
*
* @param text - Text to embed
* @param options - Embedding generation options
* @returns Embedding vector (array of numbers)
* @throws Error if OpenAI API key is not configured
*/
async generateEmbedding(
text: string,
options: EmbeddingOptions = {}
): Promise<number[]> {
if (!this.isConfigured()) {
throw new Error("OPENAI_API_KEY not configured");
}
const model = options.model || this.defaultModel;
try {
const response = await this.openai.embeddings.create({
model,
input: text,
dimensions: EMBEDDING_DIMENSION,
});
const embedding = response.data[0]?.embedding;
if (!embedding) {
throw new Error("No embedding returned from OpenAI");
}
if (embedding.length !== EMBEDDING_DIMENSION) {
throw new Error(
`Unexpected embedding dimension: ${embedding.length} (expected ${EMBEDDING_DIMENSION})`
);
}
return embedding;
} catch (error) {
this.logger.error("Failed to generate embedding", error);
throw error;
}
}
/**
* Generate and store embedding for a knowledge entry
*
* @param entryId - ID of the knowledge entry
* @param content - Content to embed (typically title + content)
* @param options - Embedding generation options
* @returns Created/updated embedding record
*/
async generateAndStoreEmbedding(
entryId: string,
content: string,
options: EmbeddingOptions = {}
): Promise<void> {
if (!this.isConfigured()) {
this.logger.warn(`Skipping embedding generation for entry ${entryId} - OpenAI not configured`);
return;
}
const model = options.model || this.defaultModel;
const embedding = await this.generateEmbedding(content, { model });
// Convert to Prisma-compatible format
const embeddingString = `[${embedding.join(",")}]`;
// Upsert the embedding
await this.prisma.$executeRaw`
INSERT INTO knowledge_embeddings (id, entry_id, embedding, model, created_at, updated_at)
VALUES (
gen_random_uuid(),
${entryId}::uuid,
${embeddingString}::vector(${EMBEDDING_DIMENSION}),
${model},
NOW(),
NOW()
)
ON CONFLICT (entry_id) DO UPDATE SET
embedding = ${embeddingString}::vector(${EMBEDDING_DIMENSION}),
model = ${model},
updated_at = NOW()
`;
this.logger.log(`Generated and stored embedding for entry ${entryId}`);
}
/**
* Batch process embeddings for multiple entries
*
* @param entries - Array of {id, content} objects
* @param options - Embedding generation options
* @returns Number of embeddings successfully generated
*/
async batchGenerateEmbeddings(
entries: Array<{ id: string; content: string }>,
options: EmbeddingOptions = {}
): Promise<number> {
if (!this.isConfigured()) {
this.logger.warn("Skipping batch embedding generation - OpenAI not configured");
return 0;
}
let successCount = 0;
for (const entry of entries) {
try {
await this.generateAndStoreEmbedding(entry.id, entry.content, options);
successCount++;
} catch (error) {
this.logger.error(`Failed to generate embedding for entry ${entry.id}`, error);
}
}
this.logger.log(`Batch generated ${successCount}/${entries.length} embeddings`);
return successCount;
}
/**
* Delete embedding for a knowledge entry
*
* @param entryId - ID of the knowledge entry
*/
async deleteEmbedding(entryId: string): Promise<void> {
await this.prisma.knowledgeEmbedding.deleteMany({
where: { entryId },
});
this.logger.log(`Deleted embedding for entry ${entryId}`);
}
/**
* Prepare content for embedding
* Combines title and content with appropriate weighting
*
* @param title - Entry title
* @param content - Entry content (markdown)
* @returns Combined text for embedding
*/
prepareContentForEmbedding(title: string, content: string): string {
// Weight title more heavily by repeating it
// This helps with semantic search matching on titles
return `${title}\n\n${title}\n\n${content}`.trim();
}
}

View File

@@ -10,3 +10,5 @@ export { GraphService } from "./graph.service";
export { StatsService } from "./stats.service";
export { KnowledgeCacheService } from "./cache.service";
export type { CacheStats, CacheOptions } from "./cache.service";
export { EmbeddingService } from "./embedding.service";
export type { EmbeddingOptions } from "./embedding.service";

View File

@@ -6,6 +6,7 @@ import type {
PaginatedEntries,
} from "../entities/knowledge-entry.entity";
import { KnowledgeCacheService } from "./cache.service";
import { EmbeddingService } from "./embedding.service";
/**
* Search options for full-text search
@@ -66,7 +67,8 @@ interface RawSearchResult {
export class SearchService {
constructor(
private readonly prisma: PrismaService,
private readonly cache: KnowledgeCacheService
private readonly cache: KnowledgeCacheService,
private readonly embedding: EmbeddingService
) {}
/**
@@ -428,4 +430,288 @@ export class SearchService {
return tagsMap;
}
/**
* Semantic search using vector similarity
*
* @param query - The search query string
* @param workspaceId - The workspace to search within
* @param options - Search options (status filter, pagination)
* @returns Paginated search results ranked by semantic similarity
*/
async semanticSearch(
query: string,
workspaceId: string,
options: SearchOptions = {}
): Promise<PaginatedSearchResults> {
if (!this.embedding.isConfigured()) {
throw new Error("Semantic search requires OPENAI_API_KEY to be configured");
}
const page = options.page || 1;
const limit = options.limit || 20;
const offset = (page - 1) * limit;
// Generate embedding for the query
const queryEmbedding = await this.embedding.generateEmbedding(query);
const embeddingString = `[${queryEmbedding.join(",")}]`;
// Build status filter
const statusFilter = options.status
? Prisma.sql`AND e.status = ${options.status}::text::"EntryStatus"`
: Prisma.sql`AND e.status != 'ARCHIVED'`;
// Vector similarity search using cosine distance
const searchResults = await this.prisma.$queryRaw<RawSearchResult[]>`
SELECT
e.id,
e.workspace_id,
e.slug,
e.title,
e.content,
e.content_html,
e.summary,
e.status,
e.visibility,
e.created_at,
e.updated_at,
e.created_by,
e.updated_by,
(1 - (emb.embedding <=> ${embeddingString}::vector)) AS rank,
NULL AS headline
FROM knowledge_entries e
INNER JOIN knowledge_embeddings emb ON e.id = emb.entry_id
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
ORDER BY emb.embedding <=> ${embeddingString}::vector
LIMIT ${limit}
OFFSET ${offset}
`;
// Get total count for pagination
const countResult = await this.prisma.$queryRaw<[{ count: bigint }]>`
SELECT COUNT(*) as count
FROM knowledge_entries e
INNER JOIN knowledge_embeddings emb ON e.id = emb.entry_id
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
`;
const total = Number(countResult[0].count);
// Fetch tags for the results
const entryIds = searchResults.map((r) => r.id);
const tagsMap = await this.fetchTagsForEntries(entryIds);
// Transform results to the expected format
const data: SearchResult[] = searchResults.map((row) => ({
id: row.id,
workspaceId: row.workspace_id,
slug: row.slug,
title: row.title,
content: row.content,
contentHtml: row.content_html,
summary: row.summary,
status: row.status,
visibility: row.visibility as "PRIVATE" | "WORKSPACE" | "PUBLIC",
createdAt: row.created_at,
updatedAt: row.updated_at,
createdBy: row.created_by,
updatedBy: row.updated_by,
rank: row.rank,
headline: row.headline ?? undefined,
tags: tagsMap.get(row.id) || [],
}));
return {
data,
pagination: {
page,
limit,
total,
totalPages: Math.ceil(total / limit),
},
query,
};
}
/**
* Hybrid search combining vector similarity and full-text search
* Uses Reciprocal Rank Fusion (RRF) to combine rankings
*
* @param query - The search query string
* @param workspaceId - The workspace to search within
* @param options - Search options (status filter, pagination)
* @returns Paginated search results ranked by combined relevance
*/
async hybridSearch(
query: string,
workspaceId: string,
options: SearchOptions = {}
): Promise<PaginatedSearchResults> {
if (!this.embedding.isConfigured()) {
// Fall back to keyword search if embeddings not configured
return this.search(query, workspaceId, options);
}
const page = options.page || 1;
const limit = options.limit || 20;
const offset = (page - 1) * limit;
// Sanitize query for keyword search
const sanitizedQuery = this.sanitizeSearchQuery(query);
if (!sanitizedQuery) {
return {
data: [],
pagination: {
page,
limit,
total: 0,
totalPages: 0,
},
query,
};
}
// Generate embedding for vector search
const queryEmbedding = await this.embedding.generateEmbedding(query);
const embeddingString = `[${queryEmbedding.join(",")}]`;
// Build status filter
const statusFilter = options.status
? Prisma.sql`AND e.status = ${options.status}::text::"EntryStatus"`
: Prisma.sql`AND e.status != 'ARCHIVED'`;
// Hybrid search using Reciprocal Rank Fusion (RRF)
// Combines vector similarity and full-text search rankings
const searchResults = await this.prisma.$queryRaw<RawSearchResult[]>`
WITH vector_search AS (
SELECT
e.id,
ROW_NUMBER() OVER (ORDER BY emb.embedding <=> ${embeddingString}::vector) AS rank
FROM knowledge_entries e
INNER JOIN knowledge_embeddings emb ON e.id = emb.entry_id
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
),
keyword_search AS (
SELECT
e.id,
ROW_NUMBER() OVER (
ORDER BY ts_rank(
setweight(to_tsvector('english', e.title), 'A') ||
setweight(to_tsvector('english', e.content), 'B'),
plainto_tsquery('english', ${sanitizedQuery})
) DESC
) AS rank
FROM knowledge_entries e
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
AND (
to_tsvector('english', e.title) @@ plainto_tsquery('english', ${sanitizedQuery})
OR to_tsvector('english', e.content) @@ plainto_tsquery('english', ${sanitizedQuery})
)
),
combined AS (
SELECT
COALESCE(v.id, k.id) AS id,
-- Reciprocal Rank Fusion: RRF(d) = sum(1 / (k + rank_i))
-- k=60 is a common constant that prevents high rankings from dominating
(COALESCE(1.0 / (60 + v.rank), 0) + COALESCE(1.0 / (60 + k.rank), 0)) AS rrf_score
FROM vector_search v
FULL OUTER JOIN keyword_search k ON v.id = k.id
)
SELECT
e.id,
e.workspace_id,
e.slug,
e.title,
e.content,
e.content_html,
e.summary,
e.status,
e.visibility,
e.created_at,
e.updated_at,
e.created_by,
e.updated_by,
c.rrf_score AS rank,
ts_headline(
'english',
e.content,
plainto_tsquery('english', ${sanitizedQuery}),
'MaxWords=50, MinWords=25, StartSel=<mark>, StopSel=</mark>'
) AS headline
FROM combined c
INNER JOIN knowledge_entries e ON c.id = e.id
ORDER BY c.rrf_score DESC, e.updated_at DESC
LIMIT ${limit}
OFFSET ${offset}
`;
// Get total count
const countResult = await this.prisma.$queryRaw<[{ count: bigint }]>`
WITH vector_search AS (
SELECT e.id
FROM knowledge_entries e
INNER JOIN knowledge_embeddings emb ON e.id = emb.entry_id
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
),
keyword_search AS (
SELECT e.id
FROM knowledge_entries e
WHERE e.workspace_id = ${workspaceId}::uuid
${statusFilter}
AND (
to_tsvector('english', e.title) @@ plainto_tsquery('english', ${sanitizedQuery})
OR to_tsvector('english', e.content) @@ plainto_tsquery('english', ${sanitizedQuery})
)
)
SELECT COUNT(DISTINCT id) as count
FROM (
SELECT id FROM vector_search
UNION
SELECT id FROM keyword_search
) AS combined
`;
const total = Number(countResult[0].count);
// Fetch tags for the results
const entryIds = searchResults.map((r) => r.id);
const tagsMap = await this.fetchTagsForEntries(entryIds);
// Transform results to the expected format
const data: SearchResult[] = searchResults.map((row) => ({
id: row.id,
workspaceId: row.workspace_id,
slug: row.slug,
title: row.title,
content: row.content,
contentHtml: row.content_html,
summary: row.summary,
status: row.status,
visibility: row.visibility as "PRIVATE" | "WORKSPACE" | "PUBLIC",
createdAt: row.created_at,
updatedAt: row.updated_at,
createdBy: row.created_by,
updatedBy: row.updated_by,
rank: row.rank,
headline: row.headline ?? undefined,
tags: tagsMap.get(row.id) || [],
}));
return {
data,
pagination: {
page,
limit,
total,
totalPages: Math.ceil(total / limit),
},
query,
};
}
}

View File

@@ -0,0 +1,257 @@
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { PrismaClient, EntryStatus } from "@prisma/client";
import { SearchService } from "./search.service";
import { EmbeddingService } from "./embedding.service";
import { KnowledgeCacheService } from "./cache.service";
import { PrismaService } from "../../prisma/prisma.service";
/**
* Integration tests for semantic search functionality
*
* These tests require:
* - A running PostgreSQL database with pgvector extension
* - OPENAI_API_KEY environment variable set
*
* Run with: pnpm test semantic-search.integration.spec.ts
*/
describe("Semantic Search Integration", () => {
let prisma: PrismaClient;
let searchService: SearchService;
let embeddingService: EmbeddingService;
let cacheService: KnowledgeCacheService;
let testWorkspaceId: string;
let testUserId: string;
beforeAll(async () => {
// Initialize services
prisma = new PrismaClient();
const prismaService = prisma as unknown as PrismaService;
// Mock cache service for testing
cacheService = {
getSearch: async () => null,
setSearch: async () => {},
isEnabled: () => false,
getStats: () => ({ hits: 0, misses: 0, hitRate: 0 }),
resetStats: () => {},
} as unknown as KnowledgeCacheService;
embeddingService = new EmbeddingService(prismaService);
searchService = new SearchService(
prismaService,
cacheService,
embeddingService
);
// Create test workspace and user
const workspace = await prisma.workspace.create({
data: {
name: "Test Workspace for Semantic Search",
owner: {
create: {
email: "semantic-test@example.com",
name: "Test User",
},
},
},
});
testWorkspaceId = workspace.id;
testUserId = workspace.ownerId;
});
afterAll(async () => {
// Cleanup test data
if (testWorkspaceId) {
await prisma.knowledgeEntry.deleteMany({
where: { workspaceId: testWorkspaceId },
});
await prisma.workspace.delete({
where: { id: testWorkspaceId },
});
}
await prisma.$disconnect();
});
describe("EmbeddingService", () => {
it("should check if OpenAI is configured", () => {
const isConfigured = embeddingService.isConfigured();
// This test will pass if OPENAI_API_KEY is set
expect(typeof isConfigured).toBe("boolean");
});
it("should prepare content for embedding correctly", () => {
const title = "Introduction to PostgreSQL";
const content = "PostgreSQL is a powerful open-source database.";
const prepared = embeddingService.prepareContentForEmbedding(
title,
content
);
// Title should appear twice for weighting
expect(prepared).toContain(title);
expect(prepared).toContain(content);
const titleCount = (prepared.match(new RegExp(title, "g")) || []).length;
expect(titleCount).toBe(2);
});
});
describe("Semantic Search", () => {
const testEntries = [
{
slug: "postgresql-intro",
title: "Introduction to PostgreSQL",
content:
"PostgreSQL is a powerful, open-source relational database system. It supports advanced data types and performance optimization features.",
},
{
slug: "mongodb-basics",
title: "MongoDB Basics",
content:
"MongoDB is a NoSQL document database. It stores data in flexible, JSON-like documents instead of tables and rows.",
},
{
slug: "database-indexing",
title: "Database Indexing Strategies",
content:
"Indexing is crucial for database performance. Both B-tree and hash indexes have their use cases depending on query patterns.",
},
];
it("should skip semantic search if OpenAI not configured", async () => {
if (!embeddingService.isConfigured()) {
await expect(
searchService.semanticSearch(
"database performance",
testWorkspaceId
)
).rejects.toThrow();
} else {
// If configured, this is expected to work (tested below)
expect(true).toBe(true);
}
});
it.skipIf(!process.env["OPENAI_API_KEY"])(
"should generate embeddings and perform semantic search",
async () => {
// Create test entries
for (const entry of testEntries) {
const created = await prisma.knowledgeEntry.create({
data: {
workspaceId: testWorkspaceId,
slug: entry.slug,
title: entry.title,
content: entry.content,
status: EntryStatus.PUBLISHED,
visibility: "WORKSPACE",
createdBy: testUserId,
updatedBy: testUserId,
},
});
// Generate embedding
const preparedContent = embeddingService.prepareContentForEmbedding(
entry.title,
entry.content
);
await embeddingService.generateAndStoreEmbedding(
created.id,
preparedContent
);
}
// Wait a bit for embeddings to be stored
await new Promise((resolve) => setTimeout(resolve, 1000));
// Perform semantic search
const results = await searchService.semanticSearch(
"relational database systems",
testWorkspaceId
);
// Should return results
expect(results.data.length).toBeGreaterThan(0);
// PostgreSQL entry should rank high for "relational database"
const postgresEntry = results.data.find(
(r) => r.slug === "postgresql-intro"
);
expect(postgresEntry).toBeDefined();
expect(postgresEntry!.rank).toBeGreaterThan(0);
},
30000 // 30 second timeout for API calls
);
it.skipIf(!process.env["OPENAI_API_KEY"])(
"should perform hybrid search combining vector and keyword",
async () => {
const results = await searchService.hybridSearch(
"indexing",
testWorkspaceId
);
// Should return results
expect(results.data.length).toBeGreaterThan(0);
// Should find the indexing entry
const indexingEntry = results.data.find(
(r) => r.slug === "database-indexing"
);
expect(indexingEntry).toBeDefined();
},
30000
);
});
describe("Batch Embedding Generation", () => {
it.skipIf(!process.env["OPENAI_API_KEY"])(
"should batch generate embeddings",
async () => {
// Create entries without embeddings
const entries = await Promise.all(
Array.from({ length: 3 }, (_, i) =>
prisma.knowledgeEntry.create({
data: {
workspaceId: testWorkspaceId,
slug: `batch-test-${i}`,
title: `Batch Test Entry ${i}`,
content: `This is test content for batch entry ${i}`,
status: EntryStatus.PUBLISHED,
visibility: "WORKSPACE",
createdBy: testUserId,
updatedBy: testUserId,
},
})
)
);
// Batch generate embeddings
const entriesForEmbedding = entries.map((e) => ({
id: e.id,
content: embeddingService.prepareContentForEmbedding(
e.title,
e.content
),
}));
const successCount = await embeddingService.batchGenerateEmbeddings(
entriesForEmbedding
);
expect(successCount).toBe(3);
// Verify embeddings were created
const embeddings = await prisma.knowledgeEmbedding.findMany({
where: {
entryId: { in: entries.map((e) => e.id) },
},
});
expect(embeddings.length).toBe(3);
},
60000 // 60 second timeout for batch operations
);
});
});

346
docs/SEMANTIC_SEARCH.md Normal file
View File

@@ -0,0 +1,346 @@
# Semantic Search Implementation
This document describes the semantic search implementation for the Mosaic Stack Knowledge Module using OpenAI embeddings and PostgreSQL pgvector.
## Overview
The semantic search feature enables AI-powered similarity search across knowledge entries using vector embeddings. It complements the existing full-text search with semantic understanding, allowing users to find relevant content even when exact keywords don't match.
## Architecture
### Components
1. **EmbeddingService** - Generates and manages OpenAI embeddings
2. **SearchService** - Enhanced with semantic and hybrid search methods
3. **KnowledgeService** - Automatically generates embeddings on entry create/update
4. **pgvector** - PostgreSQL extension for vector similarity search
### Database Schema
#### Knowledge Embeddings Table
```prisma
model KnowledgeEmbedding {
id String @id @default(uuid()) @db.Uuid
entryId String @unique @map("entry_id") @db.Uuid
entry KnowledgeEntry @relation(fields: [entryId], references: [id], onDelete: Cascade)
embedding Unsupported("vector(1536)")
model String
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz
updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamptz
@@index([entryId])
@@map("knowledge_embeddings")
}
```
#### Vector Index
An HNSW (Hierarchical Navigable Small World) index is created for fast similarity search:
```sql
CREATE INDEX knowledge_embeddings_embedding_idx
ON knowledge_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
```
## Configuration
### Environment Variables
Add to your `.env` file:
```env
# Optional: Required for semantic search
OPENAI_API_KEY=sk-...
```
Get your API key from: https://platform.openai.com/api-keys
### OpenAI Model
The default embedding model is `text-embedding-3-small` (1536 dimensions). This provides:
- High quality embeddings
- Cost-effective pricing
- Fast generation speed
## API Endpoints
### 1. Semantic Search
**POST** `/api/knowledge/search/semantic`
Search using vector similarity only.
**Request:**
```json
{
"query": "database performance optimization",
"status": "PUBLISHED"
}
```
**Query Parameters:**
- `page` (optional): Page number (default: 1)
- `limit` (optional): Results per page (default: 20)
**Response:**
```json
{
"data": [
{
"id": "uuid",
"slug": "postgres-indexing",
"title": "PostgreSQL Indexing Strategies",
"content": "...",
"rank": 0.87,
"tags": [...],
...
}
],
"pagination": {
"page": 1,
"limit": 20,
"total": 15,
"totalPages": 1
},
"query": "database performance optimization"
}
```
### 2. Hybrid Search (Recommended)
**POST** `/api/knowledge/search/hybrid`
Combines vector similarity and full-text search using Reciprocal Rank Fusion (RRF).
**Request:**
```json
{
"query": "indexing strategies",
"status": "PUBLISHED"
}
```
**Benefits of Hybrid Search:**
- Best of both worlds: semantic understanding + keyword matching
- Better ranking for exact matches
- Improved recall and precision
- Resilient to edge cases
### 3. Batch Embedding Generation
**POST** `/api/knowledge/embeddings/batch`
Generate embeddings for all existing entries. Useful for:
- Initial setup after enabling semantic search
- Regenerating embeddings after model updates
**Request:**
```json
{
"status": "PUBLISHED"
}
```
**Response:**
```json
{
"message": "Generated 42 embeddings out of 45 entries",
"total": 45,
"success": 42
}
```
**Permissions:** Requires ADMIN role
## Automatic Embedding Generation
Embeddings are automatically generated when:
1. **Creating an entry** - Embedding generated asynchronously after creation
2. **Updating an entry** - Embedding regenerated if title or content changes
The generation happens asynchronously to avoid blocking API responses.
### Content Preparation
Before generating embeddings, content is prepared by:
1. Combining title and content
2. Weighting title more heavily (appears twice)
3. This improves semantic matching on titles
```typescript
prepareContentForEmbedding(title, content) {
return `${title}\n\n${title}\n\n${content}`.trim();
}
```
## Search Algorithms
### Vector Similarity Search
Uses cosine distance to find semantically similar entries:
```sql
SELECT *
FROM knowledge_entries e
INNER JOIN knowledge_embeddings emb ON e.id = emb.entry_id
ORDER BY emb.embedding <=> query_embedding
LIMIT 20
```
- `<=>` operator: cosine distance
- Lower distance = higher similarity
- Efficient with HNSW index
### Hybrid Search (RRF Algorithm)
Reciprocal Rank Fusion combines rankings from multiple sources:
```
RRF(d) = sum(1 / (k + rank_i))
```
Where:
- `d` = document
- `k` = constant (60 is standard)
- `rank_i` = rank from source i
**Example:**
Document ranks in two searches:
- Vector search: rank 3
- Keyword search: rank 1
RRF score = 1/(60+3) + 1/(60+1) = 0.0159 + 0.0164 = 0.0323
Higher RRF score = better combined ranking.
## Performance Considerations
### Index Parameters
The HNSW index uses:
- `m = 16`: Max connections per layer (balances accuracy/memory)
- `ef_construction = 64`: Build quality (higher = more accurate, slower build)
### Query Performance
- **Typical query time:** 10-50ms (with index)
- **Without index:** 1000ms+ (not recommended)
- **Embedding generation:** 100-300ms per entry
### Cost (OpenAI API)
Using `text-embedding-3-small`:
- ~$0.00002 per 1000 tokens
- Average entry (~500 tokens): $0.00001
- 10,000 entries: ~$0.10
Very cost-effective for most use cases.
## Migration Guide
### 1. Run Migrations
```bash
cd apps/api
pnpm prisma migrate deploy
```
This creates:
- `knowledge_embeddings` table
- Vector index on embeddings
### 2. Configure OpenAI API Key
```bash
# Add to .env
OPENAI_API_KEY=sk-...
```
### 3. Generate Embeddings for Existing Entries
```bash
curl -X POST http://localhost:3001/api/knowledge/embeddings/batch \
-H "Authorization: Bearer YOUR_TOKEN" \
-H "Content-Type: application/json" \
-d '{"status": "PUBLISHED"}'
```
Or use the web UI (Admin dashboard → Knowledge → Generate Embeddings).
### 4. Test Semantic Search
```bash
curl -X POST http://localhost:3001/api/knowledge/search/hybrid \
-H "Authorization: Bearer YOUR_TOKEN" \
-H "Content-Type: application/json" \
-d '{"query": "your search query"}'
```
## Troubleshooting
### "OpenAI API key not configured"
**Cause:** `OPENAI_API_KEY` environment variable not set
**Solution:** Add the API key to your `.env` file and restart the API server
### Semantic search returns no results
**Possible causes:**
1. **No embeddings generated**
- Run batch generation endpoint
- Check `knowledge_embeddings` table
2. **Query too specific**
- Try broader terms
- Use hybrid search for better recall
3. **Index not created**
- Check migration status
- Verify index exists: `\di knowledge_embeddings_embedding_idx` in psql
### Slow query performance
**Solutions:**
1. Verify index exists and is being used:
```sql
EXPLAIN ANALYZE
SELECT * FROM knowledge_embeddings
ORDER BY embedding <=> '[...]'::vector
LIMIT 20;
```
2. Adjust index parameters (requires recreation):
```sql
DROP INDEX knowledge_embeddings_embedding_idx;
CREATE INDEX knowledge_embeddings_embedding_idx
ON knowledge_embeddings
USING hnsw (embedding vector_cosine_ops)
WITH (m = 32, ef_construction = 128); -- Higher values
```
## Future Enhancements
Potential improvements:
1. **Custom embeddings**: Support for local embedding models (Ollama, etc.)
2. **Chunking**: Split large entries into chunks for better granularity
3. **Reranking**: Add cross-encoder reranking for top results
4. **Caching**: Cache query embeddings for repeated searches
5. **Multi-modal**: Support image/file embeddings
## References
- [OpenAI Embeddings Guide](https://platform.openai.com/docs/guides/embeddings)
- [pgvector Documentation](https://github.com/pgvector/pgvector)
- [HNSW Algorithm Paper](https://arxiv.org/abs/1603.09320)
- [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf)

20
pnpm-lock.yaml generated
View File

@@ -113,6 +113,9 @@ importers:
ollama:
specifier: ^0.6.3
version: 0.6.3
openai:
specifier: ^6.17.0
version: 6.17.0(ws@8.19.0)(zod@4.3.6)
reflect-metadata:
specifier: ^0.2.2
version: 0.2.2
@@ -4076,6 +4079,18 @@ packages:
resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==}
engines: {node: '>=18'}
openai@6.17.0:
resolution: {integrity: sha512-NHRpPEUPzAvFOAFs9+9pC6+HCw/iWsYsKCMPXH5Kw7BpMxqd8g/A07/1o7Gx2TWtCnzevVRyKMRFqyiHyAlqcA==}
hasBin: true
peerDependencies:
ws: ^8.18.0
zod: ^3.25 || ^4.0
peerDependenciesMeta:
ws:
optional: true
zod:
optional: true
optionator@0.9.4:
resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==}
engines: {node: '>= 0.8.0'}
@@ -9134,6 +9149,11 @@ snapshots:
is-inside-container: 1.0.0
wsl-utils: 0.1.0
openai@6.17.0(ws@8.19.0)(zod@4.3.6):
optionalDependencies:
ws: 8.19.0
zod: 4.3.6
optionator@0.9.4:
dependencies:
deep-is: 0.1.4