feat(#69): implement embedding generation pipeline
Generate embeddings for knowledge entries using Ollama via BullMQ job queue. Changes: - Created OllamaEmbeddingService for Ollama-based embedding generation - Set up BullMQ queue and processor for async embedding jobs - Integrated queue into knowledge entry lifecycle (create/update) - Added rate limiting (1 job/second) and retry logic (3 attempts) - Added OLLAMA_EMBEDDING_MODEL environment variable configuration - Implemented dimension normalization (padding/truncating to 1536 dimensions) - Added graceful degradation when Ollama is unavailable Test Coverage: - All 31 embedding-related tests passing - ollama-embedding.service.spec.ts: 13 tests - embedding-queue.spec.ts: 6 tests - embedding.processor.spec.ts: 5 tests - Build and linting successful Fixes #69 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { Injectable, NotFoundException, ConflictException } from "@nestjs/common";
|
||||
import { Injectable, NotFoundException, ConflictException, Logger } from "@nestjs/common";
|
||||
import { EntryStatus, Prisma } from "@prisma/client";
|
||||
import slugify from "slugify";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
@@ -12,17 +12,23 @@ import { renderMarkdown } from "./utils/markdown";
|
||||
import { LinkSyncService } from "./services/link-sync.service";
|
||||
import { KnowledgeCacheService } from "./services/cache.service";
|
||||
import { EmbeddingService } from "./services/embedding.service";
|
||||
import { OllamaEmbeddingService } from "./services/ollama-embedding.service";
|
||||
import { EmbeddingQueueService } from "./queues/embedding-queue.service";
|
||||
|
||||
/**
|
||||
* Service for managing knowledge entries
|
||||
*/
|
||||
@Injectable()
|
||||
export class KnowledgeService {
|
||||
private readonly logger = new Logger(KnowledgeService.name);
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly linkSync: LinkSyncService,
|
||||
private readonly cache: KnowledgeCacheService,
|
||||
private readonly embedding: EmbeddingService
|
||||
private readonly embedding: EmbeddingService,
|
||||
private readonly ollamaEmbedding: OllamaEmbeddingService,
|
||||
private readonly embeddingQueue: EmbeddingQueueService
|
||||
) {}
|
||||
|
||||
/**
|
||||
@@ -851,14 +857,22 @@ export class KnowledgeService {
|
||||
/**
|
||||
* Generate and store embedding for a knowledge entry
|
||||
* Private helper method called asynchronously after entry create/update
|
||||
* Queues the embedding generation job instead of processing synchronously
|
||||
*/
|
||||
private async generateEntryEmbedding(
|
||||
entryId: string,
|
||||
title: string,
|
||||
content: string
|
||||
): Promise<void> {
|
||||
const combinedContent = this.embedding.prepareContentForEmbedding(title, content);
|
||||
await this.embedding.generateAndStoreEmbedding(entryId, combinedContent);
|
||||
const combinedContent = this.ollamaEmbedding.prepareContentForEmbedding(title, content);
|
||||
|
||||
try {
|
||||
const jobId = await this.embeddingQueue.queueEmbeddingJob(entryId, combinedContent);
|
||||
this.logger.log(`Queued embedding job ${jobId} for entry ${entryId}`);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to queue embedding job for entry ${entryId}`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user