feat(#69): implement embedding generation pipeline
Generate embeddings for knowledge entries using Ollama via BullMQ job queue. Changes: - Created OllamaEmbeddingService for Ollama-based embedding generation - Set up BullMQ queue and processor for async embedding jobs - Integrated queue into knowledge entry lifecycle (create/update) - Added rate limiting (1 job/second) and retry logic (3 attempts) - Added OLLAMA_EMBEDDING_MODEL environment variable configuration - Implemented dimension normalization (padding/truncating to 1536 dimensions) - Added graceful degradation when Ollama is unavailable Test Coverage: - All 31 embedding-related tests passing - ollama-embedding.service.spec.ts: 13 tests - embedding-queue.spec.ts: 6 tests - embedding.processor.spec.ts: 5 tests - Build and linting successful Fixes #69 Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { BullModule } from "@nestjs/bullmq";
|
||||
import { PrismaModule } from "../prisma/prisma.module";
|
||||
import { AuthModule } from "../auth/auth.module";
|
||||
import { OllamaModule } from "../ollama/ollama.module";
|
||||
import { KnowledgeService } from "./knowledge.service";
|
||||
import {
|
||||
KnowledgeController,
|
||||
@@ -18,9 +20,26 @@ import {
|
||||
KnowledgeCacheService,
|
||||
EmbeddingService,
|
||||
} from "./services";
|
||||
import { OllamaEmbeddingService } from "./services/ollama-embedding.service";
|
||||
import { EmbeddingQueueService } from "./queues/embedding-queue.service";
|
||||
import { EmbeddingProcessor } from "./queues/embedding.processor";
|
||||
|
||||
@Module({
|
||||
imports: [PrismaModule, AuthModule],
|
||||
imports: [
|
||||
PrismaModule,
|
||||
AuthModule,
|
||||
OllamaModule,
|
||||
BullModule.registerQueue({
|
||||
name: "embeddings",
|
||||
defaultJobOptions: {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: "exponential",
|
||||
delay: 5000,
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
controllers: [
|
||||
KnowledgeController,
|
||||
KnowledgeCacheController,
|
||||
@@ -37,7 +56,17 @@ import {
|
||||
StatsService,
|
||||
KnowledgeCacheService,
|
||||
EmbeddingService,
|
||||
OllamaEmbeddingService,
|
||||
EmbeddingQueueService,
|
||||
EmbeddingProcessor,
|
||||
],
|
||||
exports: [
|
||||
KnowledgeService,
|
||||
LinkResolutionService,
|
||||
SearchService,
|
||||
EmbeddingService,
|
||||
OllamaEmbeddingService,
|
||||
EmbeddingQueueService,
|
||||
],
|
||||
exports: [KnowledgeService, LinkResolutionService, SearchService, EmbeddingService],
|
||||
})
|
||||
export class KnowledgeModule {}
|
||||
|
||||
Reference in New Issue
Block a user