feat(#69): implement embedding generation pipeline

Generate embeddings for knowledge entries using Ollama via BullMQ job queue.

Changes:
- Created OllamaEmbeddingService for Ollama-based embedding generation
- Set up BullMQ queue and processor for async embedding jobs
- Integrated queue into knowledge entry lifecycle (create/update)
- Added rate limiting (1 job/second) and retry logic (3 attempts)
- Added OLLAMA_EMBEDDING_MODEL environment variable configuration
- Implemented dimension normalization (padding/truncating to 1536 dimensions)
- Added graceful degradation when Ollama is unavailable

Test Coverage:
- All 31 embedding-related tests passing
- ollama-embedding.service.spec.ts: 13 tests
- embedding-queue.spec.ts: 6 tests
- embedding.processor.spec.ts: 5 tests
- Build and linting successful

Fixes #69

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 15:06:11 -06:00
parent 3cb6eb7f8b
commit 3dfa603a03
12 changed files with 1099 additions and 6 deletions

View File

@@ -1,6 +1,8 @@
import { Module } from "@nestjs/common";
import { BullModule } from "@nestjs/bullmq";
import { PrismaModule } from "../prisma/prisma.module";
import { AuthModule } from "../auth/auth.module";
import { OllamaModule } from "../ollama/ollama.module";
import { KnowledgeService } from "./knowledge.service";
import {
KnowledgeController,
@@ -18,9 +20,26 @@ import {
KnowledgeCacheService,
EmbeddingService,
} from "./services";
import { OllamaEmbeddingService } from "./services/ollama-embedding.service";
import { EmbeddingQueueService } from "./queues/embedding-queue.service";
import { EmbeddingProcessor } from "./queues/embedding.processor";
@Module({
imports: [PrismaModule, AuthModule],
imports: [
PrismaModule,
AuthModule,
OllamaModule,
BullModule.registerQueue({
name: "embeddings",
defaultJobOptions: {
attempts: 3,
backoff: {
type: "exponential",
delay: 5000,
},
},
}),
],
controllers: [
KnowledgeController,
KnowledgeCacheController,
@@ -37,7 +56,17 @@ import {
StatsService,
KnowledgeCacheService,
EmbeddingService,
OllamaEmbeddingService,
EmbeddingQueueService,
EmbeddingProcessor,
],
exports: [
KnowledgeService,
LinkResolutionService,
SearchService,
EmbeddingService,
OllamaEmbeddingService,
EmbeddingQueueService,
],
exports: [KnowledgeService, LinkResolutionService, SearchService, EmbeddingService],
})
export class KnowledgeModule {}