feat(#69): implement embedding generation pipeline

Generate embeddings for knowledge entries using Ollama via BullMQ job queue.

Changes:
- Created OllamaEmbeddingService for Ollama-based embedding generation
- Set up BullMQ queue and processor for async embedding jobs
- Integrated queue into knowledge entry lifecycle (create/update)
- Added rate limiting (1 job/second) and retry logic (3 attempts)
- Added OLLAMA_EMBEDDING_MODEL environment variable configuration
- Implemented dimension normalization (padding/truncating to 1536 dimensions)
- Added graceful degradation when Ollama is unavailable

Test Coverage:
- All 31 embedding-related tests passing
- ollama-embedding.service.spec.ts: 13 tests
- embedding-queue.spec.ts: 6 tests
- embedding.processor.spec.ts: 5 tests
- Build and linting successful

Fixes #69

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 15:06:11 -06:00
parent 3cb6eb7f8b
commit 3dfa603a03
12 changed files with 1099 additions and 6 deletions

View File

@@ -1,6 +1,7 @@
import { Module } from "@nestjs/common";
import { APP_INTERCEPTOR, APP_GUARD } from "@nestjs/core";
import { ThrottlerModule } from "@nestjs/throttler";
import { BullModule } from "@nestjs/bullmq";
import { ThrottlerValkeyStorageService, ThrottlerApiKeyGuard } from "./common/throttler";
import { AppController } from "./app.controller";
import { AppService } from "./app.service";
@@ -50,6 +51,13 @@ import { CoordinatorIntegrationModule } from "./coordinator-integration/coordina
};
},
}),
// BullMQ job queue configuration
BullModule.forRoot({
connection: {
host: process.env.VALKEY_HOST ?? "localhost",
port: parseInt(process.env.VALKEY_PORT ?? "6379", 10),
},
}),
TelemetryModule,
PrismaModule,
DatabaseModule,