feat(#69): implement embedding generation pipeline

Generate embeddings for knowledge entries using Ollama via BullMQ job queue.

Changes:
- Created OllamaEmbeddingService for Ollama-based embedding generation
- Set up BullMQ queue and processor for async embedding jobs
- Integrated queue into knowledge entry lifecycle (create/update)
- Added rate limiting (1 job/second) and retry logic (3 attempts)
- Added OLLAMA_EMBEDDING_MODEL environment variable configuration
- Implemented dimension normalization (padding/truncating to 1536 dimensions)
- Added graceful degradation when Ollama is unavailable

Test Coverage:
- All 31 embedding-related tests passing
- ollama-embedding.service.spec.ts: 13 tests
- embedding-queue.spec.ts: 6 tests
- embedding.processor.spec.ts: 5 tests
- Build and linting successful

Fixes #69

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 15:06:11 -06:00
parent 3cb6eb7f8b
commit 3dfa603a03
12 changed files with 1099 additions and 6 deletions

View File

@@ -0,0 +1,95 @@
import { Processor, WorkerHost } from "@nestjs/bullmq";
import { Logger } from "@nestjs/common";
import { Job } from "bullmq";
import { OllamaEmbeddingService } from "../services/ollama-embedding.service";
import { EmbeddingJobData } from "./embedding-queue.service";
/**
* Processor for embedding generation jobs
*
* This worker processes queued embedding jobs and generates
* embeddings for knowledge entries using Ollama.
*/
@Processor("embeddings")
export class EmbeddingProcessor extends WorkerHost {
private readonly logger = new Logger(EmbeddingProcessor.name);
constructor(private readonly embeddingService: OllamaEmbeddingService) {
super();
}
/**
* Process an embedding generation job
*
* @param job - The embedding job to process
*/
async process(job: Job<EmbeddingJobData>): Promise<void> {
const { entryId, content, model } = job.data;
this.logger.log(`Processing embedding job ${job.id ?? "unknown"} for entry ${entryId}`);
try {
const options: { model?: string } = {};
if (model !== undefined) {
options.model = model;
}
await this.embeddingService.generateAndStoreEmbedding(entryId, content, options);
this.logger.log(
`Successfully generated embedding for entry ${entryId} (job: ${job.id ?? "unknown"})`
);
} catch (error) {
this.logger.error(
`Failed to generate embedding for entry ${entryId} (job: ${job.id ?? "unknown"})`,
error
);
throw error; // Re-throw to trigger retry logic
}
}
/**
* Handle successful job completion
*
* @param job - The completed job
*/
onCompleted(job: Job<EmbeddingJobData>): void {
this.logger.log(
`Successfully generated embedding for entry ${job.data.entryId} (job: ${job.id ?? "unknown"})`
);
}
/**
* Handle job failure
*
* @param job - The failed job
* @param error - The error that caused the failure
*/
onFailed(job: Job<EmbeddingJobData>, error: Error): void {
this.logger.error(
`Failed to generate embedding for entry ${job.data.entryId} (job: ${job.id ?? "unknown"}) after ${job.attemptsMade.toString()} attempts`,
error
);
}
/**
* Alias for process to match test expectations
*/
async processEmbedding(job: Job<EmbeddingJobData>): Promise<void> {
return this.process(job);
}
/**
* Alias for onCompleted to match test expectations
*/
handleCompleted(job: Job<EmbeddingJobData>): void {
this.onCompleted(job);
}
/**
* Alias for onFailed to match test expectations
*/
handleFailed(job: Job<EmbeddingJobData>, error: Error): void {
this.onFailed(job, error);
}
}