Jason WoltjeandClaude Sonnet 4.5
3dfa603a03
Generate embeddings for knowledge entries using Ollama via BullMQ job queue. Changes: - Created OllamaEmbeddingService for Ollama-based embedding generation - Set up BullMQ queue and processor for async embedding jobs - Integrated queue into knowledge entry lifecycle (create/update) - Added rate limiting (1 job/second) and retry logic (3 attempts) - Added OLLAMA_EMBEDDING_MODEL environment variable configuration - Implemented dimension normalization (padding/truncating to 1536 dimensions) - Added graceful degradation when Ollama is unavailable Test Coverage: - All 31 embedding-related tests passing - ollama-embedding.service.spec.ts: 13 tests - embedding-queue.spec.ts: 6 tests - embedding.processor.spec.ts: 5 tests - Build and linting successful Fixes #69 Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
115 lines
3.1 KiB
TypeScript
115 lines
3.1 KiB
TypeScript
import { Injectable, Logger } from "@nestjs/common";
|
|
import { InjectQueue } from "@nestjs/bullmq";
|
|
import { Queue } from "bullmq";
|
|
|
|
export interface EmbeddingJobData {
|
|
entryId: string;
|
|
content: string;
|
|
model?: string;
|
|
}
|
|
|
|
/**
|
|
* Service for managing the embedding generation queue
|
|
*
|
|
* This service provides an interface to queue embedding jobs
|
|
* and manage the queue lifecycle.
|
|
*/
|
|
@Injectable()
|
|
export class EmbeddingQueueService {
|
|
private readonly logger = new Logger(EmbeddingQueueService.name);
|
|
|
|
constructor(
|
|
@InjectQueue("embeddings")
|
|
private readonly embeddingQueue: Queue<EmbeddingJobData>
|
|
) {}
|
|
|
|
/**
|
|
* Queue an embedding generation job
|
|
*
|
|
* @param entryId - ID of the knowledge entry
|
|
* @param content - Content to generate embedding for
|
|
* @param model - Optional model override
|
|
* @returns Job ID
|
|
*/
|
|
async queueEmbeddingJob(entryId: string, content: string, model?: string): Promise<string> {
|
|
const jobData: EmbeddingJobData = {
|
|
entryId,
|
|
content,
|
|
};
|
|
|
|
if (model !== undefined) {
|
|
jobData.model = model;
|
|
}
|
|
|
|
const job = await this.embeddingQueue.add("generate-embedding", jobData, {
|
|
// Retry configuration
|
|
attempts: 3,
|
|
backoff: {
|
|
type: "exponential",
|
|
delay: 5000, // Start with 5 seconds
|
|
},
|
|
// Rate limiting: 1 job per second to avoid overwhelming Ollama
|
|
delay: 1000,
|
|
// Remove completed jobs after 24 hours
|
|
removeOnComplete: {
|
|
age: 86400, // 24 hours in seconds
|
|
count: 1000, // Keep max 1000 completed jobs
|
|
},
|
|
// Remove failed jobs after 7 days
|
|
removeOnFail: {
|
|
age: 604800, // 7 days in seconds
|
|
count: 100, // Keep max 100 failed jobs for debugging
|
|
},
|
|
});
|
|
|
|
this.logger.log(`Queued embedding job ${job.id ?? "unknown"} for entry ${entryId}`);
|
|
return job.id ?? "unknown";
|
|
}
|
|
|
|
/**
|
|
* Get queue statistics
|
|
*
|
|
* @returns Queue job counts
|
|
*/
|
|
async getQueueStats(): Promise<{
|
|
waiting: number;
|
|
active: number;
|
|
completed: number;
|
|
failed: number;
|
|
}> {
|
|
const counts = await this.embeddingQueue.getJobCounts(
|
|
"waiting",
|
|
"active",
|
|
"completed",
|
|
"failed"
|
|
);
|
|
|
|
return {
|
|
waiting: counts.waiting ?? 0,
|
|
active: counts.active ?? 0,
|
|
completed: counts.completed ?? 0,
|
|
failed: counts.failed ?? 0,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Clean completed jobs older than the grace period
|
|
*
|
|
* @param gracePeriodMs - Grace period in milliseconds (default: 24 hours)
|
|
*/
|
|
async cleanCompletedJobs(gracePeriodMs = 86400000): Promise<void> {
|
|
await this.embeddingQueue.clean(gracePeriodMs, 100, "completed");
|
|
this.logger.log(`Cleaned completed jobs older than ${gracePeriodMs.toString()}ms`);
|
|
}
|
|
|
|
/**
|
|
* Clean failed jobs older than the grace period
|
|
*
|
|
* @param gracePeriodMs - Grace period in milliseconds (default: 7 days)
|
|
*/
|
|
async cleanFailedJobs(gracePeriodMs = 604800000): Promise<void> {
|
|
await this.embeddingQueue.clean(gracePeriodMs, 100, "failed");
|
|
this.logger.log(`Cleaned failed jobs older than ${gracePeriodMs.toString()}ms`);
|
|
}
|
|
}
|