feat(#69): implement embedding generation pipeline

Generate embeddings for knowledge entries using Ollama via BullMQ job queue.

Changes:
- Created OllamaEmbeddingService for Ollama-based embedding generation
- Set up BullMQ queue and processor for async embedding jobs
- Integrated queue into knowledge entry lifecycle (create/update)
- Added rate limiting (1 job/second) and retry logic (3 attempts)
- Added OLLAMA_EMBEDDING_MODEL environment variable configuration
- Implemented dimension normalization (padding/truncating to 1536 dimensions)
- Added graceful degradation when Ollama is unavailable

Test Coverage:
- All 31 embedding-related tests passing
- ollama-embedding.service.spec.ts: 13 tests
- embedding-queue.spec.ts: 6 tests
- embedding.processor.spec.ts: 5 tests
- Build and linting successful

Fixes #69

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Jason Woltje
2026-02-02 15:06:11 -06:00
parent 3cb6eb7f8b
commit 3dfa603a03
12 changed files with 1099 additions and 6 deletions

View File

@@ -1,4 +1,4 @@
import { Injectable, NotFoundException, ConflictException } from "@nestjs/common";
import { Injectable, NotFoundException, ConflictException, Logger } from "@nestjs/common";
import { EntryStatus, Prisma } from "@prisma/client";
import slugify from "slugify";
import { PrismaService } from "../prisma/prisma.service";
@@ -12,17 +12,23 @@ import { renderMarkdown } from "./utils/markdown";
import { LinkSyncService } from "./services/link-sync.service";
import { KnowledgeCacheService } from "./services/cache.service";
import { EmbeddingService } from "./services/embedding.service";
import { OllamaEmbeddingService } from "./services/ollama-embedding.service";
import { EmbeddingQueueService } from "./queues/embedding-queue.service";
/**
* Service for managing knowledge entries
*/
@Injectable()
export class KnowledgeService {
private readonly logger = new Logger(KnowledgeService.name);
constructor(
private readonly prisma: PrismaService,
private readonly linkSync: LinkSyncService,
private readonly cache: KnowledgeCacheService,
private readonly embedding: EmbeddingService
private readonly embedding: EmbeddingService,
private readonly ollamaEmbedding: OllamaEmbeddingService,
private readonly embeddingQueue: EmbeddingQueueService
) {}
/**
@@ -851,14 +857,22 @@ export class KnowledgeService {
/**
* Generate and store embedding for a knowledge entry
* Private helper method called asynchronously after entry create/update
* Queues the embedding generation job instead of processing synchronously
*/
private async generateEntryEmbedding(
entryId: string,
title: string,
content: string
): Promise<void> {
const combinedContent = this.embedding.prepareContentForEmbedding(title, content);
await this.embedding.generateAndStoreEmbedding(entryId, combinedContent);
const combinedContent = this.ollamaEmbedding.prepareContentForEmbedding(title, content);
try {
const jobId = await this.embeddingQueue.queueEmbeddingJob(entryId, combinedContent);
this.logger.log(`Queued embedding job ${jobId} for entry ${entryId}`);
} catch (error) {
this.logger.error(`Failed to queue embedding job for entry ${entryId}`, error);
throw error;
}
}
/**