import { Injectable, Logger, OnModuleInit } from "@nestjs/common"; import { PrismaService } from "../prisma/prisma.service"; import type { LlmProviderInstance } from "@prisma/client"; import type { LlmProviderInterface, LlmProviderHealthStatus, } from "./providers/llm-provider.interface"; import { OllamaProvider, type OllamaProviderConfig } from "./providers/ollama.provider"; /** * Provider information returned by getAllProviders */ export interface ProviderInfo { id: string; name: string; provider: string; endpoint?: string; userId?: string | null; isDefault: boolean; } /** * Provider health status with instance ID */ export interface ProviderHealthInfo extends LlmProviderHealthStatus { id: string; } /** * LLM Manager Service * * Manages multiple LLM provider instances and routes requests. * Supports hot reload, provider selection, and health monitoring. * * @example * ```typescript * // Get default provider * const provider = await llmManager.getDefaultProvider(); * * // Get user-specific provider * const userProvider = await llmManager.getUserProvider("user-123"); * * // Register new provider dynamically * await llmManager.registerProvider(providerInstance); * * // Reload from database * await llmManager.reloadFromDatabase(); * ``` */ @Injectable() export class LlmManagerService implements OnModuleInit { private readonly logger = new Logger(LlmManagerService.name); private readonly providers = new Map(); private readonly instanceMetadata = new Map(); constructor(private readonly prisma: PrismaService) {} /** * Initialize the service by loading all enabled providers from the database. * Called automatically by NestJS during module initialization. * * @throws {Error} If database connection fails */ async onModuleInit(): Promise { this.logger.log("Initializing LLM Manager Service..."); try { await this.loadProvidersFromDatabase(); this.logger.log(`Loaded ${String(this.providers.size)} provider(s)`); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.error(`Failed to initialize LLM Manager: ${errorMessage}`); throw new Error(`Failed to load providers from database: ${errorMessage}`); } } /** * Register a new provider instance or update an existing one. * Supports hot reload - no restart required. * * @param instance - Provider instance from database * @throws {Error} If provider type is unknown or initialization fails */ async registerProvider(instance: LlmProviderInstance): Promise { try { this.logger.log(`Registering provider: ${instance.displayName} (${instance.id})`); // Create provider instance based on type const provider = this.createProvider(instance); // Initialize the provider await provider.initialize(); // Store in registry this.providers.set(instance.id, provider); this.instanceMetadata.set(instance.id, instance); this.logger.log(`Provider registered successfully: ${instance.displayName}`); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.error(`Failed to register provider ${instance.id}: ${errorMessage}`); throw error; } } /** * Unregister a provider instance from the registry. * Supports hot reload - no restart required. * * @param instanceId - Provider instance ID */ // eslint-disable-next-line @typescript-eslint/require-await async unregisterProvider(instanceId: string): Promise { if (this.providers.has(instanceId)) { this.logger.log(`Unregistering provider: ${instanceId}`); this.providers.delete(instanceId); this.instanceMetadata.delete(instanceId); } } /** * Get a provider by its instance ID. * * @param instanceId - Provider instance ID * @returns Provider interface * @throws {Error} If provider not found */ // eslint-disable-next-line @typescript-eslint/require-await async getProviderById(instanceId: string): Promise { const provider = this.providers.get(instanceId); if (!provider) { throw new Error(`Provider with ID ${instanceId} not found`); } return provider; } /** * Get all active provider instances. * * @returns Array of provider information */ // eslint-disable-next-line @typescript-eslint/require-await async getAllProviders(): Promise { const providers: ProviderInfo[] = []; for (const [id, provider] of this.providers.entries()) { const metadata = this.instanceMetadata.get(id); const config = provider.getConfig(); providers.push({ id, name: metadata?.displayName ?? provider.name, provider: provider.type, endpoint: config.endpoint, userId: metadata?.userId ?? null, isDefault: metadata?.isDefault ?? false, }); } return providers; } /** * Get the default system-level provider. * Default provider is identified by isDefault=true and userId=null. * * @returns Default provider interface * @throws {Error} If no default provider configured */ // eslint-disable-next-line @typescript-eslint/require-await async getDefaultProvider(): Promise { // Find default system-level provider (userId = null, isDefault = true) for (const [id, metadata] of this.instanceMetadata.entries()) { if (metadata.isDefault && metadata.userId === null) { const provider = this.providers.get(id); if (provider) { return provider; } } } throw new Error("No default provider configured"); } /** * Get a provider for a specific user. * Prioritizes user-level providers over system default. * * Selection logic: * 1. User-specific provider (userId matches) * 2. System default provider (userId = null, isDefault = true) * * @param userId - User ID * @returns Provider interface * @throws {Error} If no provider available for user */ async getUserProvider(userId: string): Promise { // First, try to find user-specific provider for (const [id, metadata] of this.instanceMetadata.entries()) { if (metadata.userId === userId) { const provider = this.providers.get(id); if (provider) { return provider; } } } // Fall back to default provider try { return await this.getDefaultProvider(); } catch { throw new Error(`No provider available for user ${userId}`); } } /** * Check health of all registered providers. * * @returns Array of health status information */ async checkAllProvidersHealth(): Promise { const healthStatuses: ProviderHealthInfo[] = []; for (const [id, provider] of this.providers.entries()) { try { const health = await provider.checkHealth(); healthStatuses.push({ id, ...health }); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.warn(`Health check failed for provider ${id}: ${errorMessage}`); // Include failed health check in results healthStatuses.push({ id, healthy: false, provider: provider.type, error: errorMessage, }); } } return healthStatuses; } /** * Reload all providers from the database. * Clears existing providers and loads fresh state from database. * Supports hot reload - no restart required. * * @throws {Error} If database connection fails */ async reloadFromDatabase(): Promise { this.logger.log("Reloading providers from database..."); // Clear existing providers this.providers.clear(); this.instanceMetadata.clear(); // Reload from database await this.loadProvidersFromDatabase(); this.logger.log(`Reloaded ${String(this.providers.size)} provider(s)`); } /** * Load all enabled providers from the database. * Private helper method called during initialization and reload. * * @throws {Error} If database query fails */ private async loadProvidersFromDatabase(): Promise { const instances = await this.prisma.llmProviderInstance.findMany({ where: { isEnabled: true }, }); // Register each provider instance for (const instance of instances) { try { await this.registerProvider(instance); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.warn(`Skipping provider ${instance.id} due to error: ${errorMessage}`); } } } /** * Create a provider instance based on provider type. * Factory method for instantiating provider implementations. * * @param instance - Provider instance from database * @returns Provider interface implementation * @throws {Error} If provider type is unknown */ private createProvider(instance: LlmProviderInstance): LlmProviderInterface { switch (instance.providerType) { case "ollama": return new OllamaProvider(instance.config as OllamaProviderConfig); // Future providers: // case "claude": // return new ClaudeProvider(instance.config); // case "openai": // return new OpenAIProvider(instance.config); default: throw new Error(`Unknown provider type: ${instance.providerType}`); } } }