import "reflect-metadata"; import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { TraceLlmCall, createLlmSpan, recordLlmUsage, type LlmTraceMetadata, } from "./llm-telemetry.decorator"; import { trace, SpanStatusCode, type Span } from "@opentelemetry/api"; describe("LlmTelemetryDecorator", () => { describe("@TraceLlmCall", () => { class TestLlmProvider { callCount = 0; @TraceLlmCall({ system: "ollama", operation: "chat" }) async chat(request: { model: string; messages: unknown[] }): Promise<{ content: string; promptEvalCount: number; evalCount: number; }> { this.callCount++; return { content: "Test response", promptEvalCount: 10, evalCount: 20, }; } @TraceLlmCall({ system: "ollama", operation: "embed" }) async embed(request: { model: string; input: string }): Promise<{ embedding: number[] }> { this.callCount++; return { embedding: [0.1, 0.2, 0.3], }; } @TraceLlmCall({ system: "ollama", operation: "error" }) async throwError(): Promise { this.callCount++; throw new Error("Test error"); } } let provider: TestLlmProvider; beforeEach(() => { provider = new TestLlmProvider(); }); it("should execute the original method", async () => { const result = await provider.chat({ model: "llama2", messages: [{ role: "user", content: "Hello" }], }); expect(result.content).toBe("Test response"); expect(provider.callCount).toBe(1); }); it("should create a span and not throw errors", async () => { // Test that the decorator creates spans without errors const result = await provider.chat({ model: "llama2", messages: [], }); expect(result).toBeDefined(); expect(provider.callCount).toBe(1); }); it("should set gen_ai.system attribute", async () => { const result = await provider.chat({ model: "llama2", messages: [], }); expect(result).toBeDefined(); // Span attributes are set internally, verifying execution doesn't throw }); it("should set gen_ai.operation.name attribute", async () => { const result = await provider.embed({ model: "nomic-embed-text", input: "test text", }); expect(result).toBeDefined(); // Span attributes are set internally, verifying execution doesn't throw }); it("should extract model from request", async () => { const result = await provider.chat({ model: "llama2", messages: [], }); expect(result).toBeDefined(); // Model attribute is set internally from request.model }); it("should record token usage from response", async () => { const result = await provider.chat({ model: "llama2", messages: [], }); expect(result.promptEvalCount).toBe(10); expect(result.evalCount).toBe(20); // Token usage attributes are set internally }); it("should set span status to OK on success", async () => { const result = await provider.chat({ model: "llama2", messages: [], }); expect(result).toBeDefined(); // Span status is set internally }); it("should record exception on error", async () => { await expect(provider.throwError()).rejects.toThrow("Test error"); expect(provider.callCount).toBe(1); }); it("should set span status to ERROR on exception", async () => { await expect(provider.throwError()).rejects.toThrow("Test error"); // Span status is set to ERROR internally }); it("should propagate the original error", async () => { try { await provider.throwError(); expect.fail("Should have thrown an error"); } catch (error) { expect(error).toBeInstanceOf(Error); expect((error as Error).message).toBe("Test error"); } }); it("should end span after execution", async () => { await provider.chat({ model: "llama2", messages: [], }); // Span is ended internally after method execution expect(provider.callCount).toBe(1); }); it("should handle requests without model property", async () => { class NoModelProvider { @TraceLlmCall({ system: "test", operation: "test" }) async noModel(): Promise { return "test"; } } const noModelProvider = new NoModelProvider(); const result = await noModelProvider.noModel(); expect(result).toBe("test"); }); it("should handle responses without token usage", async () => { class NoTokensProvider { @TraceLlmCall({ system: "test", operation: "test" }) async noTokens(): Promise<{ data: string }> { return { data: "test" }; } } const noTokensProvider = new NoTokensProvider(); const result = await noTokensProvider.noTokens(); expect(result.data).toBe("test"); }); it("should record response duration", async () => { const result = await provider.chat({ model: "llama2", messages: [], }); expect(result).toBeDefined(); // Duration is calculated and set as span attribute internally }); it("should support different LLM systems", async () => { class MultiSystemProvider { @TraceLlmCall({ system: "openai", operation: "completion" }) async openai(): Promise { return "openai"; } @TraceLlmCall({ system: "anthropic", operation: "message" }) async anthropic(): Promise { return "anthropic"; } } const multiProvider = new MultiSystemProvider(); const openaiResult = await multiProvider.openai(); const anthropicResult = await multiProvider.anthropic(); expect(openaiResult).toBe("openai"); expect(anthropicResult).toBe("anthropic"); }); }); describe("createLlmSpan", () => { it("should create a span with system and operation", () => { const span = createLlmSpan("ollama", "chat"); expect(span).toBeDefined(); span.end(); }); it("should create a span with model", () => { const span = createLlmSpan("ollama", "chat", "llama2"); expect(span).toBeDefined(); span.end(); }); it("should create span with correct name format", () => { const span = createLlmSpan("openai", "completion"); expect(span).toBeDefined(); // Span name is "openai.completion" span.end(); }); it("should handle missing model gracefully", () => { const span = createLlmSpan("ollama", "embed"); expect(span).toBeDefined(); span.end(); }); it("should allow manual span management", () => { const span = createLlmSpan("ollama", "chat.stream", "llama2"); // Simulate stream operation span.setAttribute("chunk.count", 5); span.setStatus({ code: SpanStatusCode.OK }); expect(span).toBeDefined(); span.end(); }); }); describe("recordLlmUsage", () => { let span: Span; beforeEach(() => { span = createLlmSpan("test", "test"); }); afterEach(() => { span.end(); }); it("should record prompt tokens", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, 100); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.prompt_tokens", 100); }); it("should record completion tokens", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, undefined, 50); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.completion_tokens", 50); }); it("should record both token types", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, 100, 50); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.prompt_tokens", 100); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.completion_tokens", 50); }); it("should handle undefined prompt tokens", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, undefined, 50); expect(setAttributeSpy).toHaveBeenCalledTimes(1); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.completion_tokens", 50); }); it("should handle undefined completion tokens", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, 100, undefined); expect(setAttributeSpy).toHaveBeenCalledTimes(1); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.prompt_tokens", 100); }); it("should handle both undefined", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, undefined, undefined); expect(setAttributeSpy).not.toHaveBeenCalled(); }); it("should handle zero values", () => { const setAttributeSpy = vi.spyOn(span, "setAttribute"); recordLlmUsage(span, 0, 0); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.prompt_tokens", 0); expect(setAttributeSpy).toHaveBeenCalledWith("gen_ai.usage.completion_tokens", 0); }); }); describe("integration scenarios", () => { it("should support streaming operations with createLlmSpan", async () => { async function* streamChat(model: string): AsyncGenerator { const span = createLlmSpan("ollama", "chat.stream", model); try { yield "Hello"; yield " "; yield "world"; span.setStatus({ code: SpanStatusCode.OK }); } catch (error) { span.recordException(error as Error); span.setStatus({ code: SpanStatusCode.ERROR }); throw error; } finally { span.end(); } } const chunks: string[] = []; for await (const chunk of streamChat("llama2")) { chunks.push(chunk); } expect(chunks).toEqual(["Hello", " ", "world"]); }); it("should support manual token recording in streams", async () => { async function* streamWithTokens(): AsyncGenerator<{ text: string; tokens?: number }> { const span = createLlmSpan("ollama", "chat.stream", "llama2"); try { let totalTokens = 0; yield { text: "Hello", tokens: 5 }; totalTokens += 5; yield { text: " world", tokens: 7 }; totalTokens += 7; recordLlmUsage(span, undefined, totalTokens); span.setStatus({ code: SpanStatusCode.OK }); } finally { span.end(); } } const chunks: Array<{ text: string; tokens?: number }> = []; for await (const chunk of streamWithTokens()) { chunks.push(chunk); } expect(chunks).toHaveLength(2); expect(chunks[0].tokens).toBe(5); expect(chunks[1].tokens).toBe(7); }); }); });