feat(#131): add OpenTelemetry tracing infrastructure

Implement comprehensive distributed tracing for HTTP requests and LLM
operations using OpenTelemetry with GenAI semantic conventions.

Features:
- TelemetryService: SDK initialization with OTLP HTTP exporter
- TelemetryInterceptor: Automatic HTTP request spans
- @TraceLlmCall decorator: LLM operation tracing
- GenAI semantic conventions for model/token tracking
- Graceful degradation when tracing disabled

Instrumented:
- All HTTP requests (automatic spans)
- OllamaProvider chat/chatStream/embed operations
- Token counts, model names, durations

Environment:
- OTEL_ENABLED (default: true)
- OTEL_SERVICE_NAME (default: mosaic-api)
- OTEL_EXPORTER_OTLP_ENDPOINT (default: localhost:4318)

Tests: 23 passing with full coverage

Fixes #131

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

apps/api/src/telemetry/telemetry.interceptor.spec.ts

@@ -0,0 +1,181 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { TelemetryInterceptor } from "./telemetry.interceptor";
+import { TelemetryService } from "./telemetry.service";
+import type { ExecutionContext, CallHandler } from "@nestjs/common";
+import type { Span } from "@opentelemetry/api";
+import { of, throwError } from "rxjs";
+import { lastValueFrom } from "rxjs";
+
+describe("TelemetryInterceptor", () => {
+  let interceptor: TelemetryInterceptor;
+  let telemetryService: TelemetryService;
+  let mockSpan: Span;
+  let mockContext: ExecutionContext;
+  let mockHandler: CallHandler;
+
+  beforeEach(() => {
+    // Mock span
+    mockSpan = {
+      end: vi.fn(),
+      setAttribute: vi.fn(),
+      setAttributes: vi.fn(),
+      addEvent: vi.fn(),
+      setStatus: vi.fn(),
+      updateName: vi.fn(),
+      isRecording: vi.fn().mockReturnValue(true),
+      recordException: vi.fn(),
+      spanContext: vi.fn().mockReturnValue({
+        traceId: "test-trace-id",
+        spanId: "test-span-id",
+      }),
+    } as unknown as Span;
+
+    // Mock telemetry service
+    telemetryService = {
+      startSpan: vi.fn().mockReturnValue(mockSpan),
+      recordException: vi.fn(),
+      getTracer: vi.fn(),
+      onModuleInit: vi.fn(),
+      onModuleDestroy: vi.fn(),
+    } as unknown as TelemetryService;
+
+    // Mock execution context
+    mockContext = {
+      switchToHttp: vi.fn().mockReturnValue({
+        getRequest: vi.fn().mockReturnValue({
+          method: "GET",
+          url: "/api/test",
+          path: "/api/test",
+        }),
+        getResponse: vi.fn().mockReturnValue({
+          statusCode: 200,
+          setHeader: vi.fn(),
+        }),
+      }),
+      getClass: vi.fn().mockReturnValue({ name: "TestController" }),
+      getHandler: vi.fn().mockReturnValue({ name: "testHandler" }),
+    } as unknown as ExecutionContext;
+
+    interceptor = new TelemetryInterceptor(telemetryService);
+  });
+
+  describe("intercept", () => {
+    it("should create a span for HTTP request", async () => {
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(of({ data: "test" })),
+      } as unknown as CallHandler;
+
+      await lastValueFrom(interceptor.intercept(mockContext, mockHandler));
+
+      expect(telemetryService.startSpan).toHaveBeenCalledWith(
+        "GET /api/test",
+        expect.objectContaining({
+          attributes: expect.objectContaining({
+            "http.request.method": "GET",
+            "url.path": "/api/test",
+          }),
+        })
+      );
+    });
+
+    it("should set http.status_code attribute on success", async () => {
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(of({ data: "test" })),
+      } as unknown as CallHandler;
+
+      await lastValueFrom(interceptor.intercept(mockContext, mockHandler));
+
+      expect(mockSpan.setAttribute).toHaveBeenCalledWith("http.response.status_code", 200);
+      expect(mockSpan.end).toHaveBeenCalled();
+    });
+
+    it("should add trace context to response headers", async () => {
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(of({ data: "test" })),
+      } as unknown as CallHandler;
+
+      const mockResponse = mockContext.switchToHttp().getResponse();
+
+      await lastValueFrom(interceptor.intercept(mockContext, mockHandler));
+
+      expect(mockResponse.setHeader).toHaveBeenCalledWith("x-trace-id", "test-trace-id");
+    });
+
+    it("should record exception on error", async () => {
+      const error = new Error("Test error");
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(throwError(() => error)),
+      } as unknown as CallHandler;
+
+      await expect(lastValueFrom(interceptor.intercept(mockContext, mockHandler))).rejects.toThrow(
+        "Test error"
+      );
+
+      expect(telemetryService.recordException).toHaveBeenCalledWith(mockSpan, error);
+      expect(mockSpan.end).toHaveBeenCalled();
+    });
+
+    it("should end span even if error occurs", async () => {
+      const error = new Error("Test error");
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(throwError(() => error)),
+      } as unknown as CallHandler;
+
+      await expect(
+        lastValueFrom(interceptor.intercept(mockContext, mockHandler))
+      ).rejects.toThrow();
+
+      expect(mockSpan.end).toHaveBeenCalled();
+    });
+
+    it("should handle different HTTP methods", async () => {
+      const postContext = {
+        ...mockContext,
+        switchToHttp: vi.fn().mockReturnValue({
+          getRequest: vi.fn().mockReturnValue({
+            method: "POST",
+            url: "/api/test",
+            path: "/api/test",
+          }),
+          getResponse: vi.fn().mockReturnValue({
+            statusCode: 201,
+            setHeader: vi.fn(),
+          }),
+        }),
+      } as unknown as ExecutionContext;
+
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(of({ data: "created" })),
+      } as unknown as CallHandler;
+
+      await lastValueFrom(interceptor.intercept(postContext, mockHandler));
+
+      expect(telemetryService.startSpan).toHaveBeenCalledWith(
+        "POST /api/test",
+        expect.objectContaining({
+          attributes: expect.objectContaining({
+            "http.request.method": "POST",
+          }),
+        })
+      );
+    });
+
+    it("should set controller and handler attributes", async () => {
+      mockHandler = {
+        handle: vi.fn().mockReturnValue(of({ data: "test" })),
+      } as unknown as CallHandler;
+
+      await lastValueFrom(interceptor.intercept(mockContext, mockHandler));
+
+      expect(telemetryService.startSpan).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({
+          attributes: expect.objectContaining({
+            "code.function": "testHandler",
+            "code.namespace": "TestController",
+          }),
+        })
+      );
+    });
+  });
+});