import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { TelemetryClient } from "../src/client.js"; import { TelemetryConfig } from "../src/config.js"; import { TaskCompletionEvent, TaskType, Complexity, Harness, Provider, Outcome, } from "../src/types/events.js"; import { PredictionQuery, PredictionResponse, } from "../src/types/predictions.js"; function makeConfig(overrides: Partial = {}): TelemetryConfig { return { serverUrl: "https://tel.example.com", apiKey: "a".repeat(64), instanceId: "test-instance", submitIntervalMs: 60_000, maxQueueSize: 100, batchSize: 10, requestTimeoutMs: 5000, dryRun: true, // Use dryRun by default in tests ...overrides, }; } function makeEvent(id = "evt-1"): TaskCompletionEvent { return { instance_id: "test-instance", event_id: id, schema_version: "1.0", timestamp: new Date().toISOString(), task_duration_ms: 5000, task_type: TaskType.IMPLEMENTATION, complexity: Complexity.MEDIUM, harness: Harness.CLAUDE_CODE, model: "claude-3-opus", provider: Provider.ANTHROPIC, estimated_input_tokens: 1000, estimated_output_tokens: 500, actual_input_tokens: 1100, actual_output_tokens: 550, estimated_cost_usd_micros: 50000, actual_cost_usd_micros: 55000, quality_gate_passed: true, quality_gates_run: [], quality_gates_failed: [], context_compactions: 0, context_rotations: 0, context_utilization_final: 0.5, outcome: Outcome.SUCCESS, retry_count: 0, }; } function makeQuery(): PredictionQuery { return { task_type: TaskType.IMPLEMENTATION, model: "claude-3-opus", provider: Provider.ANTHROPIC, complexity: Complexity.MEDIUM, }; } function makePredictionResponse(): PredictionResponse { return { prediction: { input_tokens: { p10: 500, p25: 750, median: 1000, p75: 1500, p90: 2000 }, output_tokens: { p10: 200, p25: 350, median: 500, p75: 750, p90: 1000 }, cost_usd_micros: { median: 50000 }, duration_ms: { median: 30000 }, correction_factors: { input: 1.1, output: 1.05 }, quality: { gate_pass_rate: 0.85, success_rate: 0.9 }, }, metadata: { sample_size: 100, fallback_level: 0, confidence: "high", last_updated: new Date().toISOString(), cache_hit: false, }, }; } describe("TelemetryClient", () => { let fetchSpy: ReturnType; beforeEach(() => { vi.useFakeTimers(); fetchSpy = vi.fn(); vi.stubGlobal("fetch", fetchSpy); }); afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); }); describe("start/stop lifecycle", () => { it("should start and stop cleanly", async () => { const client = new TelemetryClient(makeConfig()); expect(client.isRunning).toBe(false); client.start(); expect(client.isRunning).toBe(true); await client.stop(); expect(client.isRunning).toBe(false); }); it("should be idempotent on start", () => { const client = new TelemetryClient(makeConfig()); client.start(); client.start(); // Should not throw or create double intervals expect(client.isRunning).toBe(true); }); it("should be idempotent on stop", async () => { const client = new TelemetryClient(makeConfig()); await client.stop(); await client.stop(); // Should not throw expect(client.isRunning).toBe(false); }); it("should flush events on stop", async () => { const client = new TelemetryClient(makeConfig()); client.start(); client.track(makeEvent("e1")); client.track(makeEvent("e2")); expect(client.queueSize).toBe(2); await client.stop(); // In dryRun mode, flush succeeds and queue should be empty expect(client.queueSize).toBe(0); }); }); describe("track()", () => { it("should queue events", () => { const client = new TelemetryClient(makeConfig()); client.track(makeEvent("e1")); client.track(makeEvent("e2")); expect(client.queueSize).toBe(2); }); it("should silently drop events when disabled", () => { const client = new TelemetryClient(makeConfig({ enabled: false })); client.track(makeEvent()); expect(client.queueSize).toBe(0); }); it("should never throw even on internal error", () => { const errorFn = vi.fn(); const client = new TelemetryClient( makeConfig({ onError: errorFn, maxQueueSize: 0 }), ); // This should not throw. maxQueueSize of 0 could cause issues // but track() is designed to catch everything. expect(() => client.track(makeEvent())).not.toThrow(); }); }); describe("predictions", () => { it("should return null for uncached prediction", () => { const client = new TelemetryClient(makeConfig()); const result = client.getPrediction(makeQuery()); expect(result).toBeNull(); }); it("should return cached prediction after refresh", async () => { const predictionResponse = makePredictionResponse(); fetchSpy.mockResolvedValueOnce({ ok: true, status: 200, json: () => Promise.resolve({ results: [predictionResponse], }), }); const client = new TelemetryClient(makeConfig({ dryRun: false })); const query = makeQuery(); await client.refreshPredictions([query]); const result = client.getPrediction(query); expect(result).toEqual(predictionResponse); }); it("should handle refresh error gracefully", async () => { fetchSpy.mockRejectedValueOnce(new Error("Network error")); const errorFn = vi.fn(); const client = new TelemetryClient( makeConfig({ dryRun: false, onError: errorFn }), ); // Should not throw await client.refreshPredictions([makeQuery()]); expect(errorFn).toHaveBeenCalledWith(expect.any(Error)); }); it("should handle non-ok HTTP response on refresh", async () => { fetchSpy.mockResolvedValueOnce({ ok: false, status: 500, statusText: "Internal Server Error", }); const errorFn = vi.fn(); const client = new TelemetryClient( makeConfig({ dryRun: false, onError: errorFn }), ); await client.refreshPredictions([makeQuery()]); expect(errorFn).toHaveBeenCalledWith(expect.any(Error)); }); }); describe("background flush", () => { it("should trigger flush on interval", async () => { const client = new TelemetryClient( makeConfig({ submitIntervalMs: 10_000 }), ); client.start(); client.track(makeEvent("e1")); expect(client.queueSize).toBe(1); // Advance past submit interval await vi.advanceTimersByTimeAsync(11_000); // In dryRun mode, events should be flushed expect(client.queueSize).toBe(0); await client.stop(); }); }); describe("flush error handling", () => { it("should re-enqueue events on submit failure", async () => { // Use non-dryRun mode to actually hit the submitter fetchSpy.mockResolvedValueOnce({ ok: false, status: 500, statusText: "Internal Server Error", }); const errorFn = vi.fn(); const client = new TelemetryClient( makeConfig({ dryRun: false, maxRetries: 0, onError: errorFn }), ); client.track(makeEvent("e1")); expect(client.queueSize).toBe(1); // Start and trigger flush client.start(); await vi.advanceTimersByTimeAsync(70_000); // Events should be re-enqueued after failure expect(client.queueSize).toBeGreaterThan(0); await client.stop(); }); it("should handle onError callback that throws", async () => { const throwingErrorFn = () => { throw new Error("Error handler broke"); }; const client = new TelemetryClient( makeConfig({ onError: throwingErrorFn, enabled: false }), ); // This should not throw even though onError throws // Force an error path by calling track when disabled (no error), // but we can test via refreshPredictions fetchSpy.mockRejectedValueOnce(new Error("fail")); await expect( client.refreshPredictions([makeQuery()]), ).resolves.not.toThrow(); }); }); describe("event builder", () => { it("should expose an event builder", () => { const client = new TelemetryClient(makeConfig()); expect(client.eventBuilder).toBeDefined(); const event = client.eventBuilder.build({ task_duration_ms: 1000, task_type: TaskType.TESTING, complexity: Complexity.LOW, harness: Harness.AIDER, model: "gpt-4", provider: Provider.OPENAI, estimated_input_tokens: 100, estimated_output_tokens: 50, actual_input_tokens: 100, actual_output_tokens: 50, estimated_cost_usd_micros: 1000, actual_cost_usd_micros: 1000, quality_gate_passed: true, quality_gates_run: [], quality_gates_failed: [], context_compactions: 0, context_rotations: 0, context_utilization_final: 0.3, outcome: Outcome.SUCCESS, retry_count: 0, }); expect(event.instance_id).toBe("test-instance"); expect(event.schema_version).toBe("1.0"); }); }); });