telemetry-client-js/tests/client.test.ts

import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { TelemetryClient } from "../src/client.js";
import { TelemetryConfig } from "../src/config.js";
import {
  TaskCompletionEvent,
  TaskType,
  Complexity,
  Harness,
  Provider,
  Outcome,
} from "../src/types/events.js";
import {
  PredictionQuery,
  PredictionResponse,
} from "../src/types/predictions.js";

function makeConfig(overrides: Partial<TelemetryConfig> = {}): TelemetryConfig {
  return {
    serverUrl: "https://tel.example.com",
    apiKey: "a".repeat(64),
    instanceId: "test-instance",
    submitIntervalMs: 60_000,
    maxQueueSize: 100,
    batchSize: 10,
    requestTimeoutMs: 5000,
    dryRun: true, // Use dryRun by default in tests
    ...overrides,
  };
}

function makeEvent(id = "evt-1"): TaskCompletionEvent {
  return {
    instance_id: "test-instance",
    event_id: id,
    schema_version: "1.0",
    timestamp: new Date().toISOString(),
    task_duration_ms: 5000,
    task_type: TaskType.IMPLEMENTATION,
    complexity: Complexity.MEDIUM,
    harness: Harness.CLAUDE_CODE,
    model: "claude-3-opus",
    provider: Provider.ANTHROPIC,
    estimated_input_tokens: 1000,
    estimated_output_tokens: 500,
    actual_input_tokens: 1100,
    actual_output_tokens: 550,
    estimated_cost_usd_micros: 50000,
    actual_cost_usd_micros: 55000,
    quality_gate_passed: true,
    quality_gates_run: [],
    quality_gates_failed: [],
    context_compactions: 0,
    context_rotations: 0,
    context_utilization_final: 0.5,
    outcome: Outcome.SUCCESS,
    retry_count: 0,
  };
}

function makeQuery(): PredictionQuery {
  return {
    task_type: TaskType.IMPLEMENTATION,
    model: "claude-3-opus",
    provider: Provider.ANTHROPIC,
    complexity: Complexity.MEDIUM,
  };
}

function makePredictionResponse(): PredictionResponse {
  return {
    prediction: {
      input_tokens: { p10: 500, p25: 750, median: 1000, p75: 1500, p90: 2000 },
      output_tokens: { p10: 200, p25: 350, median: 500, p75: 750, p90: 1000 },
      cost_usd_micros: { median: 50000 },
      duration_ms: { median: 30000 },
      correction_factors: { input: 1.1, output: 1.05 },
      quality: { gate_pass_rate: 0.85, success_rate: 0.9 },
    },
    metadata: {
      sample_size: 100,
      fallback_level: 0,
      confidence: "high",
      last_updated: new Date().toISOString(),
      cache_hit: false,
    },
  };
}

describe("TelemetryClient", () => {
  let fetchSpy: ReturnType<typeof vi.fn>;

  beforeEach(() => {
    vi.useFakeTimers();
    fetchSpy = vi.fn();
    vi.stubGlobal("fetch", fetchSpy);
  });

  afterEach(() => {
    vi.useRealTimers();
    vi.unstubAllGlobals();
  });

  describe("start/stop lifecycle", () => {
    it("should start and stop cleanly", async () => {
      const client = new TelemetryClient(makeConfig());

      expect(client.isRunning).toBe(false);
      client.start();
      expect(client.isRunning).toBe(true);

      await client.stop();
      expect(client.isRunning).toBe(false);
    });

    it("should be idempotent on start", () => {
      const client = new TelemetryClient(makeConfig());
      client.start();
      client.start(); // Should not throw or create double intervals
      expect(client.isRunning).toBe(true);
    });

    it("should be idempotent on stop", async () => {
      const client = new TelemetryClient(makeConfig());
      await client.stop();
      await client.stop(); // Should not throw
      expect(client.isRunning).toBe(false);
    });

    it("should flush events on stop", async () => {
      const client = new TelemetryClient(makeConfig());
      client.start();

      client.track(makeEvent("e1"));
      client.track(makeEvent("e2"));
      expect(client.queueSize).toBe(2);

      await client.stop();
      // In dryRun mode, flush succeeds and queue should be empty
      expect(client.queueSize).toBe(0);
    });
  });

  describe("track()", () => {
    it("should queue events", () => {
      const client = new TelemetryClient(makeConfig());
      client.track(makeEvent("e1"));
      client.track(makeEvent("e2"));
      expect(client.queueSize).toBe(2);
    });

    it("should silently drop events when disabled", () => {
      const client = new TelemetryClient(makeConfig({ enabled: false }));
      client.track(makeEvent());
      expect(client.queueSize).toBe(0);
    });

    it("should never throw even on internal error", () => {
      const errorFn = vi.fn();
      const client = new TelemetryClient(
        makeConfig({ onError: errorFn, maxQueueSize: 0 }),
      );

      // This should not throw. maxQueueSize of 0 could cause issues
      // but track() is designed to catch everything.
      expect(() => client.track(makeEvent())).not.toThrow();
    });
  });

  describe("predictions", () => {
    it("should return null for uncached prediction", () => {
      const client = new TelemetryClient(makeConfig());
      const result = client.getPrediction(makeQuery());
      expect(result).toBeNull();
    });

    it("should return cached prediction after refresh", async () => {
      const predictionResponse = makePredictionResponse();
      fetchSpy.mockResolvedValueOnce({
        ok: true,
        status: 200,
        json: () =>
          Promise.resolve({
            results: [predictionResponse],
          }),
      });

      const client = new TelemetryClient(makeConfig({ dryRun: false }));
      const query = makeQuery();

      await client.refreshPredictions([query]);

      const result = client.getPrediction(query);
      expect(result).toEqual(predictionResponse);
    });

    it("should handle refresh error gracefully", async () => {
      fetchSpy.mockRejectedValueOnce(new Error("Network error"));

      const errorFn = vi.fn();
      const client = new TelemetryClient(
        makeConfig({ dryRun: false, onError: errorFn }),
      );

      // Should not throw
      await client.refreshPredictions([makeQuery()]);
      expect(errorFn).toHaveBeenCalledWith(expect.any(Error));
    });

    it("should handle non-ok HTTP response on refresh", async () => {
      fetchSpy.mockResolvedValueOnce({
        ok: false,
        status: 500,
        statusText: "Internal Server Error",
      });

      const errorFn = vi.fn();
      const client = new TelemetryClient(
        makeConfig({ dryRun: false, onError: errorFn }),
      );

      await client.refreshPredictions([makeQuery()]);
      expect(errorFn).toHaveBeenCalledWith(expect.any(Error));
    });
  });

  describe("background flush", () => {
    it("should trigger flush on interval", async () => {
      const client = new TelemetryClient(
        makeConfig({ submitIntervalMs: 10_000 }),
      );
      client.start();

      client.track(makeEvent("e1"));
      expect(client.queueSize).toBe(1);

      // Advance past submit interval
      await vi.advanceTimersByTimeAsync(11_000);

      // In dryRun mode, events should be flushed
      expect(client.queueSize).toBe(0);

      await client.stop();
    });
  });

  describe("flush error handling", () => {
    it("should re-enqueue events on submit failure", async () => {
      // Use non-dryRun mode to actually hit the submitter
      fetchSpy.mockResolvedValueOnce({
        ok: false,
        status: 500,
        statusText: "Internal Server Error",
      });

      const errorFn = vi.fn();
      const client = new TelemetryClient(
        makeConfig({ dryRun: false, maxRetries: 0, onError: errorFn }),
      );

      client.track(makeEvent("e1"));
      expect(client.queueSize).toBe(1);

      // Start and trigger flush
      client.start();
      await vi.advanceTimersByTimeAsync(70_000);

      // Events should be re-enqueued after failure
      expect(client.queueSize).toBeGreaterThan(0);

      await client.stop();
    });

    it("should handle onError callback that throws", async () => {
      const throwingErrorFn = () => {
        throw new Error("Error handler broke");
      };
      const client = new TelemetryClient(
        makeConfig({ onError: throwingErrorFn, enabled: false }),
      );

      // This should not throw even though onError throws
      // Force an error path by calling track when disabled (no error),
      // but we can test via refreshPredictions
      fetchSpy.mockRejectedValueOnce(new Error("fail"));
      await expect(
        client.refreshPredictions([makeQuery()]),
      ).resolves.not.toThrow();
    });
  });

  describe("event builder", () => {
    it("should expose an event builder", () => {
      const client = new TelemetryClient(makeConfig());
      expect(client.eventBuilder).toBeDefined();

      const event = client.eventBuilder.build({
        task_duration_ms: 1000,
        task_type: TaskType.TESTING,
        complexity: Complexity.LOW,
        harness: Harness.AIDER,
        model: "gpt-4",
        provider: Provider.OPENAI,
        estimated_input_tokens: 100,
        estimated_output_tokens: 50,
        actual_input_tokens: 100,
        actual_output_tokens: 50,
        estimated_cost_usd_micros: 1000,
        actual_cost_usd_micros: 1000,
        quality_gate_passed: true,
        quality_gates_run: [],
        quality_gates_failed: [],
        context_compactions: 0,
        context_rotations: 0,
        context_utilization_final: 0.3,
        outcome: Outcome.SUCCESS,
        retry_count: 0,
      });

      expect(event.instance_id).toBe("test-instance");
      expect(event.schema_version).toBe("1.0");
    });
  });
});