diff --git a/apps/orchestrator/package.json b/apps/orchestrator/package.json index 027b78c..12287d8 100644 --- a/apps/orchestrator/package.json +++ b/apps/orchestrator/package.json @@ -12,6 +12,7 @@ "test": "vitest", "test:watch": "vitest watch", "test:e2e": "vitest run --config tests/integration/vitest.config.ts", + "test:perf": "vitest run --config tests/performance/vitest.config.ts", "typecheck": "tsc --noEmit", "lint": "eslint src/", "lint:fix": "eslint src/ --fix" diff --git a/apps/orchestrator/tests/performance/queue-throughput.perf-spec.ts b/apps/orchestrator/tests/performance/queue-throughput.perf-spec.ts new file mode 100644 index 0000000..facd2e0 --- /dev/null +++ b/apps/orchestrator/tests/performance/queue-throughput.perf-spec.ts @@ -0,0 +1,99 @@ +/** + * Performance Test: Queue Service Throughput + * + * Benchmarks the queue service's pure functions and validation logic + * under load to verify performance characteristics. + * + * Covers issue #229 (ORCH-128) + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { QueueService } from "../../src/queue/queue.service"; +import { ConfigService } from "@nestjs/config"; + +describe("Performance: Queue Service", () => { + let service: QueueService; + + const mockValkeyService = { + getConnection: vi.fn().mockReturnValue({ + host: "localhost", + port: 6379, + }), + updateTaskStatus: vi.fn().mockResolvedValue(undefined), + publishEvent: vi.fn().mockResolvedValue(undefined), + }; + + const mockConfigService = { + get: vi.fn((key: string, defaultValue?: unknown) => { + const config: Record = { + "orchestrator.queue.name": "perf-test-queue", + "orchestrator.queue.maxRetries": 3, + "orchestrator.queue.baseDelay": 1000, + "orchestrator.queue.maxDelay": 60000, + }; + return config[key] ?? defaultValue; + }), + }; + + beforeEach(() => { + vi.clearAllMocks(); + service = new QueueService( + mockValkeyService as never, + mockConfigService as unknown as ConfigService + ); + }); + + describe("Backoff calculation performance", () => { + it("should calculate 10,000 backoff delays in under 10ms", () => { + const start = performance.now(); + + for (let i = 0; i < 10000; i++) { + service.calculateBackoffDelay(i % 20, 1000, 60000); + } + + const duration = performance.now() - start; + expect(duration).toBeLessThan(10); + }); + + it("should produce consistent results under rapid invocation", () => { + const results: number[] = []; + + for (let attempt = 0; attempt <= 10; attempt++) { + const delay = service.calculateBackoffDelay(attempt, 1000, 60000); + results.push(delay); + } + + // Verify expected exponential pattern + expect(results[0]).toBe(1000); // 1000 * 2^0 + expect(results[1]).toBe(2000); // 1000 * 2^1 + expect(results[2]).toBe(4000); // 1000 * 2^2 + expect(results[3]).toBe(8000); // 1000 * 2^3 + + // After attempt 6 (64000), should be capped at 60000 + expect(results[6]).toBe(60000); + expect(results[10]).toBe(60000); + }); + }); + + describe("Validation performance", () => { + it("should validate 1000 task contexts rapidly", () => { + const contexts = Array.from({ length: 1000 }, (_, i) => ({ + repository: `https://git.example.com/repo-${String(i)}.git`, + branch: `feature/task-${String(i)}`, + workItems: [`US-${String(i).padStart(3, "0")}`], + skills: ["typescript", "nestjs"], + })); + + const start = performance.now(); + + for (const context of contexts) { + // Validate context fields (simulates what addTask validates) + expect(context.repository).toBeTruthy(); + expect(context.branch).toBeTruthy(); + expect(context.workItems.length).toBeGreaterThan(0); + } + + const duration = performance.now() - start; + expect(duration).toBeLessThan(100); + }); + }); +}); diff --git a/apps/orchestrator/tests/performance/secret-scanner-throughput.perf-spec.ts b/apps/orchestrator/tests/performance/secret-scanner-throughput.perf-spec.ts new file mode 100644 index 0000000..c4663cd --- /dev/null +++ b/apps/orchestrator/tests/performance/secret-scanner-throughput.perf-spec.ts @@ -0,0 +1,123 @@ +/** + * Performance Test: Secret Scanner Throughput + * + * Benchmarks the secret scanner's ability to scan content + * at scale without degrading performance. + * + * Covers issue #229 (ORCH-128) + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { SecretScannerService } from "../../src/git/secret-scanner.service"; +import { ConfigService } from "@nestjs/config"; + +describe("Performance: Secret Scanner", () => { + let scanner: SecretScannerService; + + const mockConfigService = { + get: vi.fn((_key: string, defaultValue?: unknown) => defaultValue), + }; + + beforeEach(() => { + scanner = new SecretScannerService(mockConfigService as unknown as ConfigService); + }); + + describe("Content scanning throughput", () => { + it("should scan 1000 lines of clean code in under 50ms", () => { + const lines = Array.from( + { length: 1000 }, + (_, i) => `const value${String(i)} = computeResult(${String(i)}, "param-${String(i)}");` + ); + const content = lines.join("\n"); + + const start = performance.now(); + const result = scanner.scanContent(content, "test-file.ts"); + const duration = performance.now() - start; + + expect(duration).toBeLessThan(50); + expect(result.matches).toHaveLength(0); + }); + + it("should scan 100 files worth of content in under 500ms", () => { + const fileContent = Array.from( + { length: 100 }, + (_, i) => `export function handler${String(i)}(): string { return "result-${String(i)}"; }` + ).join("\n"); + + const start = performance.now(); + + for (let i = 0; i < 100; i++) { + scanner.scanContent(fileContent, `file-${String(i)}.ts`); + } + + const duration = performance.now() - start; + expect(duration).toBeLessThan(500); + }); + + it("should detect secrets in large content without performance regression", () => { + // Mix clean code with embedded secrets + const lines: string[] = []; + for (let i = 0; i < 500; i++) { + lines.push(`const config${String(i)} = { host: "localhost", port: ${String(3000 + i)} };`); + } + // Insert a secret at line 250 + lines[250] = 'const apiKey = "AKIA1234567890ABCDEF"; // AWS access key'; + + const content = lines.join("\n"); + + const start = performance.now(); + const result = scanner.scanContent(content, "config.ts"); + const duration = performance.now() - start; + + expect(duration).toBeLessThan(100); + expect(result.matches.length).toBeGreaterThan(0); + }); + + it("should handle content with many false-positive patterns efficiently", () => { + // Content with many patterns that look like secrets but are placeholders + const lines = Array.from( + { length: 200 }, + (_, i) => `const example_key_${String(i)} = "test-xxxx-example-${String(i)}";` + ); + const content = lines.join("\n"); + + const start = performance.now(); + const result = scanner.scanContent(content, "examples.ts"); + const duration = performance.now() - start; + + expect(duration).toBeLessThan(100); + // Placeholders should be whitelisted + expect(result.matches).toHaveLength(0); + }); + }); + + describe("Pattern matching scalability", () => { + it("should maintain consistent scan time regardless of content position", () => { + const baseContent = Array.from( + { length: 1000 }, + (_, i) => `const x${String(i)} = ${String(i)};` + ); + + // Secret at start + const contentStart = ['const key = "AKIA1234567890ABCDEF";', ...baseContent].join("\n"); + + // Secret at end + const contentEnd = [...baseContent, 'const key = "AKIA1234567890ABCDEF";'].join("\n"); + + const startTime1 = performance.now(); + scanner.scanContent(contentStart, "start.ts"); + const duration1 = performance.now() - startTime1; + + const startTime2 = performance.now(); + scanner.scanContent(contentEnd, "end.ts"); + const duration2 = performance.now() - startTime2; + + // Both should complete quickly + expect(duration1).toBeLessThan(100); + expect(duration2).toBeLessThan(100); + + // And be within 5x of each other (no pathological behavior) + const ratio = Math.max(duration1, duration2) / Math.max(0.01, Math.min(duration1, duration2)); + expect(ratio).toBeLessThan(5); + }); + }); +}); diff --git a/apps/orchestrator/tests/performance/spawner-throughput.perf-spec.ts b/apps/orchestrator/tests/performance/spawner-throughput.perf-spec.ts new file mode 100644 index 0000000..0e30f0e --- /dev/null +++ b/apps/orchestrator/tests/performance/spawner-throughput.perf-spec.ts @@ -0,0 +1,199 @@ +/** + * Performance Test: Agent Spawner Throughput + * + * Benchmarks the spawner service under concurrent load to verify + * it meets performance requirements for agent orchestration. + * + * Covers issue #229 (ORCH-128) + */ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { AgentSpawnerService } from "../../src/spawner/agent-spawner.service"; +import { AgentsController } from "../../src/api/agents/agents.controller"; +import { QueueService } from "../../src/queue/queue.service"; +import { AgentLifecycleService } from "../../src/spawner/agent-lifecycle.service"; +import { KillswitchService } from "../../src/killswitch/killswitch.service"; +import { ConfigService } from "@nestjs/config"; + +describe("Performance: Agent Spawner Throughput", () => { + let controller: AgentsController; + let spawnerService: AgentSpawnerService; + + const mockConfigService = { + get: vi.fn((key: string, defaultValue?: unknown) => { + const config: Record = { + "orchestrator.claude.apiKey": "test-api-key", + }; + return config[key] ?? defaultValue; + }), + }; + + beforeEach(() => { + vi.clearAllMocks(); + + spawnerService = new AgentSpawnerService(mockConfigService as unknown as ConfigService); + + const queueService = { + addTask: vi.fn().mockResolvedValue(undefined), + } as unknown as QueueService; + + const lifecycleService = { + getAgentLifecycleState: vi.fn(), + } as unknown as AgentLifecycleService; + + const killswitchService = { + killAgent: vi.fn(), + killAllAgents: vi.fn(), + } as unknown as KillswitchService; + + controller = new AgentsController( + queueService, + spawnerService, + lifecycleService, + killswitchService + ); + }); + + describe("Spawn latency", () => { + it("should spawn a single agent in under 10ms", async () => { + const start = performance.now(); + + await controller.spawn({ + taskId: "perf-single-001", + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: "main", + workItems: ["US-001"], + }, + }); + + const duration = performance.now() - start; + expect(duration).toBeLessThan(10); + }); + + it("should spawn 100 agents sequentially in under 500ms", async () => { + const start = performance.now(); + + for (let i = 0; i < 100; i++) { + await controller.spawn({ + taskId: `perf-seq-${String(i)}`, + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: "main", + workItems: [`US-${String(i)}`], + }, + }); + } + + const duration = performance.now() - start; + expect(duration).toBeLessThan(500); + + const agents = spawnerService.listAgentSessions(); + expect(agents).toHaveLength(100); + }); + + it("should spawn 100 agents concurrently in under 200ms", async () => { + const start = performance.now(); + + const promises = Array.from({ length: 100 }, (_, i) => + controller.spawn({ + taskId: `perf-concurrent-${String(i)}`, + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: `feature/task-${String(i)}`, + workItems: [`US-${String(i).padStart(3, "0")}`], + }, + }) + ); + + const results = await Promise.all(promises); + const duration = performance.now() - start; + + expect(duration).toBeLessThan(200); + expect(results).toHaveLength(100); + + // Verify all IDs are unique + const ids = new Set(results.map((r) => r.agentId)); + expect(ids.size).toBe(100); + }); + }); + + describe("Session lookup performance", () => { + it("should look up agents by ID in under 1ms with 1000 sessions", async () => { + // Pre-populate 1000 sessions + const agentIds: string[] = []; + for (let i = 0; i < 1000; i++) { + const result = await controller.spawn({ + taskId: `perf-lookup-${String(i)}`, + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: "main", + workItems: [`US-${String(i)}`], + }, + }); + agentIds.push(result.agentId); + } + + // Measure lookup time for random agents + const lookupStart = performance.now(); + for (let i = 0; i < 100; i++) { + const randomIdx = Math.floor(Math.random() * agentIds.length); + const session = spawnerService.getAgentSession(agentIds[randomIdx]); + expect(session).toBeDefined(); + } + const lookupDuration = performance.now() - lookupStart; + + // 100 lookups should complete in under 10ms + expect(lookupDuration).toBeLessThan(10); + }); + + it("should list all sessions in under 5ms with 1000 sessions", async () => { + // Pre-populate 1000 sessions + for (let i = 0; i < 1000; i++) { + await controller.spawn({ + taskId: `perf-list-${String(i)}`, + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: "main", + workItems: [`US-${String(i)}`], + }, + }); + } + + const listStart = performance.now(); + const sessions = spawnerService.listAgentSessions(); + const listDuration = performance.now() - listStart; + + expect(sessions).toHaveLength(1000); + expect(listDuration).toBeLessThan(5); + }); + }); + + describe("Memory efficiency", () => { + it("should not have excessive memory growth after 1000 spawns", async () => { + const memBefore = process.memoryUsage().heapUsed; + + for (let i = 0; i < 1000; i++) { + await controller.spawn({ + taskId: `perf-mem-${String(i)}`, + agentType: "worker", + context: { + repository: "https://git.example.com/repo.git", + branch: "main", + workItems: [`US-${String(i)}`], + }, + }); + } + + const memAfter = process.memoryUsage().heapUsed; + const memGrowthMB = (memAfter - memBefore) / 1024 / 1024; + + // 1000 agent sessions should use less than 50MB + expect(memGrowthMB).toBeLessThan(50); + }); + }); +}); diff --git a/apps/orchestrator/tests/performance/vitest.config.ts b/apps/orchestrator/tests/performance/vitest.config.ts new file mode 100644 index 0000000..75fb57b --- /dev/null +++ b/apps/orchestrator/tests/performance/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["**/*.perf-spec.ts"], + testTimeout: 60000, + }, +});