Merge pull request 'test(#226,#227,#228): Add E2E integration tests for agent orchestration' (#332) from feature/226-e2e-agent-lifecycle into develop

Reviewed-on: #332

apps/orchestrator/tests/integration/agent-lifecycle.e2e-spec.ts

@@ -0,0 +1,242 @@
+/**
+ * E2E Test: Full Agent Lifecycle
+ *
+ * Tests the complete lifecycle of an agent from spawn to completion/failure.
+ * Uses mocked services to simulate the full flow without external dependencies.
+ *
+ * Lifecycle: spawn → running → completed/failed/killed
+ *
+ * Covers issue #226 (ORCH-125)
+ */
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { ConfigService } from "@nestjs/config";
+import { AgentSpawnerService } from "../../src/spawner/agent-spawner.service";
+import { AgentLifecycleService } from "../../src/spawner/agent-lifecycle.service";
+import { QueueService } from "../../src/queue/queue.service";
+import { KillswitchService } from "../../src/killswitch/killswitch.service";
+import { AgentsController } from "../../src/api/agents/agents.controller";
+import type { AgentState } from "../../src/valkey/types";
+
+describe("E2E: Full Agent Lifecycle", () => {
+  let controller: AgentsController;
+  let spawnerService: AgentSpawnerService;
+  let lifecycleService: AgentLifecycleService;
+  let queueService: QueueService;
+
+  const mockConfigService = {
+    get: vi.fn((key: string, defaultValue?: unknown) => {
+      const config: Record<string, unknown> = {
+        "orchestrator.claude.apiKey": "test-api-key",
+        "orchestrator.queue.name": "test-queue",
+        "orchestrator.queue.maxRetries": 3,
+        "orchestrator.queue.baseDelay": 100,
+        "orchestrator.queue.maxDelay": 1000,
+        "orchestrator.valkey.host": "localhost",
+        "orchestrator.valkey.port": 6379,
+      };
+      return config[key] ?? defaultValue;
+    }),
+  };
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+
+    // Create real spawner service with mock config
+    spawnerService = new AgentSpawnerService(mockConfigService as unknown as ConfigService);
+
+    // Create mock lifecycle service
+    lifecycleService = {
+      transitionToRunning: vi.fn(),
+      transitionToCompleted: vi.fn(),
+      transitionToFailed: vi.fn(),
+      getAgentLifecycleState: vi.fn(),
+    } as unknown as AgentLifecycleService;
+
+    // Create mock queue service
+    queueService = {
+      addTask: vi.fn().mockResolvedValue(undefined),
+      getStats: vi.fn(),
+    } as unknown as QueueService;
+
+    const killswitchService = {
+      killAgent: vi.fn(),
+      killAllAgents: vi.fn(),
+    } as unknown as KillswitchService;
+
+    controller = new AgentsController(
+      queueService,
+      spawnerService,
+      lifecycleService,
+      killswitchService
+    );
+  });
+
+  describe("Happy path: spawn → queue → track", () => {
+    it("should spawn an agent, register it, and queue the task", async () => {
+      // Step 1: Spawn agent
+      const spawnResult = await controller.spawn({
+        taskId: "e2e-task-001",
+        agentType: "worker",
+        context: {
+          repository: "https://git.example.com/repo.git",
+          branch: "main",
+          workItems: ["US-001"],
+          skills: ["typescript"],
+        },
+      });
+
+      expect(spawnResult.agentId).toBeDefined();
+      expect(spawnResult.status).toBe("spawning");
+
+      // Step 2: Verify agent appears in list
+      const agents = spawnerService.listAgentSessions();
+      expect(agents).toHaveLength(1);
+      expect(agents[0].state).toBe("spawning");
+      expect(agents[0].taskId).toBe("e2e-task-001");
+
+      // Step 3: Verify agent status
+      const session = spawnerService.getAgentSession(spawnResult.agentId);
+      expect(session).toBeDefined();
+      expect(session?.state).toBe("spawning");
+      expect(session?.agentType).toBe("worker");
+
+      // Step 4: Verify task was queued
+      expect(queueService.addTask).toHaveBeenCalledWith(
+        "e2e-task-001",
+        expect.objectContaining({
+          repository: "https://git.example.com/repo.git",
+          branch: "main",
+        }),
+        { priority: 5 }
+      );
+    });
+
+    it("should track multiple agents spawned sequentially", async () => {
+      // Spawn 3 agents
+      const agents = [];
+      for (let i = 0; i < 3; i++) {
+        const result = await controller.spawn({
+          taskId: `e2e-task-${String(i).padStart(3, "0")}`,
+          agentType: "worker",
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: "main",
+            workItems: [`US-${String(i).padStart(3, "0")}`],
+          },
+        });
+        agents.push(result);
+      }
+
+      // Verify all 3 agents are listed
+      const listedAgents = spawnerService.listAgentSessions();
+      expect(listedAgents).toHaveLength(3);
+
+      // Verify each agent has unique ID
+      const agentIds = listedAgents.map((a) => a.agentId);
+      const uniqueIds = new Set(agentIds);
+      expect(uniqueIds.size).toBe(3);
+    });
+  });
+
+  describe("Failure path: spawn → running → failed", () => {
+    it("should handle agent spawn with invalid parameters", async () => {
+      await expect(
+        controller.spawn({
+          taskId: "",
+          agentType: "worker",
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: "main",
+            workItems: ["US-001"],
+          },
+        })
+      ).rejects.toThrow("taskId is required");
+    });
+
+    it("should reject invalid agent types", async () => {
+      await expect(
+        controller.spawn({
+          taskId: "e2e-task-001",
+          agentType: "invalid" as "worker",
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: "main",
+            workItems: ["US-001"],
+          },
+        })
+      ).rejects.toThrow("agentType must be one of");
+    });
+  });
+
+  describe("Multi-type agents", () => {
+    it("should support worker, reviewer, and tester agent types", async () => {
+      const types = ["worker", "reviewer", "tester"] as const;
+
+      for (const agentType of types) {
+        const result = await controller.spawn({
+          taskId: `e2e-task-${agentType}`,
+          agentType,
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: "main",
+            workItems: ["US-001"],
+          },
+        });
+
+        expect(result.agentId).toBeDefined();
+        expect(result.status).toBe("spawning");
+      }
+
+      const agents = spawnerService.listAgentSessions();
+      expect(agents).toHaveLength(3);
+
+      const agentTypes = agents.map((a) => a.agentType);
+      expect(agentTypes).toContain("worker");
+      expect(agentTypes).toContain("reviewer");
+      expect(agentTypes).toContain("tester");
+    });
+  });
+
+  describe("Agent status tracking", () => {
+    it("should track spawn timestamp", async () => {
+      const before = new Date();
+
+      const result = await controller.spawn({
+        taskId: "e2e-task-time",
+        agentType: "worker",
+        context: {
+          repository: "https://git.example.com/repo.git",
+          branch: "main",
+          workItems: ["US-001"],
+        },
+      });
+
+      const after = new Date();
+      const agents = spawnerService.listAgentSessions();
+      const agent = agents.find((a) => a.agentId === result.agentId);
+      expect(agent).toBeDefined();
+
+      const spawnedAt = new Date(agent!.spawnedAt);
+      expect(spawnedAt.getTime()).toBeGreaterThanOrEqual(before.getTime());
+      expect(spawnedAt.getTime()).toBeLessThanOrEqual(after.getTime());
+    });
+
+    it("should return correct status for each agent", async () => {
+      // Mock lifecycle to return specific states
+      const mockState: AgentState = {
+        agentId: "mock-agent-1",
+        taskId: "e2e-task-001",
+        status: "running",
+        startedAt: new Date().toISOString(),
+      };
+
+      (lifecycleService.getAgentLifecycleState as ReturnType<typeof vi.fn>).mockResolvedValue(
+        mockState
+      );
+
+      const status = await controller.getAgentStatus("mock-agent-1");
+      expect(status.status).toBe("running");
+      expect(status.taskId).toBe("e2e-task-001");
+    });
+  });
+});

apps/orchestrator/tests/integration/concurrent-agents.e2e-spec.ts

@@ -0,0 +1,218 @@
+/**
+ * E2E Test: Concurrent Agents
+ *
+ * Tests multiple agents running concurrently with proper isolation.
+ * Verifies agent-level isolation, queue management, and concurrent operations.
+ *
+ * Covers issue #228 (ORCH-127)
+ */
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { AgentSpawnerService } from "../../src/spawner/agent-spawner.service";
+import { AgentsController } from "../../src/api/agents/agents.controller";
+import { QueueService } from "../../src/queue/queue.service";
+import { AgentLifecycleService } from "../../src/spawner/agent-lifecycle.service";
+import { KillswitchService } from "../../src/killswitch/killswitch.service";
+import { ConfigService } from "@nestjs/config";
+
+describe("E2E: Concurrent Agents", () => {
+  let controller: AgentsController;
+  let spawnerService: AgentSpawnerService;
+
+  const mockConfigService = {
+    get: vi.fn((key: string, defaultValue?: unknown) => {
+      const config: Record<string, unknown> = {
+        "orchestrator.claude.apiKey": "test-api-key",
+      };
+      return config[key] ?? defaultValue;
+    }),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    spawnerService = new AgentSpawnerService(mockConfigService as unknown as ConfigService);
+
+    const queueService = {
+      addTask: vi.fn().mockResolvedValue(undefined),
+    } as unknown as QueueService;
+
+    const lifecycleService = {
+      getAgentLifecycleState: vi.fn(),
+    } as unknown as AgentLifecycleService;
+
+    const killswitchService = {
+      killAgent: vi.fn(),
+      killAllAgents: vi.fn(),
+    } as unknown as KillswitchService;
+
+    controller = new AgentsController(
+      queueService,
+      spawnerService,
+      lifecycleService,
+      killswitchService
+    );
+  });
+
+  describe("Concurrent spawning", () => {
+    it("should spawn multiple agents simultaneously without conflicts", async () => {
+      // Spawn 5 agents in parallel
+      const spawnPromises = Array.from({ length: 5 }, (_, i) =>
+        controller.spawn({
+          taskId: `concurrent-task-${String(i)}`,
+          agentType: "worker",
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: `feature/task-${String(i)}`,
+            workItems: [`US-${String(i).padStart(3, "0")}`],
+          },
+        })
+      );
+
+      const results = await Promise.all(spawnPromises);
+
+      // All should succeed
+      expect(results).toHaveLength(5);
+      results.forEach((result) => {
+        expect(result.agentId).toBeDefined();
+        expect(result.status).toBe("spawning");
+      });
+
+      // All IDs should be unique
+      const ids = new Set(results.map((r) => r.agentId));
+      expect(ids.size).toBe(5);
+
+      // All should appear in the list
+      const agents = spawnerService.listAgentSessions();
+      expect(agents).toHaveLength(5);
+    });
+
+    it("should assign unique IDs to every agent even under concurrent load", async () => {
+      const allIds = new Set<string>();
+      const batchSize = 10;
+
+      // Spawn agents in batches
+      for (let batch = 0; batch < 3; batch++) {
+        const promises = Array.from({ length: batchSize }, (_, i) =>
+          controller.spawn({
+            taskId: `batch-${String(batch)}-task-${String(i)}`,
+            agentType: "worker",
+            context: {
+              repository: "https://git.example.com/repo.git",
+              branch: "main",
+              workItems: [`US-${String(batch * batchSize + i)}`],
+            },
+          })
+        );
+
+        const results = await Promise.all(promises);
+        results.forEach((r) => allIds.add(r.agentId));
+      }
+
+      // All 30 IDs should be unique
+      expect(allIds.size).toBe(30);
+
+      // All 30 should be listed
+      const agents = spawnerService.listAgentSessions();
+      expect(agents).toHaveLength(30);
+    });
+  });
+
+  describe("Mixed agent types concurrently", () => {
+    it("should handle mixed worker/reviewer/tester agents concurrently", async () => {
+      const types = ["worker", "reviewer", "tester"] as const;
+
+      const promises = types.flatMap((agentType, typeIndex) =>
+        Array.from({ length: 3 }, (_, i) =>
+          controller.spawn({
+            taskId: `mixed-${agentType}-${String(i)}`,
+            agentType,
+            context: {
+              repository: "https://git.example.com/repo.git",
+              branch: `branch-${String(typeIndex * 3 + i)}`,
+              workItems: [`US-${String(typeIndex * 3 + i)}`],
+            },
+          })
+        )
+      );
+
+      const results = await Promise.all(promises);
+      expect(results).toHaveLength(9);
+
+      const agents = spawnerService.listAgentSessions();
+      expect(agents).toHaveLength(9);
+
+      // Verify type distribution
+      const typeCounts = agents.reduce(
+        (acc, a) => {
+          acc[a.agentType] = (acc[a.agentType] ?? 0) + 1;
+          return acc;
+        },
+        {} as Record<string, number>
+      );
+
+      expect(typeCounts["worker"]).toBe(3);
+      expect(typeCounts["reviewer"]).toBe(3);
+      expect(typeCounts["tester"]).toBe(3);
+    });
+  });
+
+  describe("Agent isolation", () => {
+    it("should isolate agent contexts from each other", async () => {
+      const agent1 = await controller.spawn({
+        taskId: "isolated-task-1",
+        agentType: "worker",
+        context: {
+          repository: "https://git.example.com/repo-a.git",
+          branch: "main",
+          workItems: ["US-001"],
+          skills: ["typescript"],
+        },
+      });
+
+      const agent2 = await controller.spawn({
+        taskId: "isolated-task-2",
+        agentType: "reviewer",
+        context: {
+          repository: "https://git.example.com/repo-b.git",
+          branch: "develop",
+          workItems: ["US-002"],
+          skills: ["python"],
+        },
+      });
+
+      // Verify sessions are independent
+      const session1 = spawnerService.getAgentSession(agent1.agentId);
+      const session2 = spawnerService.getAgentSession(agent2.agentId);
+
+      expect(session1?.context.repository).toBe("https://git.example.com/repo-a.git");
+      expect(session2?.context.repository).toBe("https://git.example.com/repo-b.git");
+      expect(session1?.context.branch).toBe("main");
+      expect(session2?.context.branch).toBe("develop");
+    });
+
+    it("should not leak state between concurrent agent operations", async () => {
+      // Spawn agents with different task contexts
+      const spawnPromises = Array.from({ length: 5 }, (_, i) =>
+        controller.spawn({
+          taskId: `leak-test-${String(i)}`,
+          agentType: "worker",
+          context: {
+            repository: `https://git.example.com/repo-${String(i)}.git`,
+            branch: `branch-${String(i)}`,
+            workItems: [`US-${String(i).padStart(3, "0")}`],
+          },
+        })
+      );
+
+      const results = await Promise.all(spawnPromises);
+
+      // Verify each agent has its own isolated context
+      results.forEach((result, i) => {
+        const session = spawnerService.getAgentSession(result.agentId);
+        expect(session?.taskId).toBe(`leak-test-${String(i)}`);
+        expect(session?.context.repository).toBe(`https://git.example.com/repo-${String(i)}.git`);
+        expect(session?.context.branch).toBe(`branch-${String(i)}`);
+      });
+    });
+  });
+});

apps/orchestrator/tests/integration/killswitch.e2e-spec.ts

@@ -0,0 +1,158 @@
+/**
+ * E2E Test: Killswitch
+ *
+ * Tests the emergency stop mechanism for terminating agents.
+ * Verifies single agent kill, kill-all, and cleanup operations.
+ *
+ * Covers issue #227 (ORCH-126)
+ */
+import { describe, it, expect, beforeEach, vi } from "vitest";
+import { KillswitchService } from "../../src/killswitch/killswitch.service";
+import { AgentSpawnerService } from "../../src/spawner/agent-spawner.service";
+import { AgentsController } from "../../src/api/agents/agents.controller";
+import { QueueService } from "../../src/queue/queue.service";
+import { AgentLifecycleService } from "../../src/spawner/agent-lifecycle.service";
+import { ConfigService } from "@nestjs/config";
+
+describe("E2E: Killswitch", () => {
+  let controller: AgentsController;
+  let spawnerService: AgentSpawnerService;
+  let killswitchService: KillswitchService;
+
+  const mockConfigService = {
+    get: vi.fn((key: string, defaultValue?: unknown) => {
+      const config: Record<string, unknown> = {
+        "orchestrator.claude.apiKey": "test-api-key",
+      };
+      return config[key] ?? defaultValue;
+    }),
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    spawnerService = new AgentSpawnerService(mockConfigService as unknown as ConfigService);
+
+    killswitchService = {
+      killAgent: vi.fn().mockResolvedValue(undefined),
+      killAllAgents: vi.fn().mockResolvedValue({
+        total: 3,
+        killed: 3,
+        failed: 0,
+      }),
+    } as unknown as KillswitchService;
+
+    const queueService = {
+      addTask: vi.fn().mockResolvedValue(undefined),
+    } as unknown as QueueService;
+
+    const lifecycleService = {
+      getAgentLifecycleState: vi.fn(),
+    } as unknown as AgentLifecycleService;
+
+    controller = new AgentsController(
+      queueService,
+      spawnerService,
+      lifecycleService,
+      killswitchService
+    );
+  });
+
+  describe("Single agent kill", () => {
+    it("should kill a single agent by ID", async () => {
+      // Spawn an agent first
+      const spawnResult = await controller.spawn({
+        taskId: "kill-test-001",
+        agentType: "worker",
+        context: {
+          repository: "https://git.example.com/repo.git",
+          branch: "main",
+          workItems: ["US-001"],
+        },
+      });
+
+      // Kill the agent
+      const result = await controller.killAgent(spawnResult.agentId);
+
+      expect(result.message).toContain("killed successfully");
+      expect(killswitchService.killAgent).toHaveBeenCalledWith(spawnResult.agentId);
+    });
+
+    it("should handle kill of non-existent agent gracefully", async () => {
+      (killswitchService.killAgent as ReturnType<typeof vi.fn>).mockRejectedValue(
+        new Error("Agent not found")
+      );
+
+      await expect(controller.killAgent("non-existent")).rejects.toThrow("Agent not found");
+    });
+  });
+
+  describe("Kill all agents", () => {
+    it("should kill all active agents", async () => {
+      // Spawn multiple agents to verify they exist before kill-all
+      const spawned = [];
+      for (let i = 0; i < 3; i++) {
+        const result = await controller.spawn({
+          taskId: `kill-all-test-${String(i)}`,
+          agentType: "worker",
+          context: {
+            repository: "https://git.example.com/repo.git",
+            branch: "main",
+            workItems: [`US-${String(i)}`],
+          },
+        });
+        spawned.push(result);
+      }
+
+      // Verify agents were spawned
+      expect(spawnerService.listAgentSessions()).toHaveLength(3);
+
+      // Kill all (mock returns hardcoded result matching spawn count)
+      const result = await controller.killAllAgents();
+
+      expect(result.total).toBe(3);
+      expect(result.killed).toBe(3);
+      expect(result.failed).toBe(0);
+      expect(killswitchService.killAllAgents).toHaveBeenCalled();
+    });
+
+    it("should report partial failures in kill-all", async () => {
+      (killswitchService.killAllAgents as ReturnType<typeof vi.fn>).mockResolvedValue({
+        total: 3,
+        killed: 2,
+        failed: 1,
+        errors: ["Agent abc123 unresponsive"],
+      });
+
+      const result = await controller.killAllAgents();
+
+      expect(result.total).toBe(3);
+      expect(result.killed).toBe(2);
+      expect(result.failed).toBe(1);
+      expect(result.errors).toContain("Agent abc123 unresponsive");
+    });
+  });
+
+  describe("Kill during lifecycle states", () => {
+    it("should be able to kill agent in spawning state", async () => {
+      const spawnResult = await controller.spawn({
+        taskId: "kill-spawning-test",
+        agentType: "worker",
+        context: {
+          repository: "https://git.example.com/repo.git",
+          branch: "main",
+          workItems: ["US-001"],
+        },
+      });
+
+      // Verify agent is spawning
+      const agents = spawnerService.listAgentSessions();
+      const agent = agents.find((a) => a.agentId === spawnResult.agentId);
+      expect(agent?.state).toBe("spawning");
+
+      // Kill should succeed even in spawning state
+      const result = await controller.killAgent(spawnResult.agentId);
+      expect(result.message).toContain("killed successfully");
+    });
+  });
+});

apps/orchestrator/tests/integration/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: "node",
+    include: ["**/*.e2e-spec.ts"],
+    testTimeout: 30000,
+  },
+});