stack/apps/api/test/e2e/job-orchestration.e2e-spec.ts

/**
 * End-to-End tests for job orchestration
 * Tests the complete flow from webhook to job completion
 */

import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
import { RunnerJobStatus, JobStepStatus, JobStepPhase, JobStepType } from "@prisma/client";

// Services
import { StitcherService } from "../../src/stitcher/stitcher.service";
import { RunnerJobsService } from "../../src/runner-jobs/runner-jobs.service";
import { JobStepsService } from "../../src/job-steps/job-steps.service";
import { JobEventsService } from "../../src/job-events/job-events.service";
import { CommandParserService } from "../../src/bridge/parser/command-parser.service";

// Fixtures
import {
  createMockPrismaService,
  createMockBullMqService,
  createMockDiscordClient,
  createMockDiscordMessage,
} from "../fixtures";

// DTOs and interfaces
import type { WebhookPayloadDto } from "../../src/stitcher/dto";

describe("Job Orchestration E2E", () => {
  let stitcher: StitcherService;
  let runnerJobs: RunnerJobsService;
  let jobSteps: JobStepsService;
  let jobEvents: JobEventsService;
  let mockPrisma: ReturnType<typeof createMockPrismaService>;
  let mockBullMq: ReturnType<typeof createMockBullMqService>;
  let parser: CommandParserService;

  beforeEach(async () => {
    // Create mock services
    mockPrisma = createMockPrismaService();
    mockBullMq = createMockBullMqService();

    // Create services directly with mocks
    stitcher = new StitcherService(mockPrisma as never, mockBullMq as never);
    runnerJobs = new RunnerJobsService(mockPrisma as never, mockBullMq as never);
    jobSteps = new JobStepsService(mockPrisma as never);
    jobEvents = new JobEventsService(mockPrisma as never);
    parser = new CommandParserService();
  });

  afterEach(() => {
    vi.clearAllMocks();
  });

  describe("Happy Path: Webhook to Completion", () => {
    it("should create job from webhook, track steps, and complete successfully", async () => {
      // Step 1: Webhook arrives
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "42",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);

      // Verify job was created
      expect(dispatchResult.jobId).toBeDefined();
      expect(dispatchResult.status).toBe("PENDING");
      expect(dispatchResult.queueName).toBe("mosaic-jobs"); // MAIN queue
      expect(mockPrisma.runnerJob?.create).toHaveBeenCalled();

      // Verify job was queued in BullMQ
      expect(mockBullMq.addJob).toHaveBeenCalledWith(
        "mosaic-jobs", // MAIN queue
        "code-task",
        expect.objectContaining({
          jobId: dispatchResult.jobId,
          workspaceId: "default-workspace",
          type: "code-task",
        }),
        expect.objectContaining({ priority: 10 })
      );

      // Step 2: Create job steps
      const jobId = dispatchResult.jobId;
      const step1 = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Validate requirements",
        type: JobStepType.TOOL,
      });

      expect(step1).toBeDefined();
      expect(step1.ordinal).toBe(1);
      expect(step1.status).toBe(JobStepStatus.PENDING);

      const step2 = await jobSteps.create(jobId, {
        ordinal: 2,
        phase: JobStepPhase.IMPLEMENTATION,
        name: "Implement feature",
        type: JobStepType.TOOL,
      });

      expect(step2).toBeDefined();
      expect(step2.ordinal).toBe(2);

      // Step 3: Start job execution
      await runnerJobs.updateStatus(jobId, "default-workspace", RunnerJobStatus.RUNNING);

      // Step 4: Execute steps
      await jobSteps.start(step1.id);
      await jobSteps.complete(step1.id, {
        output: "Requirements validated successfully",
        tokensInput: 100,
        tokensOutput: 50,
      });

      const updatedStep1 = await jobSteps.findOne(step1.id);
      expect(updatedStep1?.status).toBe(JobStepStatus.COMPLETED);
      expect(updatedStep1?.output).toBe("Requirements validated successfully");

      await jobSteps.start(step2.id);
      await jobSteps.complete(step2.id, {
        output: "Feature implemented successfully",
        tokensInput: 500,
        tokensOutput: 200,
      });

      // Step 5: Mark job as completed
      await runnerJobs.updateStatus(jobId, "default-workspace", RunnerJobStatus.COMPLETED, {
        result: { success: true, message: "Job completed successfully" },
      });

      // Verify final job state
      const finalJob = await runnerJobs.findOne(jobId, "default-workspace");
      expect(finalJob?.status).toBe(RunnerJobStatus.COMPLETED);
      expect(finalJob?.result).toEqual({
        success: true,
        message: "Job completed successfully",
      });

      // Verify steps were created and completed
      expect(step1).toBeDefined();
      expect(step2).toBeDefined();
      expect(updatedStep1).toBeDefined();
      expect(updatedStep1?.status).toBe(JobStepStatus.COMPLETED);
    });

    it("should emit events throughout the job lifecycle", async () => {
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "123",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Verify job.created event was emitted by stitcher
      const createdEvent = await jobEvents.findByJob(jobId);
      expect(createdEvent.some((e) => e.type === "job.created")).toBe(true);

      // Verify job.queued event was emitted by stitcher
      expect(createdEvent.some((e) => e.type === "job.queued")).toBe(true);

      // Create and start a step
      const step = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Test step",
        type: JobStepType.TOOL,
      });

      await jobSteps.start(step.id);

      // In real implementation, step.started event would be emitted here
      // For E2E test with mocks, we verify the step was started successfully
      const updatedStep = await jobSteps.findOne(step.id);
      expect(updatedStep?.status).toBe(JobStepStatus.RUNNING);

      // Complete the step
      await jobSteps.complete(step.id, {
        output: "Step completed",
      });

      // Verify step was completed
      const completedStep = await jobSteps.findOne(step.id);
      expect(completedStep?.status).toBe(JobStepStatus.COMPLETED);
      expect(completedStep?.output).toBe("Step completed");
    });
  });

  describe("Error Handling: Step Failure and Retry", () => {
    it("should handle step failure and allow retry", async () => {
      // Create a job
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "789",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Create a step
      const step = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Failing step",
        type: JobStepType.TOOL,
      });

      // Start and fail the step
      await jobSteps.start(step.id);
      await jobSteps.fail(step.id, {
        error: "Step failed due to validation error",
      });

      const failedStep = await jobSteps.findOne(step.id);
      expect(failedStep?.status).toBe(JobStepStatus.FAILED);

      // Note: In real implementation, step.failed events would be emitted automatically
      // For this E2E test, we verify the step status is FAILED
      // Events would be verified in integration tests with the full event system

      // Retry the step
      const retriedStep = await jobSteps.create(jobId, {
        ordinal: 2,
        phase: JobStepPhase.VALIDATION,
        name: "Failing step (retry)",
        type: JobStepType.TOOL,
      });

      await jobSteps.start(retriedStep.id);
      await jobSteps.complete(retriedStep.id, {
        output: "Step succeeded on retry",
      });

      const completedStep = await jobSteps.findOne(retriedStep.id);
      expect(completedStep?.status).toBe(JobStepStatus.COMPLETED);
    });

    it("should mark job as failed after max retries", async () => {
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "999",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Simulate multiple step failures
      const step1 = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Attempt 1",
        type: JobStepType.TOOL,
      });
      await jobSteps.start(step1.id);
      await jobSteps.fail(step1.id, { error: "Failure attempt 1" });

      const step2 = await jobSteps.create(jobId, {
        ordinal: 2,
        phase: JobStepPhase.VALIDATION,
        name: "Attempt 2",
        type: JobStepType.TOOL,
      });
      await jobSteps.start(step2.id);
      await jobSteps.fail(step2.id, { error: "Failure attempt 2" });

      const step3 = await jobSteps.create(jobId, {
        ordinal: 3,
        phase: JobStepPhase.VALIDATION,
        name: "Attempt 3",
        type: JobStepType.TOOL,
      });
      await jobSteps.start(step3.id);
      await jobSteps.fail(step3.id, { error: "Failure attempt 3" });

      // Mark job as failed after max retries
      await runnerJobs.updateStatus(jobId, "default-workspace", RunnerJobStatus.FAILED, {
        error: "Max retries exceeded",
      });

      const failedJob = await runnerJobs.findOne(jobId, "default-workspace");
      expect(failedJob?.status).toBe(RunnerJobStatus.FAILED);
      expect(failedJob?.error).toBe("Max retries exceeded");

      // Verify steps were created and failed
      expect(step1.status).toBe(JobStepStatus.PENDING); // Initial status
      expect(step2.status).toBe(JobStepStatus.PENDING);
      expect(step3.status).toBe(JobStepStatus.PENDING);
    });
  });

  describe("Chat Integration: Command to Job", () => {
    it("should parse Discord command and create job", async () => {
      // Mock Discord message with @mosaic command
      const message = createMockDiscordMessage("@mosaic fix #42");

      // Parse the command
      const parseResult = parser.parseCommand(message.content as string);

      expect(parseResult).toBeDefined();
      expect(parseResult.success).toBe(true);
      if (parseResult.success) {
        expect(parseResult.command.action).toBe("fix");
        expect(parseResult.command.issue?.number).toBe(42); // number, not string
      }

      // Create job from parsed command
      const dispatchResult = await stitcher.dispatchJob({
        workspaceId: "workspace-123",
        type: "code-task",
        priority: 10,
        metadata: {
          command: parseResult.success ? parseResult.command.action : "unknown",
          issueNumber: parseResult.success ? parseResult.command.issue?.number : "unknown",
          source: "discord",
        },
      });

      expect(dispatchResult.jobId).toBeDefined();
      expect(dispatchResult.status).toBe("PENDING");

      // Verify job was created with correct metadata
      const job = await runnerJobs.findOne(dispatchResult.jobId, "workspace-123");
      expect(job).toBeDefined();
      expect(job?.type).toBe("code-task");
    });

    it("should broadcast status updates via WebSocket", async () => {
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "555",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Create and start a step
      const step = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Test step",
        type: JobStepType.TOOL,
      });

      await jobSteps.start(step.id);

      // In real implementation, WebSocket events would be emitted here
      // For E2E test, we verify the step was created and started
      expect(step).toBeDefined();
      expect(step.status).toBe(JobStepStatus.PENDING);
    });
  });

  describe("Job Lifecycle Management", () => {
    it("should handle job cancellation", async () => {
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "111",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Cancel the job
      const canceledJob = await runnerJobs.cancel(jobId, "default-workspace");

      expect(canceledJob.status).toBe(RunnerJobStatus.CANCELLED);
      expect(canceledJob.completedAt).toBeDefined();
    });

    it("should support job retry", async () => {
      // Create and fail a job
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "222",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Mark as failed
      await runnerJobs.updateStatus(jobId, "default-workspace", RunnerJobStatus.FAILED, {
        error: "Job failed",
      });

      // Retry the job
      const retriedJob = await runnerJobs.retry(jobId, "default-workspace");

      expect(retriedJob).toBeDefined();
      expect(retriedJob.status).toBe(RunnerJobStatus.PENDING);
      expect(retriedJob.id).not.toBe(jobId); // New job created
    });

    it("should track progress percentage", async () => {
      const webhookPayload: WebhookPayloadDto = {
        issueNumber: "333",
        repository: "mosaic/stack",
        action: "assigned",
      };

      const dispatchResult = await stitcher.handleWebhook(webhookPayload);
      const jobId = dispatchResult.jobId;

      // Create 3 steps
      const step1 = await jobSteps.create(jobId, {
        ordinal: 1,
        phase: JobStepPhase.VALIDATION,
        name: "Step 1",
        type: JobStepType.TOOL,
      });

      const step2 = await jobSteps.create(jobId, {
        ordinal: 2,
        phase: JobStepPhase.VALIDATION,
        name: "Step 2",
        type: JobStepType.TOOL,
      });

      const step3 = await jobSteps.create(jobId, {
        ordinal: 3,
        phase: JobStepPhase.VALIDATION,
        name: "Step 3",
        type: JobStepType.TOOL,
      });

      // Complete first step - should be 33% progress
      await jobSteps.start(step1.id);
      await jobSteps.complete(step1.id, { output: "Done" });

      // Update job progress (in real implementation, this would be automatic)
      await runnerJobs.updateProgress(jobId, "default-workspace", 33);

      let job = await runnerJobs.findOne(jobId, "default-workspace");
      expect(job?.progressPercent).toBe(33);

      // Complete remaining steps
      await jobSteps.start(step2.id);
      await jobSteps.complete(step2.id, { output: "Done" });
      await runnerJobs.updateProgress(jobId, "default-workspace", 66);

      job = await runnerJobs.findOne(jobId, "default-workspace");
      expect(job?.progressPercent).toBe(66);

      await jobSteps.start(step3.id);
      await jobSteps.complete(step3.id, { output: "Done" });
      await runnerJobs.updateProgress(jobId, "default-workspace", 100);

      job = await runnerJobs.findOne(jobId, "default-workspace");
      expect(job?.progressPercent).toBe(100);
    });
  });
});