feat(#134): design Non-AI Quality Orchestrator service
Implement quality orchestration service to enforce standards on AI agent work and prevent premature completion claims. Components: - QualityOrchestratorService: Core validation and gate execution - QualityGate interface: Extensible gate definitions - CompletionClaim/Validation: Track claims and verdicts - OrchestrationConfig: Per-workspace configuration Features: - Validate completions against quality gates (build/lint/test/coverage) - Run gates with command execution and output validation - Support string and RegExp output pattern matching - Smart continuation logic with attempt tracking - Generate actionable feedback for failed gates - Strict/lenient mode for gate enforcement - 5-minute timeout, 10MB output buffer per gate Default gates: - Build Check (required) - Lint Check (required) - Test Suite (required) - Coverage Check (optional, 85% threshold) Tests: 21 passing with 85.98% coverage Fixes #134 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,470 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { Test, TestingModule } from "@nestjs/testing";
|
||||
import { QualityOrchestratorService } from "./quality-orchestrator.service";
|
||||
import type {
|
||||
QualityGate,
|
||||
CompletionClaim,
|
||||
OrchestrationConfig,
|
||||
CompletionValidation,
|
||||
} from "./interfaces";
|
||||
|
||||
describe("QualityOrchestratorService", () => {
|
||||
let service: QualityOrchestratorService;
|
||||
|
||||
const mockWorkspaceId = "workspace-1";
|
||||
const mockTaskId = "task-1";
|
||||
const mockAgentId = "agent-1";
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [QualityOrchestratorService],
|
||||
}).compile();
|
||||
|
||||
service = module.get<QualityOrchestratorService>(QualityOrchestratorService);
|
||||
});
|
||||
|
||||
it("should be defined", () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe("validateCompletion", () => {
|
||||
const claim: CompletionClaim = {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Task completed successfully",
|
||||
filesChanged: ["src/test.ts"],
|
||||
};
|
||||
|
||||
const config: OrchestrationConfig = {
|
||||
workspaceId: mockWorkspaceId,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "echo 'build success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
maxContinuations: 3,
|
||||
continuationBudget: 10000,
|
||||
strictMode: false,
|
||||
};
|
||||
|
||||
it("should accept completion when all required gates pass", async () => {
|
||||
const result = await service.validateCompletion(claim, config);
|
||||
|
||||
expect(result.verdict).toBe("accepted");
|
||||
expect(result.allGatesPassed).toBe(true);
|
||||
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should reject completion when required gates fail", async () => {
|
||||
const failingConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, failingConfig);
|
||||
|
||||
expect(result.verdict).toBe("rejected");
|
||||
expect(result.allGatesPassed).toBe(false);
|
||||
expect(result.requiredGatesFailed).toContain("build");
|
||||
});
|
||||
|
||||
it("should accept when optional gates fail but required gates pass", async () => {
|
||||
const mixedConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "echo 'success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
{
|
||||
id: "coverage",
|
||||
name: "Coverage Check",
|
||||
description: "Check coverage",
|
||||
type: "coverage",
|
||||
command: "exit 1",
|
||||
required: false,
|
||||
order: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, mixedConfig);
|
||||
|
||||
expect(result.verdict).toBe("accepted");
|
||||
expect(result.allGatesPassed).toBe(false);
|
||||
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should provide feedback when gates fail", async () => {
|
||||
const failingConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "test",
|
||||
name: "Test Suite",
|
||||
description: "Run tests",
|
||||
type: "test",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, failingConfig);
|
||||
|
||||
expect(result.feedback).toBeDefined();
|
||||
expect(result.suggestedActions).toBeDefined();
|
||||
expect(result.suggestedActions!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should run gates in order", async () => {
|
||||
const orderedConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "gate3",
|
||||
name: "Third Gate",
|
||||
description: "Third",
|
||||
type: "custom",
|
||||
command: "echo 'third'",
|
||||
required: true,
|
||||
order: 3,
|
||||
},
|
||||
{
|
||||
id: "gate1",
|
||||
name: "First Gate",
|
||||
description: "First",
|
||||
type: "custom",
|
||||
command: "echo 'first'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
{
|
||||
id: "gate2",
|
||||
name: "Second Gate",
|
||||
description: "Second",
|
||||
type: "custom",
|
||||
command: "echo 'second'",
|
||||
required: true,
|
||||
order: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, orderedConfig);
|
||||
|
||||
expect(result.gateResults[0].gateId).toBe("gate1");
|
||||
expect(result.gateResults[1].gateId).toBe("gate2");
|
||||
expect(result.gateResults[2].gateId).toBe("gate3");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runGate", () => {
|
||||
it("should successfully run a gate with passing command", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "test-gate",
|
||||
name: "Test Gate",
|
||||
description: "Test description",
|
||||
type: "custom",
|
||||
command: "echo 'success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.gateId).toBe("test-gate");
|
||||
expect(result.gateName).toBe("Test Gate");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.duration).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should fail a gate with failing command", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "fail-gate",
|
||||
name: "Failing Gate",
|
||||
description: "Should fail",
|
||||
type: "custom",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.error).toBeDefined();
|
||||
});
|
||||
|
||||
it("should capture output from gate execution", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "output-gate",
|
||||
name: "Output Gate",
|
||||
description: "Captures output",
|
||||
type: "custom",
|
||||
command: "echo 'test output'",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.output).toContain("test output");
|
||||
});
|
||||
|
||||
it("should validate expected output pattern", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "pattern-gate",
|
||||
name: "Pattern Gate",
|
||||
description: "Checks output pattern",
|
||||
type: "custom",
|
||||
command: "echo 'coverage: 90%'",
|
||||
expectedOutput: /coverage: \d+%/,
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("should fail when expected output pattern does not match", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "bad-pattern-gate",
|
||||
name: "Bad Pattern Gate",
|
||||
description: "Pattern should not match",
|
||||
type: "custom",
|
||||
command: "echo 'no coverage info'",
|
||||
expectedOutput: /coverage: \d+%/,
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldContinue", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
const config: OrchestrationConfig = {
|
||||
workspaceId: mockWorkspaceId,
|
||||
gates: [],
|
||||
maxContinuations: 3,
|
||||
continuationBudget: 10000,
|
||||
strictMode: false,
|
||||
};
|
||||
|
||||
it("should continue when under max continuations", () => {
|
||||
const result = service.shouldContinue(validation, 1, config);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it("should not continue when at max continuations", () => {
|
||||
const result = service.shouldContinue(validation, 3, config);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it("should not continue when validation is accepted", () => {
|
||||
const acceptedValidation: CompletionValidation = {
|
||||
...validation,
|
||||
verdict: "accepted",
|
||||
allGatesPassed: true,
|
||||
requiredGatesFailed: [],
|
||||
};
|
||||
|
||||
const result = service.shouldContinue(acceptedValidation, 1, config);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateContinuationPrompt", () => {
|
||||
it("should generate prompt with failed gate information", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [
|
||||
{
|
||||
gateId: "test",
|
||||
gateName: "Test Suite",
|
||||
passed: false,
|
||||
error: "Tests failed",
|
||||
duration: 1000,
|
||||
},
|
||||
],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
const prompt = service.generateContinuationPrompt(validation);
|
||||
|
||||
expect(prompt).toContain("Test Suite");
|
||||
expect(prompt).toContain("failed");
|
||||
});
|
||||
|
||||
it("should include suggested actions in prompt", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["lint"],
|
||||
verdict: "needs-continuation",
|
||||
suggestedActions: ["Run: pnpm lint --fix", "Check code style"],
|
||||
};
|
||||
|
||||
const prompt = service.generateContinuationPrompt(validation);
|
||||
|
||||
expect(prompt).toContain("pnpm lint --fix");
|
||||
expect(prompt).toContain("Check code style");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateRejectionFeedback", () => {
|
||||
it("should generate detailed rejection feedback", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [
|
||||
{
|
||||
gateId: "build",
|
||||
gateName: "Build Check",
|
||||
passed: false,
|
||||
error: "Compilation error",
|
||||
duration: 500,
|
||||
},
|
||||
],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["build"],
|
||||
verdict: "rejected",
|
||||
};
|
||||
|
||||
const feedback = service.generateRejectionFeedback(validation);
|
||||
|
||||
expect(feedback).toContain("rejected");
|
||||
expect(feedback).toContain("Build Check");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getDefaultGates", () => {
|
||||
it("should return default gates for workspace", () => {
|
||||
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||
|
||||
expect(gates).toBeDefined();
|
||||
expect(gates.length).toBeGreaterThan(0);
|
||||
expect(gates.some((g) => g.id === "build")).toBe(true);
|
||||
expect(gates.some((g) => g.id === "lint")).toBe(true);
|
||||
expect(gates.some((g) => g.id === "test")).toBe(true);
|
||||
});
|
||||
|
||||
it("should return gates in correct order", () => {
|
||||
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||
|
||||
for (let i = 1; i < gates.length; i++) {
|
||||
expect(gates[i].order).toBeGreaterThanOrEqual(gates[i - 1].order);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordContinuation", () => {
|
||||
it("should record continuation attempt", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
expect(() => service.recordContinuation(mockTaskId, 1, validation)).not.toThrow();
|
||||
});
|
||||
|
||||
it("should handle multiple continuation records", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
service.recordContinuation(mockTaskId, 1, validation);
|
||||
service.recordContinuation(mockTaskId, 2, validation);
|
||||
|
||||
expect(() => service.recordContinuation(mockTaskId, 3, validation)).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user