feat(#134): design Non-AI Quality Orchestrator service
Implement quality orchestration service to enforce standards on AI agent work and prevent premature completion claims. Components: - QualityOrchestratorService: Core validation and gate execution - QualityGate interface: Extensible gate definitions - CompletionClaim/Validation: Track claims and verdicts - OrchestrationConfig: Per-workspace configuration Features: - Validate completions against quality gates (build/lint/test/coverage) - Run gates with command execution and output validation - Support string and RegExp output pattern matching - Smart continuation logic with attempt tracking - Generate actionable feedback for failed gates - Strict/lenient mode for gate enforcement - 5-minute timeout, 10MB output buffer per gate Default gates: - Build Check (required) - Lint Check (required) - Test Suite (required) - Coverage Check (optional, 85% threshold) Tests: 21 passing with 85.98% coverage Fixes #134 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2
apps/api/src/quality-orchestrator/dto/index.ts
Normal file
2
apps/api/src/quality-orchestrator/dto/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export * from "./validate-completion.dto";
|
||||
export * from "./orchestration-result.dto";
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { QualityGateResult } from "../interfaces";
|
||||
|
||||
/**
|
||||
* DTO for orchestration results
|
||||
*/
|
||||
export interface OrchestrationResultDto {
|
||||
/** Task ID */
|
||||
taskId: string;
|
||||
|
||||
/** Whether the completion was accepted */
|
||||
accepted: boolean;
|
||||
|
||||
/** Verdict from validation */
|
||||
verdict: "accepted" | "rejected" | "needs-continuation";
|
||||
|
||||
/** All gates passed */
|
||||
allGatesPassed: boolean;
|
||||
|
||||
/** Required gates that failed */
|
||||
requiredGatesFailed: string[];
|
||||
|
||||
/** Results from each gate */
|
||||
gateResults: QualityGateResult[];
|
||||
|
||||
/** Feedback for the agent */
|
||||
feedback?: string;
|
||||
|
||||
/** Suggested actions to fix issues */
|
||||
suggestedActions?: string[];
|
||||
|
||||
/** Continuation prompt if needed */
|
||||
continuationPrompt?: string;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { IsString, IsArray, IsDateString, IsNotEmpty, ArrayMinSize } from "class-validator";
|
||||
|
||||
/**
|
||||
* DTO for validating a completion claim
|
||||
*/
|
||||
export class ValidateCompletionDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
taskId!: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
agentId!: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
workspaceId!: string;
|
||||
|
||||
@IsDateString()
|
||||
claimedAt!: string;
|
||||
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
message!: string;
|
||||
|
||||
@IsArray()
|
||||
@ArrayMinSize(0)
|
||||
@IsString({ each: true })
|
||||
filesChanged!: string[];
|
||||
}
|
||||
4
apps/api/src/quality-orchestrator/index.ts
Normal file
4
apps/api/src/quality-orchestrator/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export * from "./quality-orchestrator.module";
|
||||
export * from "./quality-orchestrator.service";
|
||||
export * from "./interfaces";
|
||||
export * from "./dto";
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { QualityGateResult } from "./quality-gate.interface";
|
||||
|
||||
/**
|
||||
* Claim by an agent that a task is complete
|
||||
*/
|
||||
export interface CompletionClaim {
|
||||
/** ID of the task being claimed as complete */
|
||||
taskId: string;
|
||||
|
||||
/** ID of the agent making the claim */
|
||||
agentId: string;
|
||||
|
||||
/** Workspace context */
|
||||
workspaceId: string;
|
||||
|
||||
/** Timestamp of claim */
|
||||
claimedAt: Date;
|
||||
|
||||
/** Agent's message about completion */
|
||||
message: string;
|
||||
|
||||
/** List of files changed during task execution */
|
||||
filesChanged: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of validating a completion claim
|
||||
*/
|
||||
export interface CompletionValidation {
|
||||
/** Original claim being validated */
|
||||
claim: CompletionClaim;
|
||||
|
||||
/** Results from all quality gates */
|
||||
gateResults: QualityGateResult[];
|
||||
|
||||
/** Whether all gates passed */
|
||||
allGatesPassed: boolean;
|
||||
|
||||
/** List of required gates that failed */
|
||||
requiredGatesFailed: string[];
|
||||
|
||||
/** Final verdict on the completion */
|
||||
verdict: "accepted" | "rejected" | "needs-continuation";
|
||||
|
||||
/** Feedback message for the agent */
|
||||
feedback?: string;
|
||||
|
||||
/** Specific actions to take to fix failures */
|
||||
suggestedActions?: string[];
|
||||
}
|
||||
3
apps/api/src/quality-orchestrator/interfaces/index.ts
Normal file
3
apps/api/src/quality-orchestrator/interfaces/index.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export * from "./quality-gate.interface";
|
||||
export * from "./completion-result.interface";
|
||||
export * from "./orchestration-config.interface";
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { QualityGate } from "./quality-gate.interface";
|
||||
|
||||
/**
|
||||
* Configuration for quality orchestration
|
||||
*/
|
||||
export interface OrchestrationConfig {
|
||||
/** Workspace this config applies to */
|
||||
workspaceId: string;
|
||||
|
||||
/** Quality gates to enforce */
|
||||
gates: QualityGate[];
|
||||
|
||||
/** Maximum number of continuation attempts */
|
||||
maxContinuations: number;
|
||||
|
||||
/** Token budget for continuations */
|
||||
continuationBudget: number;
|
||||
|
||||
/** Whether to reject on ANY failure vs only required gates */
|
||||
strictMode: boolean;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Defines a quality gate that must be passed for task completion
|
||||
*/
|
||||
export interface QualityGate {
|
||||
/** Unique identifier for the gate */
|
||||
id: string;
|
||||
|
||||
/** Human-readable name */
|
||||
name: string;
|
||||
|
||||
/** Description of what this gate checks */
|
||||
description: string;
|
||||
|
||||
/** Type of quality check */
|
||||
type: "test" | "lint" | "build" | "coverage" | "custom";
|
||||
|
||||
/** Command to execute for this gate (optional for custom gates) */
|
||||
command?: string;
|
||||
|
||||
/** Expected output pattern (optional, for validation) */
|
||||
expectedOutput?: string | RegExp;
|
||||
|
||||
/** Whether this gate must pass for completion */
|
||||
required: boolean;
|
||||
|
||||
/** Execution order (lower numbers run first) */
|
||||
order: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of running a quality gate
|
||||
*/
|
||||
export interface QualityGateResult {
|
||||
/** ID of the gate that was run */
|
||||
gateId: string;
|
||||
|
||||
/** Name of the gate */
|
||||
gateName: string;
|
||||
|
||||
/** Whether the gate passed */
|
||||
passed: boolean;
|
||||
|
||||
/** Output from running the gate */
|
||||
output?: string;
|
||||
|
||||
/** Error message if gate failed */
|
||||
error?: string;
|
||||
|
||||
/** Duration in milliseconds */
|
||||
duration: number;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { QualityOrchestratorService } from "./quality-orchestrator.service";
|
||||
|
||||
/**
|
||||
* Quality Orchestrator Module
|
||||
* Provides quality enforcement for AI agent task completions
|
||||
*/
|
||||
@Module({
|
||||
providers: [QualityOrchestratorService],
|
||||
exports: [QualityOrchestratorService],
|
||||
})
|
||||
export class QualityOrchestratorModule {}
|
||||
@@ -0,0 +1,470 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { Test, TestingModule } from "@nestjs/testing";
|
||||
import { QualityOrchestratorService } from "./quality-orchestrator.service";
|
||||
import type {
|
||||
QualityGate,
|
||||
CompletionClaim,
|
||||
OrchestrationConfig,
|
||||
CompletionValidation,
|
||||
} from "./interfaces";
|
||||
|
||||
describe("QualityOrchestratorService", () => {
|
||||
let service: QualityOrchestratorService;
|
||||
|
||||
const mockWorkspaceId = "workspace-1";
|
||||
const mockTaskId = "task-1";
|
||||
const mockAgentId = "agent-1";
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [QualityOrchestratorService],
|
||||
}).compile();
|
||||
|
||||
service = module.get<QualityOrchestratorService>(QualityOrchestratorService);
|
||||
});
|
||||
|
||||
it("should be defined", () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe("validateCompletion", () => {
|
||||
const claim: CompletionClaim = {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Task completed successfully",
|
||||
filesChanged: ["src/test.ts"],
|
||||
};
|
||||
|
||||
const config: OrchestrationConfig = {
|
||||
workspaceId: mockWorkspaceId,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "echo 'build success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
maxContinuations: 3,
|
||||
continuationBudget: 10000,
|
||||
strictMode: false,
|
||||
};
|
||||
|
||||
it("should accept completion when all required gates pass", async () => {
|
||||
const result = await service.validateCompletion(claim, config);
|
||||
|
||||
expect(result.verdict).toBe("accepted");
|
||||
expect(result.allGatesPassed).toBe(true);
|
||||
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should reject completion when required gates fail", async () => {
|
||||
const failingConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, failingConfig);
|
||||
|
||||
expect(result.verdict).toBe("rejected");
|
||||
expect(result.allGatesPassed).toBe(false);
|
||||
expect(result.requiredGatesFailed).toContain("build");
|
||||
});
|
||||
|
||||
it("should accept when optional gates fail but required gates pass", async () => {
|
||||
const mixedConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles",
|
||||
type: "build",
|
||||
command: "echo 'success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
{
|
||||
id: "coverage",
|
||||
name: "Coverage Check",
|
||||
description: "Check coverage",
|
||||
type: "coverage",
|
||||
command: "exit 1",
|
||||
required: false,
|
||||
order: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, mixedConfig);
|
||||
|
||||
expect(result.verdict).toBe("accepted");
|
||||
expect(result.allGatesPassed).toBe(false);
|
||||
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should provide feedback when gates fail", async () => {
|
||||
const failingConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "test",
|
||||
name: "Test Suite",
|
||||
description: "Run tests",
|
||||
type: "test",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, failingConfig);
|
||||
|
||||
expect(result.feedback).toBeDefined();
|
||||
expect(result.suggestedActions).toBeDefined();
|
||||
expect(result.suggestedActions!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should run gates in order", async () => {
|
||||
const orderedConfig: OrchestrationConfig = {
|
||||
...config,
|
||||
gates: [
|
||||
{
|
||||
id: "gate3",
|
||||
name: "Third Gate",
|
||||
description: "Third",
|
||||
type: "custom",
|
||||
command: "echo 'third'",
|
||||
required: true,
|
||||
order: 3,
|
||||
},
|
||||
{
|
||||
id: "gate1",
|
||||
name: "First Gate",
|
||||
description: "First",
|
||||
type: "custom",
|
||||
command: "echo 'first'",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
{
|
||||
id: "gate2",
|
||||
name: "Second Gate",
|
||||
description: "Second",
|
||||
type: "custom",
|
||||
command: "echo 'second'",
|
||||
required: true,
|
||||
order: 2,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.validateCompletion(claim, orderedConfig);
|
||||
|
||||
expect(result.gateResults[0].gateId).toBe("gate1");
|
||||
expect(result.gateResults[1].gateId).toBe("gate2");
|
||||
expect(result.gateResults[2].gateId).toBe("gate3");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runGate", () => {
|
||||
it("should successfully run a gate with passing command", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "test-gate",
|
||||
name: "Test Gate",
|
||||
description: "Test description",
|
||||
type: "custom",
|
||||
command: "echo 'success'",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.gateId).toBe("test-gate");
|
||||
expect(result.gateName).toBe("Test Gate");
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.duration).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should fail a gate with failing command", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "fail-gate",
|
||||
name: "Failing Gate",
|
||||
description: "Should fail",
|
||||
type: "custom",
|
||||
command: "exit 1",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.error).toBeDefined();
|
||||
});
|
||||
|
||||
it("should capture output from gate execution", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "output-gate",
|
||||
name: "Output Gate",
|
||||
description: "Captures output",
|
||||
type: "custom",
|
||||
command: "echo 'test output'",
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.output).toContain("test output");
|
||||
});
|
||||
|
||||
it("should validate expected output pattern", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "pattern-gate",
|
||||
name: "Pattern Gate",
|
||||
description: "Checks output pattern",
|
||||
type: "custom",
|
||||
command: "echo 'coverage: 90%'",
|
||||
expectedOutput: /coverage: \d+%/,
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
});
|
||||
|
||||
it("should fail when expected output pattern does not match", async () => {
|
||||
const gate: QualityGate = {
|
||||
id: "bad-pattern-gate",
|
||||
name: "Bad Pattern Gate",
|
||||
description: "Pattern should not match",
|
||||
type: "custom",
|
||||
command: "echo 'no coverage info'",
|
||||
expectedOutput: /coverage: \d+%/,
|
||||
required: true,
|
||||
order: 1,
|
||||
};
|
||||
|
||||
const result = await service.runGate(gate);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldContinue", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
const config: OrchestrationConfig = {
|
||||
workspaceId: mockWorkspaceId,
|
||||
gates: [],
|
||||
maxContinuations: 3,
|
||||
continuationBudget: 10000,
|
||||
strictMode: false,
|
||||
};
|
||||
|
||||
it("should continue when under max continuations", () => {
|
||||
const result = service.shouldContinue(validation, 1, config);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
it("should not continue when at max continuations", () => {
|
||||
const result = service.shouldContinue(validation, 3, config);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it("should not continue when validation is accepted", () => {
|
||||
const acceptedValidation: CompletionValidation = {
|
||||
...validation,
|
||||
verdict: "accepted",
|
||||
allGatesPassed: true,
|
||||
requiredGatesFailed: [],
|
||||
};
|
||||
|
||||
const result = service.shouldContinue(acceptedValidation, 1, config);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateContinuationPrompt", () => {
|
||||
it("should generate prompt with failed gate information", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [
|
||||
{
|
||||
gateId: "test",
|
||||
gateName: "Test Suite",
|
||||
passed: false,
|
||||
error: "Tests failed",
|
||||
duration: 1000,
|
||||
},
|
||||
],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
const prompt = service.generateContinuationPrompt(validation);
|
||||
|
||||
expect(prompt).toContain("Test Suite");
|
||||
expect(prompt).toContain("failed");
|
||||
});
|
||||
|
||||
it("should include suggested actions in prompt", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["lint"],
|
||||
verdict: "needs-continuation",
|
||||
suggestedActions: ["Run: pnpm lint --fix", "Check code style"],
|
||||
};
|
||||
|
||||
const prompt = service.generateContinuationPrompt(validation);
|
||||
|
||||
expect(prompt).toContain("pnpm lint --fix");
|
||||
expect(prompt).toContain("Check code style");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateRejectionFeedback", () => {
|
||||
it("should generate detailed rejection feedback", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [
|
||||
{
|
||||
gateId: "build",
|
||||
gateName: "Build Check",
|
||||
passed: false,
|
||||
error: "Compilation error",
|
||||
duration: 500,
|
||||
},
|
||||
],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["build"],
|
||||
verdict: "rejected",
|
||||
};
|
||||
|
||||
const feedback = service.generateRejectionFeedback(validation);
|
||||
|
||||
expect(feedback).toContain("rejected");
|
||||
expect(feedback).toContain("Build Check");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getDefaultGates", () => {
|
||||
it("should return default gates for workspace", () => {
|
||||
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||
|
||||
expect(gates).toBeDefined();
|
||||
expect(gates.length).toBeGreaterThan(0);
|
||||
expect(gates.some((g) => g.id === "build")).toBe(true);
|
||||
expect(gates.some((g) => g.id === "lint")).toBe(true);
|
||||
expect(gates.some((g) => g.id === "test")).toBe(true);
|
||||
});
|
||||
|
||||
it("should return gates in correct order", () => {
|
||||
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||
|
||||
for (let i = 1; i < gates.length; i++) {
|
||||
expect(gates[i].order).toBeGreaterThanOrEqual(gates[i - 1].order);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("recordContinuation", () => {
|
||||
it("should record continuation attempt", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
expect(() => service.recordContinuation(mockTaskId, 1, validation)).not.toThrow();
|
||||
});
|
||||
|
||||
it("should handle multiple continuation records", () => {
|
||||
const validation: CompletionValidation = {
|
||||
claim: {
|
||||
taskId: mockTaskId,
|
||||
agentId: mockAgentId,
|
||||
workspaceId: mockWorkspaceId,
|
||||
claimedAt: new Date(),
|
||||
message: "Done",
|
||||
filesChanged: [],
|
||||
},
|
||||
gateResults: [],
|
||||
allGatesPassed: false,
|
||||
requiredGatesFailed: ["test"],
|
||||
verdict: "needs-continuation",
|
||||
};
|
||||
|
||||
service.recordContinuation(mockTaskId, 1, validation);
|
||||
service.recordContinuation(mockTaskId, 2, validation);
|
||||
|
||||
expect(() => service.recordContinuation(mockTaskId, 3, validation)).not.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,316 @@
|
||||
import { Injectable, Logger } from "@nestjs/common";
|
||||
import { exec } from "child_process";
|
||||
import { promisify } from "util";
|
||||
import type {
|
||||
QualityGate,
|
||||
QualityGateResult,
|
||||
CompletionClaim,
|
||||
CompletionValidation,
|
||||
OrchestrationConfig,
|
||||
} from "./interfaces";
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
/**
|
||||
* Default quality gates for all workspaces
|
||||
*/
|
||||
const DEFAULT_GATES: QualityGate[] = [
|
||||
{
|
||||
id: "build",
|
||||
name: "Build Check",
|
||||
description: "Verify code compiles without errors",
|
||||
type: "build",
|
||||
command: "pnpm build",
|
||||
required: true,
|
||||
order: 1,
|
||||
},
|
||||
{
|
||||
id: "lint",
|
||||
name: "Lint Check",
|
||||
description: "Code follows style guidelines",
|
||||
type: "lint",
|
||||
command: "pnpm lint",
|
||||
required: true,
|
||||
order: 2,
|
||||
},
|
||||
{
|
||||
id: "test",
|
||||
name: "Test Suite",
|
||||
description: "All tests pass",
|
||||
type: "test",
|
||||
command: "pnpm test",
|
||||
required: true,
|
||||
order: 3,
|
||||
},
|
||||
{
|
||||
id: "coverage",
|
||||
name: "Coverage Check",
|
||||
description: "Test coverage >= 85%",
|
||||
type: "coverage",
|
||||
command: "pnpm test:coverage",
|
||||
expectedOutput: /All files.*[89]\d|100/,
|
||||
required: false,
|
||||
order: 4,
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Quality Orchestrator Service
|
||||
* Validates AI agent task completions and enforces quality gates
|
||||
*/
|
||||
@Injectable()
|
||||
export class QualityOrchestratorService {
|
||||
private readonly logger = new Logger(QualityOrchestratorService.name);
|
||||
|
||||
/**
|
||||
* Validate a completion claim against quality gates
|
||||
*/
|
||||
async validateCompletion(
|
||||
claim: CompletionClaim,
|
||||
config: OrchestrationConfig
|
||||
): Promise<CompletionValidation> {
|
||||
this.logger.log(
|
||||
`Validating completion claim for task ${claim.taskId} by agent ${claim.agentId}`
|
||||
);
|
||||
|
||||
// Sort gates by order
|
||||
const sortedGates = [...config.gates].sort((a, b) => a.order - b.order);
|
||||
|
||||
// Run all gates
|
||||
const gateResults: QualityGateResult[] = [];
|
||||
for (const gate of sortedGates) {
|
||||
const result = await this.runGate(gate);
|
||||
gateResults.push(result);
|
||||
}
|
||||
|
||||
// Analyze results
|
||||
const allGatesPassed = gateResults.every((r) => r.passed);
|
||||
const requiredGatesFailed = gateResults
|
||||
.filter((r) => !r.passed)
|
||||
.map((r) => r.gateId)
|
||||
.filter((id) => {
|
||||
const gate = config.gates.find((g) => g.id === id);
|
||||
return gate?.required ?? false;
|
||||
});
|
||||
|
||||
// Determine verdict
|
||||
let verdict: "accepted" | "rejected" | "needs-continuation";
|
||||
if (allGatesPassed) {
|
||||
verdict = "accepted";
|
||||
} else if (requiredGatesFailed.length > 0) {
|
||||
verdict = "rejected";
|
||||
} else if (config.strictMode) {
|
||||
verdict = "rejected";
|
||||
} else {
|
||||
verdict = "accepted";
|
||||
}
|
||||
|
||||
// Generate feedback and suggestions
|
||||
const result: CompletionValidation = {
|
||||
claim,
|
||||
gateResults,
|
||||
allGatesPassed,
|
||||
requiredGatesFailed,
|
||||
verdict,
|
||||
};
|
||||
|
||||
if (verdict !== "accepted") {
|
||||
result.feedback = this.generateRejectionFeedback(result);
|
||||
result.suggestedActions = this.generateSuggestedActions(gateResults, config);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a single quality gate
|
||||
*/
|
||||
async runGate(gate: QualityGate): Promise<QualityGateResult> {
|
||||
this.logger.debug(`Running gate: ${gate.name} (${gate.id})`);
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
if (!gate.command) {
|
||||
// Custom gates without commands always pass
|
||||
return {
|
||||
gateId: gate.id,
|
||||
gateName: gate.name,
|
||||
passed: true,
|
||||
duration: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
const { stdout, stderr } = await execAsync(gate.command, {
|
||||
timeout: 300000, // 5 minute timeout
|
||||
maxBuffer: 10 * 1024 * 1024, // 10MB buffer
|
||||
});
|
||||
|
||||
const output = stdout + stderr;
|
||||
let passed = true;
|
||||
|
||||
// Check expected output pattern if provided
|
||||
if (gate.expectedOutput) {
|
||||
if (typeof gate.expectedOutput === "string") {
|
||||
passed = output.includes(gate.expectedOutput);
|
||||
} else {
|
||||
// RegExp
|
||||
passed = gate.expectedOutput.test(output);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
gateId: gate.id,
|
||||
gateName: gate.name,
|
||||
passed,
|
||||
output,
|
||||
duration: Date.now() - startTime,
|
||||
};
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
return {
|
||||
gateId: gate.id,
|
||||
gateName: gate.name,
|
||||
passed: false,
|
||||
error: errorMessage,
|
||||
duration: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if continuation is needed
|
||||
*/
|
||||
shouldContinue(
|
||||
validation: CompletionValidation,
|
||||
continuationCount: number,
|
||||
config: OrchestrationConfig
|
||||
): boolean {
|
||||
// Don't continue if already accepted
|
||||
if (validation.verdict === "accepted") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Don't continue if at max continuations
|
||||
if (continuationCount >= config.maxContinuations) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate continuation prompt based on failures
|
||||
*/
|
||||
generateContinuationPrompt(validation: CompletionValidation): string {
|
||||
const failedGates = validation.gateResults.filter((r) => !r.passed);
|
||||
|
||||
let prompt = "Quality gates failed. Please address the following issues:\n\n";
|
||||
|
||||
for (const gate of failedGates) {
|
||||
prompt += `**${gate.gateName}** failed:\n`;
|
||||
if (gate.error) {
|
||||
prompt += ` Error: ${gate.error}\n`;
|
||||
}
|
||||
if (gate.output) {
|
||||
const outputPreview = gate.output.substring(0, 500);
|
||||
prompt += ` Output: ${outputPreview}\n`;
|
||||
}
|
||||
prompt += "\n";
|
||||
}
|
||||
|
||||
if (validation.suggestedActions && validation.suggestedActions.length > 0) {
|
||||
prompt += "Suggested actions:\n";
|
||||
for (const action of validation.suggestedActions) {
|
||||
prompt += `- ${action}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return prompt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate rejection feedback
|
||||
*/
|
||||
generateRejectionFeedback(validation: CompletionValidation): string {
|
||||
const failedGates = validation.gateResults.filter((r) => !r.passed);
|
||||
const failedCount = String(failedGates.length);
|
||||
|
||||
let feedback = `Task completion rejected. ${failedCount} quality gate(s) failed:\n\n`;
|
||||
|
||||
for (const gate of failedGates) {
|
||||
feedback += `- ${gate.gateName}: `;
|
||||
if (gate.error) {
|
||||
feedback += gate.error;
|
||||
} else {
|
||||
feedback += "Failed validation";
|
||||
}
|
||||
feedback += "\n";
|
||||
}
|
||||
|
||||
return feedback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate suggested actions based on gate failures
|
||||
*/
|
||||
private generateSuggestedActions(
|
||||
gateResults: QualityGateResult[],
|
||||
config: OrchestrationConfig
|
||||
): string[] {
|
||||
const actions: string[] = [];
|
||||
const failedGates = gateResults.filter((r) => !r.passed);
|
||||
|
||||
for (const result of failedGates) {
|
||||
const gate = config.gates.find((g) => g.id === result.gateId);
|
||||
if (!gate) continue;
|
||||
|
||||
switch (gate.type) {
|
||||
case "build":
|
||||
actions.push("Fix compilation errors in the code");
|
||||
actions.push("Run: pnpm build");
|
||||
break;
|
||||
case "lint":
|
||||
actions.push("Fix linting issues");
|
||||
actions.push("Run: pnpm lint --fix");
|
||||
break;
|
||||
case "test":
|
||||
actions.push("Fix failing tests");
|
||||
actions.push("Run: pnpm test");
|
||||
break;
|
||||
case "coverage":
|
||||
actions.push("Add tests to improve coverage to >= 85%");
|
||||
actions.push("Run: pnpm test:coverage");
|
||||
break;
|
||||
default:
|
||||
if (gate.command) {
|
||||
actions.push(`Run: ${gate.command}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get default gates for a workspace
|
||||
*/
|
||||
getDefaultGates(workspaceId: string): QualityGate[] {
|
||||
// For now, return the default gates
|
||||
// In the future, this could be customized per workspace from database
|
||||
this.logger.debug(`Getting default gates for workspace ${workspaceId}`);
|
||||
return DEFAULT_GATES;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track continuation attempts
|
||||
*/
|
||||
recordContinuation(taskId: string, attempt: number, validation: CompletionValidation): void {
|
||||
const attemptStr = String(attempt);
|
||||
const failedCount = String(validation.requiredGatesFailed.length);
|
||||
this.logger.log(`Recording continuation attempt ${attemptStr} for task ${taskId}`);
|
||||
|
||||
// Store continuation record
|
||||
// For now, just log it. In production, this would be stored in the database
|
||||
this.logger.debug(`Continuation ${attemptStr}: ${failedCount} required gates failed`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user