feat(#134): design Non-AI Quality Orchestrator service
Implement quality orchestration service to enforce standards on AI agent work and prevent premature completion claims. Components: - QualityOrchestratorService: Core validation and gate execution - QualityGate interface: Extensible gate definitions - CompletionClaim/Validation: Track claims and verdicts - OrchestrationConfig: Per-workspace configuration Features: - Validate completions against quality gates (build/lint/test/coverage) - Run gates with command execution and output validation - Support string and RegExp output pattern matching - Smart continuation logic with attempt tracking - Generate actionable feedback for failed gates - Strict/lenient mode for gate enforcement - 5-minute timeout, 10MB output buffer per gate Default gates: - Build Check (required) - Lint Check (required) - Test Suite (required) - Coverage Check (optional, 85% threshold) Tests: 21 passing with 85.98% coverage Fixes #134 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2
apps/api/src/quality-orchestrator/dto/index.ts
Normal file
2
apps/api/src/quality-orchestrator/dto/index.ts
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
export * from "./validate-completion.dto";
|
||||||
|
export * from "./orchestration-result.dto";
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import type { QualityGateResult } from "../interfaces";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DTO for orchestration results
|
||||||
|
*/
|
||||||
|
export interface OrchestrationResultDto {
|
||||||
|
/** Task ID */
|
||||||
|
taskId: string;
|
||||||
|
|
||||||
|
/** Whether the completion was accepted */
|
||||||
|
accepted: boolean;
|
||||||
|
|
||||||
|
/** Verdict from validation */
|
||||||
|
verdict: "accepted" | "rejected" | "needs-continuation";
|
||||||
|
|
||||||
|
/** All gates passed */
|
||||||
|
allGatesPassed: boolean;
|
||||||
|
|
||||||
|
/** Required gates that failed */
|
||||||
|
requiredGatesFailed: string[];
|
||||||
|
|
||||||
|
/** Results from each gate */
|
||||||
|
gateResults: QualityGateResult[];
|
||||||
|
|
||||||
|
/** Feedback for the agent */
|
||||||
|
feedback?: string;
|
||||||
|
|
||||||
|
/** Suggested actions to fix issues */
|
||||||
|
suggestedActions?: string[];
|
||||||
|
|
||||||
|
/** Continuation prompt if needed */
|
||||||
|
continuationPrompt?: string;
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
import { IsString, IsArray, IsDateString, IsNotEmpty, ArrayMinSize } from "class-validator";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DTO for validating a completion claim
|
||||||
|
*/
|
||||||
|
export class ValidateCompletionDto {
|
||||||
|
@IsString()
|
||||||
|
@IsNotEmpty()
|
||||||
|
taskId!: string;
|
||||||
|
|
||||||
|
@IsString()
|
||||||
|
@IsNotEmpty()
|
||||||
|
agentId!: string;
|
||||||
|
|
||||||
|
@IsString()
|
||||||
|
@IsNotEmpty()
|
||||||
|
workspaceId!: string;
|
||||||
|
|
||||||
|
@IsDateString()
|
||||||
|
claimedAt!: string;
|
||||||
|
|
||||||
|
@IsString()
|
||||||
|
@IsNotEmpty()
|
||||||
|
message!: string;
|
||||||
|
|
||||||
|
@IsArray()
|
||||||
|
@ArrayMinSize(0)
|
||||||
|
@IsString({ each: true })
|
||||||
|
filesChanged!: string[];
|
||||||
|
}
|
||||||
4
apps/api/src/quality-orchestrator/index.ts
Normal file
4
apps/api/src/quality-orchestrator/index.ts
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
export * from "./quality-orchestrator.module";
|
||||||
|
export * from "./quality-orchestrator.service";
|
||||||
|
export * from "./interfaces";
|
||||||
|
export * from "./dto";
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import type { QualityGateResult } from "./quality-gate.interface";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Claim by an agent that a task is complete
|
||||||
|
*/
|
||||||
|
export interface CompletionClaim {
|
||||||
|
/** ID of the task being claimed as complete */
|
||||||
|
taskId: string;
|
||||||
|
|
||||||
|
/** ID of the agent making the claim */
|
||||||
|
agentId: string;
|
||||||
|
|
||||||
|
/** Workspace context */
|
||||||
|
workspaceId: string;
|
||||||
|
|
||||||
|
/** Timestamp of claim */
|
||||||
|
claimedAt: Date;
|
||||||
|
|
||||||
|
/** Agent's message about completion */
|
||||||
|
message: string;
|
||||||
|
|
||||||
|
/** List of files changed during task execution */
|
||||||
|
filesChanged: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Result of validating a completion claim
|
||||||
|
*/
|
||||||
|
export interface CompletionValidation {
|
||||||
|
/** Original claim being validated */
|
||||||
|
claim: CompletionClaim;
|
||||||
|
|
||||||
|
/** Results from all quality gates */
|
||||||
|
gateResults: QualityGateResult[];
|
||||||
|
|
||||||
|
/** Whether all gates passed */
|
||||||
|
allGatesPassed: boolean;
|
||||||
|
|
||||||
|
/** List of required gates that failed */
|
||||||
|
requiredGatesFailed: string[];
|
||||||
|
|
||||||
|
/** Final verdict on the completion */
|
||||||
|
verdict: "accepted" | "rejected" | "needs-continuation";
|
||||||
|
|
||||||
|
/** Feedback message for the agent */
|
||||||
|
feedback?: string;
|
||||||
|
|
||||||
|
/** Specific actions to take to fix failures */
|
||||||
|
suggestedActions?: string[];
|
||||||
|
}
|
||||||
3
apps/api/src/quality-orchestrator/interfaces/index.ts
Normal file
3
apps/api/src/quality-orchestrator/interfaces/index.ts
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
export * from "./quality-gate.interface";
|
||||||
|
export * from "./completion-result.interface";
|
||||||
|
export * from "./orchestration-config.interface";
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import type { QualityGate } from "./quality-gate.interface";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration for quality orchestration
|
||||||
|
*/
|
||||||
|
export interface OrchestrationConfig {
|
||||||
|
/** Workspace this config applies to */
|
||||||
|
workspaceId: string;
|
||||||
|
|
||||||
|
/** Quality gates to enforce */
|
||||||
|
gates: QualityGate[];
|
||||||
|
|
||||||
|
/** Maximum number of continuation attempts */
|
||||||
|
maxContinuations: number;
|
||||||
|
|
||||||
|
/** Token budget for continuations */
|
||||||
|
continuationBudget: number;
|
||||||
|
|
||||||
|
/** Whether to reject on ANY failure vs only required gates */
|
||||||
|
strictMode: boolean;
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
/**
|
||||||
|
* Defines a quality gate that must be passed for task completion
|
||||||
|
*/
|
||||||
|
export interface QualityGate {
|
||||||
|
/** Unique identifier for the gate */
|
||||||
|
id: string;
|
||||||
|
|
||||||
|
/** Human-readable name */
|
||||||
|
name: string;
|
||||||
|
|
||||||
|
/** Description of what this gate checks */
|
||||||
|
description: string;
|
||||||
|
|
||||||
|
/** Type of quality check */
|
||||||
|
type: "test" | "lint" | "build" | "coverage" | "custom";
|
||||||
|
|
||||||
|
/** Command to execute for this gate (optional for custom gates) */
|
||||||
|
command?: string;
|
||||||
|
|
||||||
|
/** Expected output pattern (optional, for validation) */
|
||||||
|
expectedOutput?: string | RegExp;
|
||||||
|
|
||||||
|
/** Whether this gate must pass for completion */
|
||||||
|
required: boolean;
|
||||||
|
|
||||||
|
/** Execution order (lower numbers run first) */
|
||||||
|
order: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Result of running a quality gate
|
||||||
|
*/
|
||||||
|
export interface QualityGateResult {
|
||||||
|
/** ID of the gate that was run */
|
||||||
|
gateId: string;
|
||||||
|
|
||||||
|
/** Name of the gate */
|
||||||
|
gateName: string;
|
||||||
|
|
||||||
|
/** Whether the gate passed */
|
||||||
|
passed: boolean;
|
||||||
|
|
||||||
|
/** Output from running the gate */
|
||||||
|
output?: string;
|
||||||
|
|
||||||
|
/** Error message if gate failed */
|
||||||
|
error?: string;
|
||||||
|
|
||||||
|
/** Duration in milliseconds */
|
||||||
|
duration: number;
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
import { Module } from "@nestjs/common";
|
||||||
|
import { QualityOrchestratorService } from "./quality-orchestrator.service";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Quality Orchestrator Module
|
||||||
|
* Provides quality enforcement for AI agent task completions
|
||||||
|
*/
|
||||||
|
@Module({
|
||||||
|
providers: [QualityOrchestratorService],
|
||||||
|
exports: [QualityOrchestratorService],
|
||||||
|
})
|
||||||
|
export class QualityOrchestratorModule {}
|
||||||
@@ -0,0 +1,470 @@
|
|||||||
|
import { describe, it, expect, beforeEach } from "vitest";
|
||||||
|
import { Test, TestingModule } from "@nestjs/testing";
|
||||||
|
import { QualityOrchestratorService } from "./quality-orchestrator.service";
|
||||||
|
import type {
|
||||||
|
QualityGate,
|
||||||
|
CompletionClaim,
|
||||||
|
OrchestrationConfig,
|
||||||
|
CompletionValidation,
|
||||||
|
} from "./interfaces";
|
||||||
|
|
||||||
|
describe("QualityOrchestratorService", () => {
|
||||||
|
let service: QualityOrchestratorService;
|
||||||
|
|
||||||
|
const mockWorkspaceId = "workspace-1";
|
||||||
|
const mockTaskId = "task-1";
|
||||||
|
const mockAgentId = "agent-1";
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
|
providers: [QualityOrchestratorService],
|
||||||
|
}).compile();
|
||||||
|
|
||||||
|
service = module.get<QualityOrchestratorService>(QualityOrchestratorService);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should be defined", () => {
|
||||||
|
expect(service).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("validateCompletion", () => {
|
||||||
|
const claim: CompletionClaim = {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Task completed successfully",
|
||||||
|
filesChanged: ["src/test.ts"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const config: OrchestrationConfig = {
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
gates: [
|
||||||
|
{
|
||||||
|
id: "build",
|
||||||
|
name: "Build Check",
|
||||||
|
description: "Verify code compiles",
|
||||||
|
type: "build",
|
||||||
|
command: "echo 'build success'",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
maxContinuations: 3,
|
||||||
|
continuationBudget: 10000,
|
||||||
|
strictMode: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
it("should accept completion when all required gates pass", async () => {
|
||||||
|
const result = await service.validateCompletion(claim, config);
|
||||||
|
|
||||||
|
expect(result.verdict).toBe("accepted");
|
||||||
|
expect(result.allGatesPassed).toBe(true);
|
||||||
|
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should reject completion when required gates fail", async () => {
|
||||||
|
const failingConfig: OrchestrationConfig = {
|
||||||
|
...config,
|
||||||
|
gates: [
|
||||||
|
{
|
||||||
|
id: "build",
|
||||||
|
name: "Build Check",
|
||||||
|
description: "Verify code compiles",
|
||||||
|
type: "build",
|
||||||
|
command: "exit 1",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.validateCompletion(claim, failingConfig);
|
||||||
|
|
||||||
|
expect(result.verdict).toBe("rejected");
|
||||||
|
expect(result.allGatesPassed).toBe(false);
|
||||||
|
expect(result.requiredGatesFailed).toContain("build");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should accept when optional gates fail but required gates pass", async () => {
|
||||||
|
const mixedConfig: OrchestrationConfig = {
|
||||||
|
...config,
|
||||||
|
gates: [
|
||||||
|
{
|
||||||
|
id: "build",
|
||||||
|
name: "Build Check",
|
||||||
|
description: "Verify code compiles",
|
||||||
|
type: "build",
|
||||||
|
command: "echo 'success'",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "coverage",
|
||||||
|
name: "Coverage Check",
|
||||||
|
description: "Check coverage",
|
||||||
|
type: "coverage",
|
||||||
|
command: "exit 1",
|
||||||
|
required: false,
|
||||||
|
order: 2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.validateCompletion(claim, mixedConfig);
|
||||||
|
|
||||||
|
expect(result.verdict).toBe("accepted");
|
||||||
|
expect(result.allGatesPassed).toBe(false);
|
||||||
|
expect(result.requiredGatesFailed).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should provide feedback when gates fail", async () => {
|
||||||
|
const failingConfig: OrchestrationConfig = {
|
||||||
|
...config,
|
||||||
|
gates: [
|
||||||
|
{
|
||||||
|
id: "test",
|
||||||
|
name: "Test Suite",
|
||||||
|
description: "Run tests",
|
||||||
|
type: "test",
|
||||||
|
command: "exit 1",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.validateCompletion(claim, failingConfig);
|
||||||
|
|
||||||
|
expect(result.feedback).toBeDefined();
|
||||||
|
expect(result.suggestedActions).toBeDefined();
|
||||||
|
expect(result.suggestedActions!.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should run gates in order", async () => {
|
||||||
|
const orderedConfig: OrchestrationConfig = {
|
||||||
|
...config,
|
||||||
|
gates: [
|
||||||
|
{
|
||||||
|
id: "gate3",
|
||||||
|
name: "Third Gate",
|
||||||
|
description: "Third",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'third'",
|
||||||
|
required: true,
|
||||||
|
order: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "gate1",
|
||||||
|
name: "First Gate",
|
||||||
|
description: "First",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'first'",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "gate2",
|
||||||
|
name: "Second Gate",
|
||||||
|
description: "Second",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'second'",
|
||||||
|
required: true,
|
||||||
|
order: 2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.validateCompletion(claim, orderedConfig);
|
||||||
|
|
||||||
|
expect(result.gateResults[0].gateId).toBe("gate1");
|
||||||
|
expect(result.gateResults[1].gateId).toBe("gate2");
|
||||||
|
expect(result.gateResults[2].gateId).toBe("gate3");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("runGate", () => {
|
||||||
|
it("should successfully run a gate with passing command", async () => {
|
||||||
|
const gate: QualityGate = {
|
||||||
|
id: "test-gate",
|
||||||
|
name: "Test Gate",
|
||||||
|
description: "Test description",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'success'",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.runGate(gate);
|
||||||
|
|
||||||
|
expect(result.gateId).toBe("test-gate");
|
||||||
|
expect(result.gateName).toBe("Test Gate");
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
expect(result.duration).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should fail a gate with failing command", async () => {
|
||||||
|
const gate: QualityGate = {
|
||||||
|
id: "fail-gate",
|
||||||
|
name: "Failing Gate",
|
||||||
|
description: "Should fail",
|
||||||
|
type: "custom",
|
||||||
|
command: "exit 1",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.runGate(gate);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
expect(result.error).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should capture output from gate execution", async () => {
|
||||||
|
const gate: QualityGate = {
|
||||||
|
id: "output-gate",
|
||||||
|
name: "Output Gate",
|
||||||
|
description: "Captures output",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'test output'",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.runGate(gate);
|
||||||
|
|
||||||
|
expect(result.output).toContain("test output");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should validate expected output pattern", async () => {
|
||||||
|
const gate: QualityGate = {
|
||||||
|
id: "pattern-gate",
|
||||||
|
name: "Pattern Gate",
|
||||||
|
description: "Checks output pattern",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'coverage: 90%'",
|
||||||
|
expectedOutput: /coverage: \d+%/,
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.runGate(gate);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should fail when expected output pattern does not match", async () => {
|
||||||
|
const gate: QualityGate = {
|
||||||
|
id: "bad-pattern-gate",
|
||||||
|
name: "Bad Pattern Gate",
|
||||||
|
description: "Pattern should not match",
|
||||||
|
type: "custom",
|
||||||
|
command: "echo 'no coverage info'",
|
||||||
|
expectedOutput: /coverage: \d+%/,
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await service.runGate(gate);
|
||||||
|
|
||||||
|
expect(result.passed).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("shouldContinue", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["test"],
|
||||||
|
verdict: "needs-continuation",
|
||||||
|
};
|
||||||
|
|
||||||
|
const config: OrchestrationConfig = {
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
gates: [],
|
||||||
|
maxContinuations: 3,
|
||||||
|
continuationBudget: 10000,
|
||||||
|
strictMode: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
it("should continue when under max continuations", () => {
|
||||||
|
const result = service.shouldContinue(validation, 1, config);
|
||||||
|
expect(result).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should not continue when at max continuations", () => {
|
||||||
|
const result = service.shouldContinue(validation, 3, config);
|
||||||
|
expect(result).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should not continue when validation is accepted", () => {
|
||||||
|
const acceptedValidation: CompletionValidation = {
|
||||||
|
...validation,
|
||||||
|
verdict: "accepted",
|
||||||
|
allGatesPassed: true,
|
||||||
|
requiredGatesFailed: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = service.shouldContinue(acceptedValidation, 1, config);
|
||||||
|
expect(result).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("generateContinuationPrompt", () => {
|
||||||
|
it("should generate prompt with failed gate information", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [
|
||||||
|
{
|
||||||
|
gateId: "test",
|
||||||
|
gateName: "Test Suite",
|
||||||
|
passed: false,
|
||||||
|
error: "Tests failed",
|
||||||
|
duration: 1000,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["test"],
|
||||||
|
verdict: "needs-continuation",
|
||||||
|
};
|
||||||
|
|
||||||
|
const prompt = service.generateContinuationPrompt(validation);
|
||||||
|
|
||||||
|
expect(prompt).toContain("Test Suite");
|
||||||
|
expect(prompt).toContain("failed");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should include suggested actions in prompt", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["lint"],
|
||||||
|
verdict: "needs-continuation",
|
||||||
|
suggestedActions: ["Run: pnpm lint --fix", "Check code style"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const prompt = service.generateContinuationPrompt(validation);
|
||||||
|
|
||||||
|
expect(prompt).toContain("pnpm lint --fix");
|
||||||
|
expect(prompt).toContain("Check code style");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("generateRejectionFeedback", () => {
|
||||||
|
it("should generate detailed rejection feedback", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [
|
||||||
|
{
|
||||||
|
gateId: "build",
|
||||||
|
gateName: "Build Check",
|
||||||
|
passed: false,
|
||||||
|
error: "Compilation error",
|
||||||
|
duration: 500,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["build"],
|
||||||
|
verdict: "rejected",
|
||||||
|
};
|
||||||
|
|
||||||
|
const feedback = service.generateRejectionFeedback(validation);
|
||||||
|
|
||||||
|
expect(feedback).toContain("rejected");
|
||||||
|
expect(feedback).toContain("Build Check");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("getDefaultGates", () => {
|
||||||
|
it("should return default gates for workspace", () => {
|
||||||
|
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||||
|
|
||||||
|
expect(gates).toBeDefined();
|
||||||
|
expect(gates.length).toBeGreaterThan(0);
|
||||||
|
expect(gates.some((g) => g.id === "build")).toBe(true);
|
||||||
|
expect(gates.some((g) => g.id === "lint")).toBe(true);
|
||||||
|
expect(gates.some((g) => g.id === "test")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should return gates in correct order", () => {
|
||||||
|
const gates = service.getDefaultGates(mockWorkspaceId);
|
||||||
|
|
||||||
|
for (let i = 1; i < gates.length; i++) {
|
||||||
|
expect(gates[i].order).toBeGreaterThanOrEqual(gates[i - 1].order);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("recordContinuation", () => {
|
||||||
|
it("should record continuation attempt", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["test"],
|
||||||
|
verdict: "needs-continuation",
|
||||||
|
};
|
||||||
|
|
||||||
|
expect(() => service.recordContinuation(mockTaskId, 1, validation)).not.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle multiple continuation records", () => {
|
||||||
|
const validation: CompletionValidation = {
|
||||||
|
claim: {
|
||||||
|
taskId: mockTaskId,
|
||||||
|
agentId: mockAgentId,
|
||||||
|
workspaceId: mockWorkspaceId,
|
||||||
|
claimedAt: new Date(),
|
||||||
|
message: "Done",
|
||||||
|
filesChanged: [],
|
||||||
|
},
|
||||||
|
gateResults: [],
|
||||||
|
allGatesPassed: false,
|
||||||
|
requiredGatesFailed: ["test"],
|
||||||
|
verdict: "needs-continuation",
|
||||||
|
};
|
||||||
|
|
||||||
|
service.recordContinuation(mockTaskId, 1, validation);
|
||||||
|
service.recordContinuation(mockTaskId, 2, validation);
|
||||||
|
|
||||||
|
expect(() => service.recordContinuation(mockTaskId, 3, validation)).not.toThrow();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,316 @@
|
|||||||
|
import { Injectable, Logger } from "@nestjs/common";
|
||||||
|
import { exec } from "child_process";
|
||||||
|
import { promisify } from "util";
|
||||||
|
import type {
|
||||||
|
QualityGate,
|
||||||
|
QualityGateResult,
|
||||||
|
CompletionClaim,
|
||||||
|
CompletionValidation,
|
||||||
|
OrchestrationConfig,
|
||||||
|
} from "./interfaces";
|
||||||
|
|
||||||
|
const execAsync = promisify(exec);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default quality gates for all workspaces
|
||||||
|
*/
|
||||||
|
const DEFAULT_GATES: QualityGate[] = [
|
||||||
|
{
|
||||||
|
id: "build",
|
||||||
|
name: "Build Check",
|
||||||
|
description: "Verify code compiles without errors",
|
||||||
|
type: "build",
|
||||||
|
command: "pnpm build",
|
||||||
|
required: true,
|
||||||
|
order: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "lint",
|
||||||
|
name: "Lint Check",
|
||||||
|
description: "Code follows style guidelines",
|
||||||
|
type: "lint",
|
||||||
|
command: "pnpm lint",
|
||||||
|
required: true,
|
||||||
|
order: 2,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "test",
|
||||||
|
name: "Test Suite",
|
||||||
|
description: "All tests pass",
|
||||||
|
type: "test",
|
||||||
|
command: "pnpm test",
|
||||||
|
required: true,
|
||||||
|
order: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "coverage",
|
||||||
|
name: "Coverage Check",
|
||||||
|
description: "Test coverage >= 85%",
|
||||||
|
type: "coverage",
|
||||||
|
command: "pnpm test:coverage",
|
||||||
|
expectedOutput: /All files.*[89]\d|100/,
|
||||||
|
required: false,
|
||||||
|
order: 4,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Quality Orchestrator Service
|
||||||
|
* Validates AI agent task completions and enforces quality gates
|
||||||
|
*/
|
||||||
|
@Injectable()
|
||||||
|
export class QualityOrchestratorService {
|
||||||
|
private readonly logger = new Logger(QualityOrchestratorService.name);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate a completion claim against quality gates
|
||||||
|
*/
|
||||||
|
async validateCompletion(
|
||||||
|
claim: CompletionClaim,
|
||||||
|
config: OrchestrationConfig
|
||||||
|
): Promise<CompletionValidation> {
|
||||||
|
this.logger.log(
|
||||||
|
`Validating completion claim for task ${claim.taskId} by agent ${claim.agentId}`
|
||||||
|
);
|
||||||
|
|
||||||
|
// Sort gates by order
|
||||||
|
const sortedGates = [...config.gates].sort((a, b) => a.order - b.order);
|
||||||
|
|
||||||
|
// Run all gates
|
||||||
|
const gateResults: QualityGateResult[] = [];
|
||||||
|
for (const gate of sortedGates) {
|
||||||
|
const result = await this.runGate(gate);
|
||||||
|
gateResults.push(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze results
|
||||||
|
const allGatesPassed = gateResults.every((r) => r.passed);
|
||||||
|
const requiredGatesFailed = gateResults
|
||||||
|
.filter((r) => !r.passed)
|
||||||
|
.map((r) => r.gateId)
|
||||||
|
.filter((id) => {
|
||||||
|
const gate = config.gates.find((g) => g.id === id);
|
||||||
|
return gate?.required ?? false;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Determine verdict
|
||||||
|
let verdict: "accepted" | "rejected" | "needs-continuation";
|
||||||
|
if (allGatesPassed) {
|
||||||
|
verdict = "accepted";
|
||||||
|
} else if (requiredGatesFailed.length > 0) {
|
||||||
|
verdict = "rejected";
|
||||||
|
} else if (config.strictMode) {
|
||||||
|
verdict = "rejected";
|
||||||
|
} else {
|
||||||
|
verdict = "accepted";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate feedback and suggestions
|
||||||
|
const result: CompletionValidation = {
|
||||||
|
claim,
|
||||||
|
gateResults,
|
||||||
|
allGatesPassed,
|
||||||
|
requiredGatesFailed,
|
||||||
|
verdict,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (verdict !== "accepted") {
|
||||||
|
result.feedback = this.generateRejectionFeedback(result);
|
||||||
|
result.suggestedActions = this.generateSuggestedActions(gateResults, config);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a single quality gate
|
||||||
|
*/
|
||||||
|
async runGate(gate: QualityGate): Promise<QualityGateResult> {
|
||||||
|
this.logger.debug(`Running gate: ${gate.name} (${gate.id})`);
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!gate.command) {
|
||||||
|
// Custom gates without commands always pass
|
||||||
|
return {
|
||||||
|
gateId: gate.id,
|
||||||
|
gateName: gate.name,
|
||||||
|
passed: true,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const { stdout, stderr } = await execAsync(gate.command, {
|
||||||
|
timeout: 300000, // 5 minute timeout
|
||||||
|
maxBuffer: 10 * 1024 * 1024, // 10MB buffer
|
||||||
|
});
|
||||||
|
|
||||||
|
const output = stdout + stderr;
|
||||||
|
let passed = true;
|
||||||
|
|
||||||
|
// Check expected output pattern if provided
|
||||||
|
if (gate.expectedOutput) {
|
||||||
|
if (typeof gate.expectedOutput === "string") {
|
||||||
|
passed = output.includes(gate.expectedOutput);
|
||||||
|
} else {
|
||||||
|
// RegExp
|
||||||
|
passed = gate.expectedOutput.test(output);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
gateId: gate.id,
|
||||||
|
gateName: gate.name,
|
||||||
|
passed,
|
||||||
|
output,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
|
return {
|
||||||
|
gateId: gate.id,
|
||||||
|
gateName: gate.name,
|
||||||
|
passed: false,
|
||||||
|
error: errorMessage,
|
||||||
|
duration: Date.now() - startTime,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if continuation is needed
|
||||||
|
*/
|
||||||
|
shouldContinue(
|
||||||
|
validation: CompletionValidation,
|
||||||
|
continuationCount: number,
|
||||||
|
config: OrchestrationConfig
|
||||||
|
): boolean {
|
||||||
|
// Don't continue if already accepted
|
||||||
|
if (validation.verdict === "accepted") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Don't continue if at max continuations
|
||||||
|
if (continuationCount >= config.maxContinuations) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate continuation prompt based on failures
|
||||||
|
*/
|
||||||
|
generateContinuationPrompt(validation: CompletionValidation): string {
|
||||||
|
const failedGates = validation.gateResults.filter((r) => !r.passed);
|
||||||
|
|
||||||
|
let prompt = "Quality gates failed. Please address the following issues:\n\n";
|
||||||
|
|
||||||
|
for (const gate of failedGates) {
|
||||||
|
prompt += `**${gate.gateName}** failed:\n`;
|
||||||
|
if (gate.error) {
|
||||||
|
prompt += ` Error: ${gate.error}\n`;
|
||||||
|
}
|
||||||
|
if (gate.output) {
|
||||||
|
const outputPreview = gate.output.substring(0, 500);
|
||||||
|
prompt += ` Output: ${outputPreview}\n`;
|
||||||
|
}
|
||||||
|
prompt += "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (validation.suggestedActions && validation.suggestedActions.length > 0) {
|
||||||
|
prompt += "Suggested actions:\n";
|
||||||
|
for (const action of validation.suggestedActions) {
|
||||||
|
prompt += `- ${action}\n`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return prompt;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate rejection feedback
|
||||||
|
*/
|
||||||
|
generateRejectionFeedback(validation: CompletionValidation): string {
|
||||||
|
const failedGates = validation.gateResults.filter((r) => !r.passed);
|
||||||
|
const failedCount = String(failedGates.length);
|
||||||
|
|
||||||
|
let feedback = `Task completion rejected. ${failedCount} quality gate(s) failed:\n\n`;
|
||||||
|
|
||||||
|
for (const gate of failedGates) {
|
||||||
|
feedback += `- ${gate.gateName}: `;
|
||||||
|
if (gate.error) {
|
||||||
|
feedback += gate.error;
|
||||||
|
} else {
|
||||||
|
feedback += "Failed validation";
|
||||||
|
}
|
||||||
|
feedback += "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return feedback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate suggested actions based on gate failures
|
||||||
|
*/
|
||||||
|
private generateSuggestedActions(
|
||||||
|
gateResults: QualityGateResult[],
|
||||||
|
config: OrchestrationConfig
|
||||||
|
): string[] {
|
||||||
|
const actions: string[] = [];
|
||||||
|
const failedGates = gateResults.filter((r) => !r.passed);
|
||||||
|
|
||||||
|
for (const result of failedGates) {
|
||||||
|
const gate = config.gates.find((g) => g.id === result.gateId);
|
||||||
|
if (!gate) continue;
|
||||||
|
|
||||||
|
switch (gate.type) {
|
||||||
|
case "build":
|
||||||
|
actions.push("Fix compilation errors in the code");
|
||||||
|
actions.push("Run: pnpm build");
|
||||||
|
break;
|
||||||
|
case "lint":
|
||||||
|
actions.push("Fix linting issues");
|
||||||
|
actions.push("Run: pnpm lint --fix");
|
||||||
|
break;
|
||||||
|
case "test":
|
||||||
|
actions.push("Fix failing tests");
|
||||||
|
actions.push("Run: pnpm test");
|
||||||
|
break;
|
||||||
|
case "coverage":
|
||||||
|
actions.push("Add tests to improve coverage to >= 85%");
|
||||||
|
actions.push("Run: pnpm test:coverage");
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (gate.command) {
|
||||||
|
actions.push(`Run: ${gate.command}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return actions;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get default gates for a workspace
|
||||||
|
*/
|
||||||
|
getDefaultGates(workspaceId: string): QualityGate[] {
|
||||||
|
// For now, return the default gates
|
||||||
|
// In the future, this could be customized per workspace from database
|
||||||
|
this.logger.debug(`Getting default gates for workspace ${workspaceId}`);
|
||||||
|
return DEFAULT_GATES;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Track continuation attempts
|
||||||
|
*/
|
||||||
|
recordContinuation(taskId: string, attempt: number, validation: CompletionValidation): void {
|
||||||
|
const attemptStr = String(attempt);
|
||||||
|
const failedCount = String(validation.requiredGatesFailed.length);
|
||||||
|
this.logger.log(`Recording continuation attempt ${attemptStr} for task ${taskId}`);
|
||||||
|
|
||||||
|
// Store continuation record
|
||||||
|
// For now, just log it. In production, this would be stored in the database
|
||||||
|
this.logger.debug(`Continuation ${attemptStr}: ${failedCount} required gates failed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user