Release: CI/CD Pipeline & Architecture Updates #177

Merged
jason.woltje merged 173 commits from develop into main 2026-02-01 19:18:48 +00:00
11 changed files with 992 additions and 0 deletions
Showing only changes of commit a25e9048be - Show all commits

View File

@@ -0,0 +1,2 @@
export * from "./validate-completion.dto";
export * from "./orchestration-result.dto";

View File

@@ -0,0 +1,33 @@
import type { QualityGateResult } from "../interfaces";
/**
* DTO for orchestration results
*/
export interface OrchestrationResultDto {
/** Task ID */
taskId: string;
/** Whether the completion was accepted */
accepted: boolean;
/** Verdict from validation */
verdict: "accepted" | "rejected" | "needs-continuation";
/** All gates passed */
allGatesPassed: boolean;
/** Required gates that failed */
requiredGatesFailed: string[];
/** Results from each gate */
gateResults: QualityGateResult[];
/** Feedback for the agent */
feedback?: string;
/** Suggested actions to fix issues */
suggestedActions?: string[];
/** Continuation prompt if needed */
continuationPrompt?: string;
}

View File

@@ -0,0 +1,30 @@
import { IsString, IsArray, IsDateString, IsNotEmpty, ArrayMinSize } from "class-validator";
/**
* DTO for validating a completion claim
*/
export class ValidateCompletionDto {
@IsString()
@IsNotEmpty()
taskId!: string;
@IsString()
@IsNotEmpty()
agentId!: string;
@IsString()
@IsNotEmpty()
workspaceId!: string;
@IsDateString()
claimedAt!: string;
@IsString()
@IsNotEmpty()
message!: string;
@IsArray()
@ArrayMinSize(0)
@IsString({ each: true })
filesChanged!: string[];
}

View File

@@ -0,0 +1,4 @@
export * from "./quality-orchestrator.module";
export * from "./quality-orchestrator.service";
export * from "./interfaces";
export * from "./dto";

View File

@@ -0,0 +1,50 @@
import type { QualityGateResult } from "./quality-gate.interface";
/**
* Claim by an agent that a task is complete
*/
export interface CompletionClaim {
/** ID of the task being claimed as complete */
taskId: string;
/** ID of the agent making the claim */
agentId: string;
/** Workspace context */
workspaceId: string;
/** Timestamp of claim */
claimedAt: Date;
/** Agent's message about completion */
message: string;
/** List of files changed during task execution */
filesChanged: string[];
}
/**
* Result of validating a completion claim
*/
export interface CompletionValidation {
/** Original claim being validated */
claim: CompletionClaim;
/** Results from all quality gates */
gateResults: QualityGateResult[];
/** Whether all gates passed */
allGatesPassed: boolean;
/** List of required gates that failed */
requiredGatesFailed: string[];
/** Final verdict on the completion */
verdict: "accepted" | "rejected" | "needs-continuation";
/** Feedback message for the agent */
feedback?: string;
/** Specific actions to take to fix failures */
suggestedActions?: string[];
}

View File

@@ -0,0 +1,3 @@
export * from "./quality-gate.interface";
export * from "./completion-result.interface";
export * from "./orchestration-config.interface";

View File

@@ -0,0 +1,21 @@
import type { QualityGate } from "./quality-gate.interface";
/**
* Configuration for quality orchestration
*/
export interface OrchestrationConfig {
/** Workspace this config applies to */
workspaceId: string;
/** Quality gates to enforce */
gates: QualityGate[];
/** Maximum number of continuation attempts */
maxContinuations: number;
/** Token budget for continuations */
continuationBudget: number;
/** Whether to reject on ANY failure vs only required gates */
strictMode: boolean;
}

View File

@@ -0,0 +1,51 @@
/**
* Defines a quality gate that must be passed for task completion
*/
export interface QualityGate {
/** Unique identifier for the gate */
id: string;
/** Human-readable name */
name: string;
/** Description of what this gate checks */
description: string;
/** Type of quality check */
type: "test" | "lint" | "build" | "coverage" | "custom";
/** Command to execute for this gate (optional for custom gates) */
command?: string;
/** Expected output pattern (optional, for validation) */
expectedOutput?: string | RegExp;
/** Whether this gate must pass for completion */
required: boolean;
/** Execution order (lower numbers run first) */
order: number;
}
/**
* Result of running a quality gate
*/
export interface QualityGateResult {
/** ID of the gate that was run */
gateId: string;
/** Name of the gate */
gateName: string;
/** Whether the gate passed */
passed: boolean;
/** Output from running the gate */
output?: string;
/** Error message if gate failed */
error?: string;
/** Duration in milliseconds */
duration: number;
}

View File

@@ -0,0 +1,12 @@
import { Module } from "@nestjs/common";
import { QualityOrchestratorService } from "./quality-orchestrator.service";
/**
* Quality Orchestrator Module
* Provides quality enforcement for AI agent task completions
*/
@Module({
providers: [QualityOrchestratorService],
exports: [QualityOrchestratorService],
})
export class QualityOrchestratorModule {}

View File

@@ -0,0 +1,470 @@
import { describe, it, expect, beforeEach } from "vitest";
import { Test, TestingModule } from "@nestjs/testing";
import { QualityOrchestratorService } from "./quality-orchestrator.service";
import type {
QualityGate,
CompletionClaim,
OrchestrationConfig,
CompletionValidation,
} from "./interfaces";
describe("QualityOrchestratorService", () => {
let service: QualityOrchestratorService;
const mockWorkspaceId = "workspace-1";
const mockTaskId = "task-1";
const mockAgentId = "agent-1";
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [QualityOrchestratorService],
}).compile();
service = module.get<QualityOrchestratorService>(QualityOrchestratorService);
});
it("should be defined", () => {
expect(service).toBeDefined();
});
describe("validateCompletion", () => {
const claim: CompletionClaim = {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Task completed successfully",
filesChanged: ["src/test.ts"],
};
const config: OrchestrationConfig = {
workspaceId: mockWorkspaceId,
gates: [
{
id: "build",
name: "Build Check",
description: "Verify code compiles",
type: "build",
command: "echo 'build success'",
required: true,
order: 1,
},
],
maxContinuations: 3,
continuationBudget: 10000,
strictMode: false,
};
it("should accept completion when all required gates pass", async () => {
const result = await service.validateCompletion(claim, config);
expect(result.verdict).toBe("accepted");
expect(result.allGatesPassed).toBe(true);
expect(result.requiredGatesFailed).toHaveLength(0);
});
it("should reject completion when required gates fail", async () => {
const failingConfig: OrchestrationConfig = {
...config,
gates: [
{
id: "build",
name: "Build Check",
description: "Verify code compiles",
type: "build",
command: "exit 1",
required: true,
order: 1,
},
],
};
const result = await service.validateCompletion(claim, failingConfig);
expect(result.verdict).toBe("rejected");
expect(result.allGatesPassed).toBe(false);
expect(result.requiredGatesFailed).toContain("build");
});
it("should accept when optional gates fail but required gates pass", async () => {
const mixedConfig: OrchestrationConfig = {
...config,
gates: [
{
id: "build",
name: "Build Check",
description: "Verify code compiles",
type: "build",
command: "echo 'success'",
required: true,
order: 1,
},
{
id: "coverage",
name: "Coverage Check",
description: "Check coverage",
type: "coverage",
command: "exit 1",
required: false,
order: 2,
},
],
};
const result = await service.validateCompletion(claim, mixedConfig);
expect(result.verdict).toBe("accepted");
expect(result.allGatesPassed).toBe(false);
expect(result.requiredGatesFailed).toHaveLength(0);
});
it("should provide feedback when gates fail", async () => {
const failingConfig: OrchestrationConfig = {
...config,
gates: [
{
id: "test",
name: "Test Suite",
description: "Run tests",
type: "test",
command: "exit 1",
required: true,
order: 1,
},
],
};
const result = await service.validateCompletion(claim, failingConfig);
expect(result.feedback).toBeDefined();
expect(result.suggestedActions).toBeDefined();
expect(result.suggestedActions!.length).toBeGreaterThan(0);
});
it("should run gates in order", async () => {
const orderedConfig: OrchestrationConfig = {
...config,
gates: [
{
id: "gate3",
name: "Third Gate",
description: "Third",
type: "custom",
command: "echo 'third'",
required: true,
order: 3,
},
{
id: "gate1",
name: "First Gate",
description: "First",
type: "custom",
command: "echo 'first'",
required: true,
order: 1,
},
{
id: "gate2",
name: "Second Gate",
description: "Second",
type: "custom",
command: "echo 'second'",
required: true,
order: 2,
},
],
};
const result = await service.validateCompletion(claim, orderedConfig);
expect(result.gateResults[0].gateId).toBe("gate1");
expect(result.gateResults[1].gateId).toBe("gate2");
expect(result.gateResults[2].gateId).toBe("gate3");
});
});
describe("runGate", () => {
it("should successfully run a gate with passing command", async () => {
const gate: QualityGate = {
id: "test-gate",
name: "Test Gate",
description: "Test description",
type: "custom",
command: "echo 'success'",
required: true,
order: 1,
};
const result = await service.runGate(gate);
expect(result.gateId).toBe("test-gate");
expect(result.gateName).toBe("Test Gate");
expect(result.passed).toBe(true);
expect(result.duration).toBeGreaterThan(0);
});
it("should fail a gate with failing command", async () => {
const gate: QualityGate = {
id: "fail-gate",
name: "Failing Gate",
description: "Should fail",
type: "custom",
command: "exit 1",
required: true,
order: 1,
};
const result = await service.runGate(gate);
expect(result.passed).toBe(false);
expect(result.error).toBeDefined();
});
it("should capture output from gate execution", async () => {
const gate: QualityGate = {
id: "output-gate",
name: "Output Gate",
description: "Captures output",
type: "custom",
command: "echo 'test output'",
required: true,
order: 1,
};
const result = await service.runGate(gate);
expect(result.output).toContain("test output");
});
it("should validate expected output pattern", async () => {
const gate: QualityGate = {
id: "pattern-gate",
name: "Pattern Gate",
description: "Checks output pattern",
type: "custom",
command: "echo 'coverage: 90%'",
expectedOutput: /coverage: \d+%/,
required: true,
order: 1,
};
const result = await service.runGate(gate);
expect(result.passed).toBe(true);
});
it("should fail when expected output pattern does not match", async () => {
const gate: QualityGate = {
id: "bad-pattern-gate",
name: "Bad Pattern Gate",
description: "Pattern should not match",
type: "custom",
command: "echo 'no coverage info'",
expectedOutput: /coverage: \d+%/,
required: true,
order: 1,
};
const result = await service.runGate(gate);
expect(result.passed).toBe(false);
});
});
describe("shouldContinue", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [],
allGatesPassed: false,
requiredGatesFailed: ["test"],
verdict: "needs-continuation",
};
const config: OrchestrationConfig = {
workspaceId: mockWorkspaceId,
gates: [],
maxContinuations: 3,
continuationBudget: 10000,
strictMode: false,
};
it("should continue when under max continuations", () => {
const result = service.shouldContinue(validation, 1, config);
expect(result).toBe(true);
});
it("should not continue when at max continuations", () => {
const result = service.shouldContinue(validation, 3, config);
expect(result).toBe(false);
});
it("should not continue when validation is accepted", () => {
const acceptedValidation: CompletionValidation = {
...validation,
verdict: "accepted",
allGatesPassed: true,
requiredGatesFailed: [],
};
const result = service.shouldContinue(acceptedValidation, 1, config);
expect(result).toBe(false);
});
});
describe("generateContinuationPrompt", () => {
it("should generate prompt with failed gate information", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [
{
gateId: "test",
gateName: "Test Suite",
passed: false,
error: "Tests failed",
duration: 1000,
},
],
allGatesPassed: false,
requiredGatesFailed: ["test"],
verdict: "needs-continuation",
};
const prompt = service.generateContinuationPrompt(validation);
expect(prompt).toContain("Test Suite");
expect(prompt).toContain("failed");
});
it("should include suggested actions in prompt", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [],
allGatesPassed: false,
requiredGatesFailed: ["lint"],
verdict: "needs-continuation",
suggestedActions: ["Run: pnpm lint --fix", "Check code style"],
};
const prompt = service.generateContinuationPrompt(validation);
expect(prompt).toContain("pnpm lint --fix");
expect(prompt).toContain("Check code style");
});
});
describe("generateRejectionFeedback", () => {
it("should generate detailed rejection feedback", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [
{
gateId: "build",
gateName: "Build Check",
passed: false,
error: "Compilation error",
duration: 500,
},
],
allGatesPassed: false,
requiredGatesFailed: ["build"],
verdict: "rejected",
};
const feedback = service.generateRejectionFeedback(validation);
expect(feedback).toContain("rejected");
expect(feedback).toContain("Build Check");
});
});
describe("getDefaultGates", () => {
it("should return default gates for workspace", () => {
const gates = service.getDefaultGates(mockWorkspaceId);
expect(gates).toBeDefined();
expect(gates.length).toBeGreaterThan(0);
expect(gates.some((g) => g.id === "build")).toBe(true);
expect(gates.some((g) => g.id === "lint")).toBe(true);
expect(gates.some((g) => g.id === "test")).toBe(true);
});
it("should return gates in correct order", () => {
const gates = service.getDefaultGates(mockWorkspaceId);
for (let i = 1; i < gates.length; i++) {
expect(gates[i].order).toBeGreaterThanOrEqual(gates[i - 1].order);
}
});
});
describe("recordContinuation", () => {
it("should record continuation attempt", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [],
allGatesPassed: false,
requiredGatesFailed: ["test"],
verdict: "needs-continuation",
};
expect(() => service.recordContinuation(mockTaskId, 1, validation)).not.toThrow();
});
it("should handle multiple continuation records", () => {
const validation: CompletionValidation = {
claim: {
taskId: mockTaskId,
agentId: mockAgentId,
workspaceId: mockWorkspaceId,
claimedAt: new Date(),
message: "Done",
filesChanged: [],
},
gateResults: [],
allGatesPassed: false,
requiredGatesFailed: ["test"],
verdict: "needs-continuation",
};
service.recordContinuation(mockTaskId, 1, validation);
service.recordContinuation(mockTaskId, 2, validation);
expect(() => service.recordContinuation(mockTaskId, 3, validation)).not.toThrow();
});
});
});

View File

@@ -0,0 +1,316 @@
import { Injectable, Logger } from "@nestjs/common";
import { exec } from "child_process";
import { promisify } from "util";
import type {
QualityGate,
QualityGateResult,
CompletionClaim,
CompletionValidation,
OrchestrationConfig,
} from "./interfaces";
const execAsync = promisify(exec);
/**
* Default quality gates for all workspaces
*/
const DEFAULT_GATES: QualityGate[] = [
{
id: "build",
name: "Build Check",
description: "Verify code compiles without errors",
type: "build",
command: "pnpm build",
required: true,
order: 1,
},
{
id: "lint",
name: "Lint Check",
description: "Code follows style guidelines",
type: "lint",
command: "pnpm lint",
required: true,
order: 2,
},
{
id: "test",
name: "Test Suite",
description: "All tests pass",
type: "test",
command: "pnpm test",
required: true,
order: 3,
},
{
id: "coverage",
name: "Coverage Check",
description: "Test coverage >= 85%",
type: "coverage",
command: "pnpm test:coverage",
expectedOutput: /All files.*[89]\d|100/,
required: false,
order: 4,
},
];
/**
* Quality Orchestrator Service
* Validates AI agent task completions and enforces quality gates
*/
@Injectable()
export class QualityOrchestratorService {
private readonly logger = new Logger(QualityOrchestratorService.name);
/**
* Validate a completion claim against quality gates
*/
async validateCompletion(
claim: CompletionClaim,
config: OrchestrationConfig
): Promise<CompletionValidation> {
this.logger.log(
`Validating completion claim for task ${claim.taskId} by agent ${claim.agentId}`
);
// Sort gates by order
const sortedGates = [...config.gates].sort((a, b) => a.order - b.order);
// Run all gates
const gateResults: QualityGateResult[] = [];
for (const gate of sortedGates) {
const result = await this.runGate(gate);
gateResults.push(result);
}
// Analyze results
const allGatesPassed = gateResults.every((r) => r.passed);
const requiredGatesFailed = gateResults
.filter((r) => !r.passed)
.map((r) => r.gateId)
.filter((id) => {
const gate = config.gates.find((g) => g.id === id);
return gate?.required ?? false;
});
// Determine verdict
let verdict: "accepted" | "rejected" | "needs-continuation";
if (allGatesPassed) {
verdict = "accepted";
} else if (requiredGatesFailed.length > 0) {
verdict = "rejected";
} else if (config.strictMode) {
verdict = "rejected";
} else {
verdict = "accepted";
}
// Generate feedback and suggestions
const result: CompletionValidation = {
claim,
gateResults,
allGatesPassed,
requiredGatesFailed,
verdict,
};
if (verdict !== "accepted") {
result.feedback = this.generateRejectionFeedback(result);
result.suggestedActions = this.generateSuggestedActions(gateResults, config);
}
return result;
}
/**
* Run a single quality gate
*/
async runGate(gate: QualityGate): Promise<QualityGateResult> {
this.logger.debug(`Running gate: ${gate.name} (${gate.id})`);
const startTime = Date.now();
try {
if (!gate.command) {
// Custom gates without commands always pass
return {
gateId: gate.id,
gateName: gate.name,
passed: true,
duration: Date.now() - startTime,
};
}
const { stdout, stderr } = await execAsync(gate.command, {
timeout: 300000, // 5 minute timeout
maxBuffer: 10 * 1024 * 1024, // 10MB buffer
});
const output = stdout + stderr;
let passed = true;
// Check expected output pattern if provided
if (gate.expectedOutput) {
if (typeof gate.expectedOutput === "string") {
passed = output.includes(gate.expectedOutput);
} else {
// RegExp
passed = gate.expectedOutput.test(output);
}
}
return {
gateId: gate.id,
gateName: gate.name,
passed,
output,
duration: Date.now() - startTime,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
return {
gateId: gate.id,
gateName: gate.name,
passed: false,
error: errorMessage,
duration: Date.now() - startTime,
};
}
}
/**
* Check if continuation is needed
*/
shouldContinue(
validation: CompletionValidation,
continuationCount: number,
config: OrchestrationConfig
): boolean {
// Don't continue if already accepted
if (validation.verdict === "accepted") {
return false;
}
// Don't continue if at max continuations
if (continuationCount >= config.maxContinuations) {
return false;
}
return true;
}
/**
* Generate continuation prompt based on failures
*/
generateContinuationPrompt(validation: CompletionValidation): string {
const failedGates = validation.gateResults.filter((r) => !r.passed);
let prompt = "Quality gates failed. Please address the following issues:\n\n";
for (const gate of failedGates) {
prompt += `**${gate.gateName}** failed:\n`;
if (gate.error) {
prompt += ` Error: ${gate.error}\n`;
}
if (gate.output) {
const outputPreview = gate.output.substring(0, 500);
prompt += ` Output: ${outputPreview}\n`;
}
prompt += "\n";
}
if (validation.suggestedActions && validation.suggestedActions.length > 0) {
prompt += "Suggested actions:\n";
for (const action of validation.suggestedActions) {
prompt += `- ${action}\n`;
}
}
return prompt;
}
/**
* Generate rejection feedback
*/
generateRejectionFeedback(validation: CompletionValidation): string {
const failedGates = validation.gateResults.filter((r) => !r.passed);
const failedCount = String(failedGates.length);
let feedback = `Task completion rejected. ${failedCount} quality gate(s) failed:\n\n`;
for (const gate of failedGates) {
feedback += `- ${gate.gateName}: `;
if (gate.error) {
feedback += gate.error;
} else {
feedback += "Failed validation";
}
feedback += "\n";
}
return feedback;
}
/**
* Generate suggested actions based on gate failures
*/
private generateSuggestedActions(
gateResults: QualityGateResult[],
config: OrchestrationConfig
): string[] {
const actions: string[] = [];
const failedGates = gateResults.filter((r) => !r.passed);
for (const result of failedGates) {
const gate = config.gates.find((g) => g.id === result.gateId);
if (!gate) continue;
switch (gate.type) {
case "build":
actions.push("Fix compilation errors in the code");
actions.push("Run: pnpm build");
break;
case "lint":
actions.push("Fix linting issues");
actions.push("Run: pnpm lint --fix");
break;
case "test":
actions.push("Fix failing tests");
actions.push("Run: pnpm test");
break;
case "coverage":
actions.push("Add tests to improve coverage to >= 85%");
actions.push("Run: pnpm test:coverage");
break;
default:
if (gate.command) {
actions.push(`Run: ${gate.command}`);
}
}
}
return actions;
}
/**
* Get default gates for a workspace
*/
getDefaultGates(workspaceId: string): QualityGate[] {
// For now, return the default gates
// In the future, this could be customized per workspace from database
this.logger.debug(`Getting default gates for workspace ${workspaceId}`);
return DEFAULT_GATES;
}
/**
* Track continuation attempts
*/
recordContinuation(taskId: string, attempt: number, validation: CompletionValidation): void {
const attemptStr = String(attempt);
const failedCount = String(validation.requiredGatesFailed.length);
this.logger.log(`Recording continuation attempt ${attemptStr} for task ${taskId}`);
// Store continuation record
// For now, just log it. In production, this would be stored in the database
this.logger.debug(`Continuation ${attemptStr}: ${failedCount} required gates failed`);
}
}