import { Injectable, Logger } from "@nestjs/common"; import { exec } from "child_process"; import { promisify } from "util"; import type { QualityGate, QualityGateResult, CompletionClaim, CompletionValidation, OrchestrationConfig, } from "./interfaces"; import { TokenBudgetService } from "../token-budget/token-budget.service"; const execAsync = promisify(exec); /** * Default quality gates for all workspaces */ const DEFAULT_GATES: QualityGate[] = [ { id: "build", name: "Build Check", description: "Verify code compiles without errors", type: "build", command: "pnpm build", required: true, order: 1, }, { id: "lint", name: "Lint Check", description: "Code follows style guidelines", type: "lint", command: "pnpm lint", required: true, order: 2, }, { id: "test", name: "Test Suite", description: "All tests pass", type: "test", command: "pnpm test", required: true, order: 3, }, { id: "coverage", name: "Coverage Check", description: "Test coverage >= 85%", type: "coverage", command: "pnpm test:coverage", expectedOutput: /All files.*[89]\d|100/, required: false, order: 4, }, ]; /** * Quality Orchestrator Service * Validates AI agent task completions and enforces quality gates */ @Injectable() export class QualityOrchestratorService { private readonly logger = new Logger(QualityOrchestratorService.name); constructor(private readonly tokenBudgetService: TokenBudgetService) {} /** * Validate a completion claim against quality gates */ async validateCompletion( claim: CompletionClaim, config: OrchestrationConfig ): Promise { this.logger.log( `Validating completion claim for task ${claim.taskId} by agent ${claim.agentId}` ); // Sort gates by order const sortedGates = [...config.gates].sort((a, b) => a.order - b.order); // Run all gates const gateResults: QualityGateResult[] = []; for (const gate of sortedGates) { const result = await this.runGate(gate); gateResults.push(result); } // Analyze results const allGatesPassed = gateResults.every((r) => r.passed); const requiredGatesFailed = gateResults .filter((r) => !r.passed) .map((r) => r.gateId) .filter((id) => { const gate = config.gates.find((g) => g.id === id); return gate?.required ?? false; }); // Check token budget for suspicious patterns let budgetCheck: { suspicious: boolean; reason?: string } | null = null; try { budgetCheck = await this.tokenBudgetService.checkSuspiciousDoneClaim(claim.taskId); } catch { // Token budget not found - not an error, just means tracking wasn't enabled this.logger.debug(`No token budget found for task ${claim.taskId}`); } // Determine verdict let verdict: "accepted" | "rejected" | "needs-continuation"; if (allGatesPassed) { // Even if all gates passed, check for suspicious budget patterns if (budgetCheck?.suspicious) { verdict = "needs-continuation"; this.logger.warn( `Suspicious budget pattern detected for task ${claim.taskId}: ${budgetCheck.reason ?? "unknown reason"}` ); } else { verdict = "accepted"; } } else if (requiredGatesFailed.length > 0) { verdict = "rejected"; } else if (config.strictMode) { verdict = "rejected"; } else { verdict = "accepted"; } // Generate feedback and suggestions const result: CompletionValidation = { claim, gateResults, allGatesPassed, requiredGatesFailed, verdict, }; if (verdict !== "accepted") { result.feedback = this.generateRejectionFeedback(result); result.suggestedActions = this.generateSuggestedActions(gateResults, config); // Add budget feedback if suspicious pattern detected if (budgetCheck?.suspicious && budgetCheck.reason) { result.feedback += `\n\nToken budget analysis: ${budgetCheck.reason}`; result.suggestedActions.push( "Review task completion - significant budget remains or suspicious usage pattern detected" ); } } return result; } /** * Run a single quality gate */ async runGate(gate: QualityGate): Promise { this.logger.debug(`Running gate: ${gate.name} (${gate.id})`); const startTime = Date.now(); try { if (!gate.command) { // Custom gates without commands always pass return { gateId: gate.id, gateName: gate.name, passed: true, duration: Date.now() - startTime, }; } const { stdout, stderr } = await execAsync(gate.command, { timeout: 300000, // 5 minute timeout maxBuffer: 10 * 1024 * 1024, // 10MB buffer }); const output = stdout + stderr; let passed = true; // Check expected output pattern if provided if (gate.expectedOutput) { if (typeof gate.expectedOutput === "string") { passed = output.includes(gate.expectedOutput); } else { // RegExp passed = gate.expectedOutput.test(output); } } return { gateId: gate.id, gateName: gate.name, passed, output, duration: Date.now() - startTime, }; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); return { gateId: gate.id, gateName: gate.name, passed: false, error: errorMessage, duration: Date.now() - startTime, }; } } /** * Check if continuation is needed */ shouldContinue( validation: CompletionValidation, continuationCount: number, config: OrchestrationConfig ): boolean { // Don't continue if already accepted if (validation.verdict === "accepted") { return false; } // Don't continue if at max continuations if (continuationCount >= config.maxContinuations) { return false; } return true; } /** * Generate continuation prompt based on failures */ generateContinuationPrompt(validation: CompletionValidation): string { const failedGates = validation.gateResults.filter((r) => !r.passed); let prompt = "Quality gates failed. Please address the following issues:\n\n"; for (const gate of failedGates) { prompt += `**${gate.gateName}** failed:\n`; if (gate.error) { prompt += ` Error: ${gate.error}\n`; } if (gate.output) { const outputPreview = gate.output.substring(0, 500); prompt += ` Output: ${outputPreview}\n`; } prompt += "\n"; } if (validation.suggestedActions && validation.suggestedActions.length > 0) { prompt += "Suggested actions:\n"; for (const action of validation.suggestedActions) { prompt += `- ${action}\n`; } } return prompt; } /** * Generate rejection feedback */ generateRejectionFeedback(validation: CompletionValidation): string { const failedGates = validation.gateResults.filter((r) => !r.passed); const failedCount = String(failedGates.length); let feedback = `Task completion rejected. ${failedCount} quality gate(s) failed:\n\n`; for (const gate of failedGates) { feedback += `- ${gate.gateName}: `; if (gate.error) { feedback += gate.error; } else { feedback += "Failed validation"; } feedback += "\n"; } return feedback; } /** * Generate suggested actions based on gate failures */ private generateSuggestedActions( gateResults: QualityGateResult[], config: OrchestrationConfig ): string[] { const actions: string[] = []; const failedGates = gateResults.filter((r) => !r.passed); for (const result of failedGates) { const gate = config.gates.find((g) => g.id === result.gateId); if (!gate) continue; switch (gate.type) { case "build": actions.push("Fix compilation errors in the code"); actions.push("Run: pnpm build"); break; case "lint": actions.push("Fix linting issues"); actions.push("Run: pnpm lint --fix"); break; case "test": actions.push("Fix failing tests"); actions.push("Run: pnpm test"); break; case "coverage": actions.push("Add tests to improve coverage to >= 85%"); actions.push("Run: pnpm test:coverage"); break; default: if (gate.command) { actions.push(`Run: ${gate.command}`); } } } return actions; } /** * Get default gates for a workspace */ getDefaultGates(workspaceId: string): QualityGate[] { // For now, return the default gates // In the future, this could be customized per workspace from database this.logger.debug(`Getting default gates for workspace ${workspaceId}`); return DEFAULT_GATES; } /** * Track continuation attempts */ recordContinuation(taskId: string, attempt: number, validation: CompletionValidation): void { const attemptStr = String(attempt); const failedCount = String(validation.requiredGatesFailed.length); this.logger.log(`Recording continuation attempt ${attemptStr} for task ${taskId}`); // Store continuation record // For now, just log it. In production, this would be stored in the database this.logger.debug(`Continuation ${attemptStr}: ${failedCount} required gates failed`); } }