feat(#136): build Completion Verification Engine
Implement verification engine to determine if AI agent work is truly complete by analyzing outputs and detecting deferred work patterns. Strategies: - FileChangeStrategy: Detect TODO/FIXME, placeholders, stubs - TestOutputStrategy: Validate pass rates, coverage (85%), skipped tests - BuildOutputStrategy: Detect TS errors, ESLint errors, build failures Deferred work detection patterns: - "follow-up", "to be added later" - "incremental improvement", "future enhancement" - "TODO: complete", "placeholder implementation" - "stub", "work in progress", "partially implemented" Features: - Confidence scoring (0-100%) - Verdict system: complete/incomplete/needs-review - Actionable suggestions for improvements - Strategy-based extensibility Integration: - Complements Quality Orchestrator (#134) - Uses Quality Gate Config (#135) Tests: 46 passing with 95.27% coverage Fixes #136 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { BuildOutputStrategy } from "./build-output.strategy";
|
||||
import { VerificationContext } from "../interfaces";
|
||||
|
||||
describe("BuildOutputStrategy", () => {
|
||||
let strategy: BuildOutputStrategy;
|
||||
let baseContext: VerificationContext;
|
||||
|
||||
beforeEach(() => {
|
||||
strategy = new BuildOutputStrategy();
|
||||
baseContext = {
|
||||
taskId: "task-1",
|
||||
workspaceId: "workspace-1",
|
||||
agentId: "agent-1",
|
||||
claimMessage: "Built successfully",
|
||||
filesChanged: ["src/feature.ts"],
|
||||
outputLogs: "",
|
||||
previousAttempts: 0,
|
||||
};
|
||||
});
|
||||
|
||||
describe("verify", () => {
|
||||
it("should pass when build succeeds", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "Build completed successfully\nNo errors found",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.strategyName).toBe("build-output");
|
||||
expect(result.confidence).toBeGreaterThanOrEqual(90);
|
||||
expect(result.issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should fail when TypeScript errors found", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: 'error TS2304: Cannot find name "unknown".\nBuild failed',
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.issues.some((i) => i.type === "build-error")).toBe(true);
|
||||
expect(result.issues.some((i) => i.message.includes("TypeScript"))).toBe(true);
|
||||
});
|
||||
|
||||
it("should fail when build errors found", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "Error: Module not found\nBuild failed with 1 error",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.issues.some((i) => i.type === "build-error")).toBe(true);
|
||||
});
|
||||
|
||||
it("should detect ESLint errors", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "ESLint error: no-unused-vars\n1 error found",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.issues.some((i) => i.message.includes("ESLint"))).toBe(true);
|
||||
});
|
||||
|
||||
it("should warn about lint warnings", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "warning: unused variable\nBuild completed with warnings",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.issues.some((i) => i.severity === "warning")).toBe(true);
|
||||
});
|
||||
|
||||
it("should pass when no build output provided", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: undefined,
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.confidence).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should reduce confidence with multiple errors", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput:
|
||||
"error TS2304: Cannot find name\nerror TS2345: Type mismatch\nerror TS1005: Syntax error\nBuild failed",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.confidence).toBeLessThan(50);
|
||||
expect(result.issues.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("should detect compilation failures", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "Compilation failed\nProcess exited with code 1",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.issues.some((i) => i.type === "build-error")).toBe(true);
|
||||
});
|
||||
|
||||
it("should have high confidence with clean build", async () => {
|
||||
const context: VerificationContext = {
|
||||
...baseContext,
|
||||
buildOutput: "Build successful\nNo errors or warnings\nCompleted in 5s",
|
||||
};
|
||||
|
||||
const result = await strategy.verify(context);
|
||||
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.confidence).toBeGreaterThanOrEqual(95);
|
||||
expect(result.issues).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user