Implement prompt generation system that produces continuation prompts based on verification failures to force AI agents to complete work. Service: - generatePrompt: Complete prompt from failure context - generateTestFailurePrompt: Test-specific guidance - generateBuildErrorPrompt: Build error resolution - generateCoveragePrompt: Coverage improvement strategy - generateIncompleteWorkPrompt: Completion requirements Templates: - base.template: System/user prompt structure - test-failure.template: Test fix guidance - build-error.template: Compilation error guidance - coverage.template: Coverage improvement strategy - incomplete-work.template: Completion requirements Constraint escalation: - Attempt 1: Normal guidance - Attempt 2: Focus only on failures - Attempt 3: Minimal changes only - Final: Last attempt warning Priority levels: critical/high/normal based on failure severity Tests: 24 passing with 95.31% coverage Fixes #137 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
388 lines
11 KiB
TypeScript
388 lines
11 KiB
TypeScript
import { describe, it, expect, beforeEach } from "vitest";
|
|
import { ContinuationPromptsService } from "./continuation-prompts.service";
|
|
import { ContinuationPromptContext, FailureDetail, ContinuationPrompt } from "./interfaces";
|
|
|
|
describe("ContinuationPromptsService", () => {
|
|
let service: ContinuationPromptsService;
|
|
let baseContext: ContinuationPromptContext;
|
|
|
|
beforeEach(() => {
|
|
service = new ContinuationPromptsService();
|
|
baseContext = {
|
|
taskId: "task-1",
|
|
originalTask: "Implement user authentication",
|
|
attemptNumber: 1,
|
|
maxAttempts: 3,
|
|
failures: [],
|
|
filesChanged: ["src/auth/auth.service.ts"],
|
|
};
|
|
});
|
|
|
|
describe("generatePrompt", () => {
|
|
it("should generate a prompt with system and user sections", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
failures: [
|
|
{
|
|
type: "test-failure",
|
|
message: "Test failed: should authenticate user",
|
|
details: "Expected 200, got 401",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt).toBeDefined();
|
|
expect(prompt.systemPrompt).toContain("CRITICAL RULES");
|
|
expect(prompt.userPrompt).toContain("Implement user authentication");
|
|
expect(prompt.userPrompt).toContain("Test failed");
|
|
expect(prompt.constraints).toBeInstanceOf(Array);
|
|
expect(prompt.priority).toBe("high");
|
|
});
|
|
|
|
it("should include attempt number in prompt", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
attemptNumber: 2,
|
|
failures: [
|
|
{
|
|
type: "build-error",
|
|
message: "Type error in auth.service.ts",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.userPrompt).toContain("attempt 2 of 3");
|
|
});
|
|
|
|
it("should escalate priority on final attempt", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
attemptNumber: 3,
|
|
maxAttempts: 3,
|
|
failures: [
|
|
{
|
|
type: "test-failure",
|
|
message: "Tests still failing",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.priority).toBe("critical");
|
|
expect(prompt.constraints).toContain(
|
|
"This is your LAST attempt. Failure means manual intervention required."
|
|
);
|
|
});
|
|
|
|
it("should handle multiple failure types", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
failures: [
|
|
{
|
|
type: "test-failure",
|
|
message: "Auth test failed",
|
|
},
|
|
{
|
|
type: "build-error",
|
|
message: "Type error",
|
|
},
|
|
{
|
|
type: "coverage",
|
|
message: "Coverage below 85%",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.userPrompt).toContain("Auth test failed");
|
|
expect(prompt.userPrompt).toContain("Type error");
|
|
expect(prompt.userPrompt).toContain("Coverage below 85%");
|
|
});
|
|
});
|
|
|
|
describe("generateTestFailurePrompt", () => {
|
|
it("should format test failures with details", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "test-failure",
|
|
message: "should authenticate user",
|
|
details: "Expected 200, got 401",
|
|
location: "auth.service.spec.ts:42",
|
|
},
|
|
{
|
|
type: "test-failure",
|
|
message: "should reject invalid credentials",
|
|
details: "AssertionError: expected false to be true",
|
|
location: "auth.service.spec.ts:58",
|
|
},
|
|
];
|
|
|
|
const prompt = service.generateTestFailurePrompt(failures);
|
|
|
|
expect(prompt).toContain("should authenticate user");
|
|
expect(prompt).toContain("Expected 200, got 401");
|
|
expect(prompt).toContain("auth.service.spec.ts:42");
|
|
expect(prompt).toContain("should reject invalid credentials");
|
|
expect(prompt).toContain("Fix the implementation");
|
|
});
|
|
|
|
it("should include guidance for fixing tests", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "test-failure",
|
|
message: "Test failed",
|
|
},
|
|
];
|
|
|
|
const prompt = service.generateTestFailurePrompt(failures);
|
|
|
|
expect(prompt).toContain("Read the test");
|
|
expect(prompt).toContain("Fix the implementation");
|
|
expect(prompt).toContain("Run the test");
|
|
});
|
|
});
|
|
|
|
describe("generateBuildErrorPrompt", () => {
|
|
it("should format build errors with location", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "build-error",
|
|
message: "Type 'string' is not assignable to type 'number'",
|
|
location: "auth.service.ts:25",
|
|
},
|
|
{
|
|
type: "build-error",
|
|
message: "Cannot find name 'User'",
|
|
location: "auth.service.ts:42",
|
|
suggestion: "Import User from '@/entities'",
|
|
},
|
|
];
|
|
|
|
const prompt = service.generateBuildErrorPrompt(failures);
|
|
|
|
expect(prompt).toContain("Type 'string' is not assignable");
|
|
expect(prompt).toContain("auth.service.ts:25");
|
|
expect(prompt).toContain("Cannot find name 'User'");
|
|
expect(prompt).toContain("Import User from");
|
|
});
|
|
|
|
it("should include build-specific guidance", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "build-error",
|
|
message: "Syntax error",
|
|
},
|
|
];
|
|
|
|
const prompt = service.generateBuildErrorPrompt(failures);
|
|
|
|
expect(prompt).toContain("TypeScript");
|
|
expect(prompt).toContain("Do not proceed until build passes");
|
|
});
|
|
});
|
|
|
|
describe("generateCoveragePrompt", () => {
|
|
it("should show coverage gap", () => {
|
|
const prompt = service.generateCoveragePrompt(72, 85);
|
|
|
|
expect(prompt).toContain("72%");
|
|
expect(prompt).toContain("85%");
|
|
expect(prompt).toContain("13%"); // gap
|
|
});
|
|
|
|
it("should provide guidance for improving coverage", () => {
|
|
const prompt = service.generateCoveragePrompt(80, 85);
|
|
|
|
expect(prompt).toContain("uncovered code paths");
|
|
expect(prompt).toContain("edge cases");
|
|
expect(prompt).toContain("error handling");
|
|
});
|
|
});
|
|
|
|
describe("generateIncompleteWorkPrompt", () => {
|
|
it("should list incomplete work items", () => {
|
|
const issues = [
|
|
"TODO: Implement password hashing",
|
|
"FIXME: Add error handling",
|
|
"Missing validation for email format",
|
|
];
|
|
|
|
const prompt = service.generateIncompleteWorkPrompt(issues);
|
|
|
|
expect(prompt).toContain("TODO: Implement password hashing");
|
|
expect(prompt).toContain("FIXME: Add error handling");
|
|
expect(prompt).toContain("Missing validation");
|
|
});
|
|
|
|
it("should emphasize completion requirement", () => {
|
|
const issues = ["Missing feature X"];
|
|
|
|
const prompt = service.generateIncompleteWorkPrompt(issues);
|
|
|
|
expect(prompt).toContain("MUST complete ALL aspects");
|
|
expect(prompt).toContain("Do not leave TODO");
|
|
});
|
|
});
|
|
|
|
describe("getConstraints", () => {
|
|
it("should return basic constraints for first attempt", () => {
|
|
const constraints = service.getConstraints(1, 3);
|
|
|
|
expect(constraints).toBeInstanceOf(Array);
|
|
expect(constraints.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("should escalate constraints on second attempt", () => {
|
|
const constraints = service.getConstraints(2, 3);
|
|
|
|
expect(constraints).toContain("Focus only on failures, no new features");
|
|
});
|
|
|
|
it("should add strict constraints on third attempt", () => {
|
|
const constraints = service.getConstraints(3, 3);
|
|
|
|
expect(constraints).toContain("Minimal changes only, fix exact issues");
|
|
});
|
|
|
|
it("should add final warning on last attempt", () => {
|
|
const constraints = service.getConstraints(3, 3);
|
|
|
|
expect(constraints).toContain(
|
|
"This is your LAST attempt. Failure means manual intervention required."
|
|
);
|
|
});
|
|
|
|
it("should handle different max attempts", () => {
|
|
const constraints = service.getConstraints(5, 5);
|
|
|
|
expect(constraints).toContain(
|
|
"This is your LAST attempt. Failure means manual intervention required."
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("formatFailuresForPrompt", () => {
|
|
it("should format failures with all details", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "test-failure",
|
|
message: "Test failed",
|
|
details: "Expected true, got false",
|
|
location: "file.spec.ts:10",
|
|
suggestion: "Check the implementation",
|
|
},
|
|
];
|
|
|
|
const formatted = service.formatFailuresForPrompt(failures);
|
|
|
|
expect(formatted).toContain("test-failure");
|
|
expect(formatted).toContain("Test failed");
|
|
expect(formatted).toContain("Expected true, got false");
|
|
expect(formatted).toContain("file.spec.ts:10");
|
|
expect(formatted).toContain("Check the implementation");
|
|
});
|
|
|
|
it("should handle failures without optional fields", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "lint-error",
|
|
message: "Unused variable",
|
|
},
|
|
];
|
|
|
|
const formatted = service.formatFailuresForPrompt(failures);
|
|
|
|
expect(formatted).toContain("lint-error");
|
|
expect(formatted).toContain("Unused variable");
|
|
});
|
|
|
|
it("should format multiple failures", () => {
|
|
const failures: FailureDetail[] = [
|
|
{
|
|
type: "test-failure",
|
|
message: "Test 1 failed",
|
|
},
|
|
{
|
|
type: "build-error",
|
|
message: "Build error",
|
|
},
|
|
{
|
|
type: "coverage",
|
|
message: "Low coverage",
|
|
},
|
|
];
|
|
|
|
const formatted = service.formatFailuresForPrompt(failures);
|
|
|
|
expect(formatted).toContain("Test 1 failed");
|
|
expect(formatted).toContain("Build error");
|
|
expect(formatted).toContain("Low coverage");
|
|
});
|
|
|
|
it("should handle empty failures array", () => {
|
|
const failures: FailureDetail[] = [];
|
|
|
|
const formatted = service.formatFailuresForPrompt(failures);
|
|
|
|
expect(formatted).toBe("");
|
|
});
|
|
});
|
|
|
|
describe("priority assignment", () => {
|
|
it("should set normal priority for first attempt with minor issues", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
attemptNumber: 1,
|
|
failures: [
|
|
{
|
|
type: "lint-error",
|
|
message: "Minor lint issue",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.priority).toBe("normal");
|
|
});
|
|
|
|
it("should set high priority for build errors", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
failures: [
|
|
{
|
|
type: "build-error",
|
|
message: "Build failed",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.priority).toBe("high");
|
|
});
|
|
|
|
it("should set high priority for test failures", () => {
|
|
const context: ContinuationPromptContext = {
|
|
...baseContext,
|
|
failures: [
|
|
{
|
|
type: "test-failure",
|
|
message: "Test failed",
|
|
},
|
|
],
|
|
};
|
|
|
|
const prompt = service.generatePrompt(context);
|
|
|
|
expect(prompt.priority).toBe("high");
|
|
});
|
|
});
|
|
});
|