fix(quality): preserve binding diagnostics

This commit is contained in:
2026-08-01 09:58:47 -05:00
committed by f10-coder
parent 8b1b873056
commit 9e1a7a44b7
6 changed files with 91 additions and 19 deletions
+49 -15
View File
@@ -326,8 +326,13 @@ function validateClosedSchema(manifest, failures) {
);
if (!Array.isArray(criterion.caseRefs) || criterion.caseRefs.length === 0) {
failures.push(`${criterion.id}: no declared exercising cases`);
} else if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
failures.push(`${criterion.id}: duplicate declared exercising case`);
} else {
if (criterion.caseRefs.some((caseRef) => typeof caseRef !== 'string' || !caseRef)) {
failures.push(`${criterion.id}: declared exercising cases must be non-empty strings`);
}
if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
failures.push(`${criterion.id}: duplicate declared exercising case`);
}
}
}
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
@@ -683,6 +688,7 @@ async function validateMutation(root, gate, failures, observations) {
}
function findGateCase(manifest, caseRef) {
if (typeof caseRef !== 'string') return undefined;
const separator = caseRef.indexOf('/');
if (separator < 1) return undefined;
const gateId = caseRef.slice(0, separator);
@@ -772,21 +778,45 @@ export async function verifyRegistry(options) {
validateStructure(manifest, failures);
if (options.structureOnly) return { failures, manifest, observations };
await validateClaims(options.root, manifest, failures);
await validateDiscovery(options.root, manifest, failures);
async function collectPhaseFailure(label, action) {
try {
return await action();
} catch (error) {
failures.push(`${label}: ${error.message}`);
return undefined;
}
}
await collectPhaseFailure('governing claim validation failed', () =>
validateClaims(options.root, manifest, failures),
);
await collectPhaseFailure('gate discovery failed', () =>
validateDiscovery(options.root, manifest, failures),
);
for (const gate of manifest.gates ?? []) {
await validateDeployment(options.root, gate, failures, observations);
await collectPhaseFailure(`${gate.id}: deployment validation failed`, () =>
validateDeployment(options.root, gate, failures, observations),
);
for (const gateCase of gate.cases ?? []) {
const result = await runCase(options.root, gate, gateCase);
const result = await collectPhaseFailure(
`${gate.id}/${gateCase.id}: case execution failed`,
() => runCase(options.root, gate, gateCase),
);
if (!result) continue;
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
if (!outcomeMatches(gateCase.actual, result)) {
failures.push(
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
);
}
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
try {
if (!outcomeMatches(gateCase.actual, result)) {
failures.push(
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
);
}
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
}
} catch (error) {
failures.push(`${gate.id}/${gateCase.id}: outcome validation failed: ${error.message}`);
}
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && gateCase.defect?.owner) {
observations.push(
@@ -794,11 +824,15 @@ export async function verifyRegistry(options) {
);
}
}
await validateMutation(options.root, gate, failures, observations);
await collectPhaseFailure(`${gate.id}: mutation validation failed`, () =>
validateMutation(options.root, gate, failures, observations),
);
}
for (const scenario of manifest.compatibilityScenarios ?? []) {
await runCompatibilityScenario(options.root, manifest, scenario, failures, observations);
await collectPhaseFailure(`${scenario.id}: compatibility validation failed`, () =>
runCompatibilityScenario(options.root, manifest, scenario, failures, observations),
);
}
return { failures, manifest, observations };
+37
View File
@@ -296,6 +296,43 @@ test('moving criterion bindings to unrelated cases is rejected', async () => {
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
});
test('canonical verification preserves binding diagnostics when a case fixture is also stale', async () => {
const root = await fixture('misbinding-plus-stale-fixture');
await writeGate(root);
const manifest = baseManifest();
manifest.criteria.push({
id: 'META-CRIT-2',
originalText: 'The second case rejects its own bad input.',
currentText: 'The second case rejects its own bad input.',
claimType: 'integrity',
source: 'fixture',
meaningChanges: [],
caseRefs: ['meta-fixture/rejects-second-input'],
});
manifest.gates[0].cases.push({
...manifest.gates[0].cases[0],
id: 'rejects-second-input',
criterionIds: ['META-CRIT-1'],
});
manifest.gates[0].cases[0].criterionIds = ['META-CRIT-2'];
manifest.gates[0].cases[0].fixture = {
replaceFiles: [
{
path: 'gates/meta-fixture.sh',
find: 'text that is not present',
replace: 'irrelevant',
},
],
};
await writeManifest(root, manifest);
const result = verify(root);
assert.notEqual(result.status, 0);
assert.match(output(result), /META-CRIT-1.*rejects-bad-input.*not bound/i);
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
assert.match(output(result), /fixture replace.*stale or ambiguous/i);
});
test('moving meaning and prose criteria to an unrelated type error is rejected', async () => {
const root = await fixture('real-manifest-misbinding');
const manifest = JSON.parse(