fix(quality): preserve binding diagnostics
This commit is contained in:
+49
-15
@@ -326,8 +326,13 @@ function validateClosedSchema(manifest, failures) {
|
||||
);
|
||||
if (!Array.isArray(criterion.caseRefs) || criterion.caseRefs.length === 0) {
|
||||
failures.push(`${criterion.id}: no declared exercising cases`);
|
||||
} else if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
|
||||
failures.push(`${criterion.id}: duplicate declared exercising case`);
|
||||
} else {
|
||||
if (criterion.caseRefs.some((caseRef) => typeof caseRef !== 'string' || !caseRef)) {
|
||||
failures.push(`${criterion.id}: declared exercising cases must be non-empty strings`);
|
||||
}
|
||||
if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
|
||||
failures.push(`${criterion.id}: duplicate declared exercising case`);
|
||||
}
|
||||
}
|
||||
}
|
||||
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
|
||||
@@ -683,6 +688,7 @@ async function validateMutation(root, gate, failures, observations) {
|
||||
}
|
||||
|
||||
function findGateCase(manifest, caseRef) {
|
||||
if (typeof caseRef !== 'string') return undefined;
|
||||
const separator = caseRef.indexOf('/');
|
||||
if (separator < 1) return undefined;
|
||||
const gateId = caseRef.slice(0, separator);
|
||||
@@ -772,21 +778,45 @@ export async function verifyRegistry(options) {
|
||||
|
||||
validateStructure(manifest, failures);
|
||||
if (options.structureOnly) return { failures, manifest, observations };
|
||||
await validateClaims(options.root, manifest, failures);
|
||||
await validateDiscovery(options.root, manifest, failures);
|
||||
|
||||
async function collectPhaseFailure(label, action) {
|
||||
try {
|
||||
return await action();
|
||||
} catch (error) {
|
||||
failures.push(`${label}: ${error.message}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
await collectPhaseFailure('governing claim validation failed', () =>
|
||||
validateClaims(options.root, manifest, failures),
|
||||
);
|
||||
await collectPhaseFailure('gate discovery failed', () =>
|
||||
validateDiscovery(options.root, manifest, failures),
|
||||
);
|
||||
|
||||
for (const gate of manifest.gates ?? []) {
|
||||
await validateDeployment(options.root, gate, failures, observations);
|
||||
await collectPhaseFailure(`${gate.id}: deployment validation failed`, () =>
|
||||
validateDeployment(options.root, gate, failures, observations),
|
||||
);
|
||||
for (const gateCase of gate.cases ?? []) {
|
||||
const result = await runCase(options.root, gate, gateCase);
|
||||
const result = await collectPhaseFailure(
|
||||
`${gate.id}/${gateCase.id}: case execution failed`,
|
||||
() => runCase(options.root, gate, gateCase),
|
||||
);
|
||||
if (!result) continue;
|
||||
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
||||
if (!outcomeMatches(gateCase.actual, result)) {
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
|
||||
);
|
||||
}
|
||||
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
|
||||
try {
|
||||
if (!outcomeMatches(gateCase.actual, result)) {
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
|
||||
);
|
||||
}
|
||||
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
|
||||
}
|
||||
} catch (error) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: outcome validation failed: ${error.message}`);
|
||||
}
|
||||
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && gateCase.defect?.owner) {
|
||||
observations.push(
|
||||
@@ -794,11 +824,15 @@ export async function verifyRegistry(options) {
|
||||
);
|
||||
}
|
||||
}
|
||||
await validateMutation(options.root, gate, failures, observations);
|
||||
await collectPhaseFailure(`${gate.id}: mutation validation failed`, () =>
|
||||
validateMutation(options.root, gate, failures, observations),
|
||||
);
|
||||
}
|
||||
|
||||
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
||||
await runCompatibilityScenario(options.root, manifest, scenario, failures, observations);
|
||||
await collectPhaseFailure(`${scenario.id}: compatibility validation failed`, () =>
|
||||
runCompatibilityScenario(options.root, manifest, scenario, failures, observations),
|
||||
);
|
||||
}
|
||||
|
||||
return { failures, manifest, observations };
|
||||
|
||||
@@ -296,6 +296,43 @@ test('moving criterion bindings to unrelated cases is rejected', async () => {
|
||||
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
|
||||
});
|
||||
|
||||
test('canonical verification preserves binding diagnostics when a case fixture is also stale', async () => {
|
||||
const root = await fixture('misbinding-plus-stale-fixture');
|
||||
await writeGate(root);
|
||||
const manifest = baseManifest();
|
||||
manifest.criteria.push({
|
||||
id: 'META-CRIT-2',
|
||||
originalText: 'The second case rejects its own bad input.',
|
||||
currentText: 'The second case rejects its own bad input.',
|
||||
claimType: 'integrity',
|
||||
source: 'fixture',
|
||||
meaningChanges: [],
|
||||
caseRefs: ['meta-fixture/rejects-second-input'],
|
||||
});
|
||||
manifest.gates[0].cases.push({
|
||||
...manifest.gates[0].cases[0],
|
||||
id: 'rejects-second-input',
|
||||
criterionIds: ['META-CRIT-1'],
|
||||
});
|
||||
manifest.gates[0].cases[0].criterionIds = ['META-CRIT-2'];
|
||||
manifest.gates[0].cases[0].fixture = {
|
||||
replaceFiles: [
|
||||
{
|
||||
path: 'gates/meta-fixture.sh',
|
||||
find: 'text that is not present',
|
||||
replace: 'irrelevant',
|
||||
},
|
||||
],
|
||||
};
|
||||
await writeManifest(root, manifest);
|
||||
|
||||
const result = verify(root);
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.match(output(result), /META-CRIT-1.*rejects-bad-input.*not bound/i);
|
||||
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
|
||||
assert.match(output(result), /fixture replace.*stale or ambiguous/i);
|
||||
});
|
||||
|
||||
test('moving meaning and prose criteria to an unrelated type error is rejected', async () => {
|
||||
const root = await fixture('real-manifest-misbinding');
|
||||
const manifest = JSON.parse(
|
||||
|
||||
Reference in New Issue
Block a user