fix(quality): prove criterion binding semantics
ci/woodpecker/pr/ci Pipeline was successful

This commit is contained in:
2026-08-01 00:17:50 -05:00
parent 9b4d4beb0e
commit 38f1b249cc
9 changed files with 414 additions and 79 deletions
+77 -11
View File
@@ -29,12 +29,14 @@ function parseArgs(argv) {
root: process.cwd(),
manifest: 'gates/gates.manifest.json',
skipHistory: false,
structureOnly: false,
};
for (let index = 0; index < argv.length; index += 1) {
const value = argv[index];
if (value === '--root') options.root = path.resolve(argv[++index]);
else if (value === '--manifest') options.manifest = argv[++index];
else if (value === '--skip-history') options.skipHistory = true;
else if (value === '--structure-only') options.structureOnly = true;
else throw new Error(`unknown option: ${value}`);
}
return options;
@@ -307,7 +309,39 @@ function validateClosedSchema(manifest, failures) {
failures,
);
rejectDuplicateIds(manifest.criteria, 'criterion', failures);
for (const criterion of manifest.criteria ?? []) {
rejectUnknownKeys(
criterion,
new Set([
'id',
'originalText',
'currentText',
'claimType',
'source',
'meaningChanges',
'caseRefs',
]),
`criterion ${criterion.id}`,
failures,
);
if (!Array.isArray(criterion.caseRefs) || criterion.caseRefs.length === 0) {
failures.push(`${criterion.id}: no declared exercising cases`);
} else if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
failures.push(`${criterion.id}: duplicate declared exercising case`);
}
}
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
for (const claim of manifest.proseClaims ?? []) {
rejectUnknownKeys(
claim,
new Set(['id', 'criterionId', 'caseRef']),
`prose claim ${claim.id}`,
failures,
);
if (typeof claim.caseRef !== 'string' || claim.caseRef.length === 0) {
failures.push(`GATE-CLAIM:${claim.id} has no declared exercising case`);
}
}
rejectDuplicateIds(manifest.compatibilityScenarios, 'compatibility scenario', failures);
for (const scenario of manifest.compatibilityScenarios ?? []) {
rejectUnknownKeys(
@@ -390,7 +424,9 @@ function validateClosedSchema(manifest, failures) {
failures.push(`${gate.id}/${gateCase.id}: case invocation must be non-empty`);
}
if (gateCase.mustFail === true && !gateCase.reasonPattern?.trim()) {
failures.push(`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`);
failures.push(
`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`,
);
}
}
}
@@ -401,6 +437,7 @@ function validateStructure(manifest, failures) {
const criteria = new Map((manifest.criteria ?? []).map((criterion) => [criterion.id, criterion]));
const boundCriteria = new Set();
const negativeBoundCriteria = new Set();
const observedCaseRefs = new Map();
for (const gate of manifest.gates ?? []) {
const negativeCases = (gate.cases ?? []).filter((gateCase) => gateCase.mustFail === true);
@@ -413,12 +450,12 @@ function validateStructure(manifest, failures) {
failures.push(`${gate.id}/${gateCase.id}: unknown criterion ${criterionId}`);
}
boundCriteria.add(criterionId);
const caseRef = `${gate.id}/${gateCase.id}`;
if (!observedCaseRefs.has(criterionId)) observedCaseRefs.set(criterionId, new Set());
observedCaseRefs.get(criterionId).add(caseRef);
if (gateCase.mustFail === true) negativeBoundCriteria.add(criterionId);
}
if (
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
!gateCase.defect?.owner
) {
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && !gateCase.defect?.owner) {
failures.push(`${gate.id}/${gateCase.id}: behavior delta requires a tracked owner`);
}
}
@@ -427,6 +464,18 @@ function validateStructure(manifest, failures) {
for (const claim of manifest.proseClaims ?? []) {
if (!criteria.has(claim.criterionId)) {
failures.push(`GATE-CLAIM:${claim.id} references unknown criterion ${claim.criterionId}`);
continue;
}
const found = findGateCase(manifest, claim.caseRef ?? '');
if (!found) {
failures.push(`GATE-CLAIM:${claim.id} references missing exercising case ${claim.caseRef}`);
} else if (
found.gateCase.mustFail !== true ||
!found.gateCase.criterionIds?.includes(claim.criterionId)
) {
failures.push(
`GATE-CLAIM:${claim.id} exercising case ${claim.caseRef} does not exercise criterion ${claim.criterionId}`,
);
}
}
@@ -435,6 +484,20 @@ function validateStructure(manifest, failures) {
else if (!negativeBoundCriteria.has(criterion.id)) {
failures.push(`${criterion.id}: no must-fail case exercises this criterion`);
}
const declaredRefs = new Set(criterion.caseRefs ?? []);
const actualRefs = observedCaseRefs.get(criterion.id) ?? new Set();
for (const caseRef of declaredRefs) {
const found = findGateCase(manifest, caseRef);
if (!found) failures.push(`${criterion.id}: declared exercising case ${caseRef} is missing`);
else if (!actualRefs.has(caseRef)) {
failures.push(`${criterion.id}: declared exercising case ${caseRef} is not bound`);
}
}
for (const caseRef of actualRefs) {
if (!declaredRefs.has(caseRef)) {
failures.push(`${criterion.id}: bound to undeclared exercising case ${caseRef}`);
}
}
if (
criterion.originalText !== criterion.currentText &&
(!Array.isArray(criterion.meaningChanges) || criterion.meaningChanges.length === 0)
@@ -634,7 +697,9 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
for (const caseRef of scenario.caseRefs ?? []) {
const found = findGateCase(manifest, caseRef);
if (!found) {
failures.push(`${scenario.id}: compatibility construction references missing case ${caseRef}`);
failures.push(
`${scenario.id}: compatibility construction references missing case ${caseRef}`,
);
} else {
referenced.push({ caseRef, ...found });
}
@@ -671,7 +736,10 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
}
await applyFixture(sandbox, fixture);
}
for (const source of [...referenced.map(({ gateCase }) => gateCase.environment), scenario.environment]) {
for (const source of [
...referenced.map(({ gateCase }) => gateCase.environment),
scenario.environment,
]) {
for (const [key, value] of Object.entries(source ?? {})) {
if (environment[key] !== undefined && environment[key] !== value) {
failures.push(`${scenario.id}: incompatible environment values for ${key}`);
@@ -703,6 +771,7 @@ export async function verifyRegistry(options) {
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
validateStructure(manifest, failures);
if (options.structureOnly) return { failures, manifest, observations };
await validateClaims(options.root, manifest, failures);
await validateDiscovery(options.root, manifest, failures);
@@ -719,10 +788,7 @@ export async function verifyRegistry(options) {
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
}
if (
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
gateCase.defect?.owner
) {
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && gateCase.defect?.owner) {
observations.push(
`DEFECT (owner: ${gateCase.defect.owner}) ${gate.id}/${gateCase.id}: required ${JSON.stringify(gateCase.required)}, actual ${JSON.stringify(gateCase.actual)}`,
);