This commit is contained in:
+77
-11
@@ -29,12 +29,14 @@ function parseArgs(argv) {
|
||||
root: process.cwd(),
|
||||
manifest: 'gates/gates.manifest.json',
|
||||
skipHistory: false,
|
||||
structureOnly: false,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const value = argv[index];
|
||||
if (value === '--root') options.root = path.resolve(argv[++index]);
|
||||
else if (value === '--manifest') options.manifest = argv[++index];
|
||||
else if (value === '--skip-history') options.skipHistory = true;
|
||||
else if (value === '--structure-only') options.structureOnly = true;
|
||||
else throw new Error(`unknown option: ${value}`);
|
||||
}
|
||||
return options;
|
||||
@@ -307,7 +309,39 @@ function validateClosedSchema(manifest, failures) {
|
||||
failures,
|
||||
);
|
||||
rejectDuplicateIds(manifest.criteria, 'criterion', failures);
|
||||
for (const criterion of manifest.criteria ?? []) {
|
||||
rejectUnknownKeys(
|
||||
criterion,
|
||||
new Set([
|
||||
'id',
|
||||
'originalText',
|
||||
'currentText',
|
||||
'claimType',
|
||||
'source',
|
||||
'meaningChanges',
|
||||
'caseRefs',
|
||||
]),
|
||||
`criterion ${criterion.id}`,
|
||||
failures,
|
||||
);
|
||||
if (!Array.isArray(criterion.caseRefs) || criterion.caseRefs.length === 0) {
|
||||
failures.push(`${criterion.id}: no declared exercising cases`);
|
||||
} else if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
|
||||
failures.push(`${criterion.id}: duplicate declared exercising case`);
|
||||
}
|
||||
}
|
||||
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
|
||||
for (const claim of manifest.proseClaims ?? []) {
|
||||
rejectUnknownKeys(
|
||||
claim,
|
||||
new Set(['id', 'criterionId', 'caseRef']),
|
||||
`prose claim ${claim.id}`,
|
||||
failures,
|
||||
);
|
||||
if (typeof claim.caseRef !== 'string' || claim.caseRef.length === 0) {
|
||||
failures.push(`GATE-CLAIM:${claim.id} has no declared exercising case`);
|
||||
}
|
||||
}
|
||||
rejectDuplicateIds(manifest.compatibilityScenarios, 'compatibility scenario', failures);
|
||||
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
||||
rejectUnknownKeys(
|
||||
@@ -390,7 +424,9 @@ function validateClosedSchema(manifest, failures) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: case invocation must be non-empty`);
|
||||
}
|
||||
if (gateCase.mustFail === true && !gateCase.reasonPattern?.trim()) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`);
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -401,6 +437,7 @@ function validateStructure(manifest, failures) {
|
||||
const criteria = new Map((manifest.criteria ?? []).map((criterion) => [criterion.id, criterion]));
|
||||
const boundCriteria = new Set();
|
||||
const negativeBoundCriteria = new Set();
|
||||
const observedCaseRefs = new Map();
|
||||
|
||||
for (const gate of manifest.gates ?? []) {
|
||||
const negativeCases = (gate.cases ?? []).filter((gateCase) => gateCase.mustFail === true);
|
||||
@@ -413,12 +450,12 @@ function validateStructure(manifest, failures) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: unknown criterion ${criterionId}`);
|
||||
}
|
||||
boundCriteria.add(criterionId);
|
||||
const caseRef = `${gate.id}/${gateCase.id}`;
|
||||
if (!observedCaseRefs.has(criterionId)) observedCaseRefs.set(criterionId, new Set());
|
||||
observedCaseRefs.get(criterionId).add(caseRef);
|
||||
if (gateCase.mustFail === true) negativeBoundCriteria.add(criterionId);
|
||||
}
|
||||
if (
|
||||
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
||||
!gateCase.defect?.owner
|
||||
) {
|
||||
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && !gateCase.defect?.owner) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: behavior delta requires a tracked owner`);
|
||||
}
|
||||
}
|
||||
@@ -427,6 +464,18 @@ function validateStructure(manifest, failures) {
|
||||
for (const claim of manifest.proseClaims ?? []) {
|
||||
if (!criteria.has(claim.criterionId)) {
|
||||
failures.push(`GATE-CLAIM:${claim.id} references unknown criterion ${claim.criterionId}`);
|
||||
continue;
|
||||
}
|
||||
const found = findGateCase(manifest, claim.caseRef ?? '');
|
||||
if (!found) {
|
||||
failures.push(`GATE-CLAIM:${claim.id} references missing exercising case ${claim.caseRef}`);
|
||||
} else if (
|
||||
found.gateCase.mustFail !== true ||
|
||||
!found.gateCase.criterionIds?.includes(claim.criterionId)
|
||||
) {
|
||||
failures.push(
|
||||
`GATE-CLAIM:${claim.id} exercising case ${claim.caseRef} does not exercise criterion ${claim.criterionId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -435,6 +484,20 @@ function validateStructure(manifest, failures) {
|
||||
else if (!negativeBoundCriteria.has(criterion.id)) {
|
||||
failures.push(`${criterion.id}: no must-fail case exercises this criterion`);
|
||||
}
|
||||
const declaredRefs = new Set(criterion.caseRefs ?? []);
|
||||
const actualRefs = observedCaseRefs.get(criterion.id) ?? new Set();
|
||||
for (const caseRef of declaredRefs) {
|
||||
const found = findGateCase(manifest, caseRef);
|
||||
if (!found) failures.push(`${criterion.id}: declared exercising case ${caseRef} is missing`);
|
||||
else if (!actualRefs.has(caseRef)) {
|
||||
failures.push(`${criterion.id}: declared exercising case ${caseRef} is not bound`);
|
||||
}
|
||||
}
|
||||
for (const caseRef of actualRefs) {
|
||||
if (!declaredRefs.has(caseRef)) {
|
||||
failures.push(`${criterion.id}: bound to undeclared exercising case ${caseRef}`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
criterion.originalText !== criterion.currentText &&
|
||||
(!Array.isArray(criterion.meaningChanges) || criterion.meaningChanges.length === 0)
|
||||
@@ -634,7 +697,9 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
|
||||
for (const caseRef of scenario.caseRefs ?? []) {
|
||||
const found = findGateCase(manifest, caseRef);
|
||||
if (!found) {
|
||||
failures.push(`${scenario.id}: compatibility construction references missing case ${caseRef}`);
|
||||
failures.push(
|
||||
`${scenario.id}: compatibility construction references missing case ${caseRef}`,
|
||||
);
|
||||
} else {
|
||||
referenced.push({ caseRef, ...found });
|
||||
}
|
||||
@@ -671,7 +736,10 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
|
||||
}
|
||||
await applyFixture(sandbox, fixture);
|
||||
}
|
||||
for (const source of [...referenced.map(({ gateCase }) => gateCase.environment), scenario.environment]) {
|
||||
for (const source of [
|
||||
...referenced.map(({ gateCase }) => gateCase.environment),
|
||||
scenario.environment,
|
||||
]) {
|
||||
for (const [key, value] of Object.entries(source ?? {})) {
|
||||
if (environment[key] !== undefined && environment[key] !== value) {
|
||||
failures.push(`${scenario.id}: incompatible environment values for ${key}`);
|
||||
@@ -703,6 +771,7 @@ export async function verifyRegistry(options) {
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||||
|
||||
validateStructure(manifest, failures);
|
||||
if (options.structureOnly) return { failures, manifest, observations };
|
||||
await validateClaims(options.root, manifest, failures);
|
||||
await validateDiscovery(options.root, manifest, failures);
|
||||
|
||||
@@ -719,10 +788,7 @@ export async function verifyRegistry(options) {
|
||||
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
|
||||
}
|
||||
if (
|
||||
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
||||
gateCase.defect?.owner
|
||||
) {
|
||||
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && gateCase.defect?.owner) {
|
||||
observations.push(
|
||||
`DEFECT (owner: ${gateCase.defect.owner}) ${gate.id}/${gateCase.id}: required ${JSON.stringify(gateCase.required)}, actual ${JSON.stringify(gateCase.actual)}`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user