mosaicstack/stack
1
0
Fork 1
Code Issues 140 Pull Requests 5 Actions Packages Projects Releases 15 Wiki Activity

fix(quality): prove criterion binding semantics

Browse Source
This commit is contained in:
coder-mos1
2026-08-01 09:58:47 -05:00
committed by f10-coder
parent d119635265
commit 8b1b873056
9 changed files with 414 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gate-history.mjs
+25 -5
View File
@@ -1,5 +1,5 @@
import { existsSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { access, lstat, mkdir, mkdtemp, readFile, readdir, readlink, rm } from 'node:fs/promises';
import { spawnSync } from 'node:child_process';
import path from 'node:path';
@@ -40,7 +40,9 @@ async function snapshotAuthoritativeTree(root) {
} else if (stats.isSymbolicLink()) {
snapshot.set(relative, `symlink:${stats.mode}:${await readlink(absolute)}`);
} else if (stats.isFile()) {
const digest = createHash('sha256').update(await readFile(absolute)).digest('hex');
const digest = createHash('sha256')
.update(await readFile(absolute))
.digest('hex');
snapshot.set(relative, `file:${stats.mode}:${digest}`);
}
}
@@ -59,7 +61,9 @@ async function authoritativeTreeChanges(root, snapshot) {
if (stats.isSymbolicLink()) {
actual = `symlink:${stats.mode}:${await readlink(absolute)}`;
} else if (stats.isFile()) {
const digest = createHash('sha256').update(await readFile(absolute)).digest('hex');
const digest = createHash('sha256')
.update(await readFile(absolute))
.digest('hex');
actual = `file:${stats.mode}:${digest}`;
} else {
actual = `other:${stats.mode}`;
@@ -108,8 +112,24 @@ function bubblewrap(root, command, args, { storePath, timeout = 300_000 } = {})
);
if (storePath) sandboxArgs.push('--setenv', 'NPM_CONFIG_STORE_DIR', '/pnpm-store');
if (existsSync(corepackHome)) sandboxArgs.push('--setenv', 'COREPACK_HOME', '/corepack');
sandboxArgs.push(command, ...args);
return spawnSync('bwrap', sandboxArgs, { encoding: 'utf8', timeout });
const enteredMarker = `__MOSAIC_BWRAP_ENTERED_${randomUUID()}__`;
sandboxArgs.push(
'/bin/sh',
'-c',
'printf "%s\\n" "$1"; shift; exec "$@"',
'mosaic-bwrap-entry',
enteredMarker,
command,
...args,
);
const result = spawnSync('bwrap', sandboxArgs, { encoding: 'utf8', timeout });
const sandboxEntered = result.stdout?.includes(enteredMarker) === true;
return {
...result,
stdout: (result.stdout ?? '').replace(`${enteredMarker}\n`, ''),
sandboxLauncher: 'bwrap',
sandboxEntered,
};
}
export async function replayCommit(root, commit) {
scripts/gate-history.test.mjs
+32 -4
View File
@@ -15,6 +15,7 @@ import {
const fixtureRoot = path.join(process.cwd(), '.mosaic-test-work', `gate-history-${process.pid}`);
function sandboxUnavailable(result) {
if (result.sandboxLauncher !== 'bwrap' || result.sandboxEntered === true) return false;
const detail = `${result.stdout ?? ''}${result.stderr ?? ''}${result.error?.message ?? ''}`;
const bubblewrapSpawnDenied =
['EPERM', 'EACCES', 'ENOENT'].includes(result.error?.code) &&
@@ -56,6 +57,8 @@ test('sandbox refusal classification requires Bubblewrap provenance', () => {
sandboxUnavailable({
status: null,
error: { code, message: `spawnSync bwrap ${code}` },
sandboxLauncher: 'bwrap',
sandboxEntered: false,
}),
true,
);
@@ -71,6 +74,31 @@ test('sandbox refusal classification requires Bubblewrap provenance', () => {
sandboxUnavailable({ status: 1, stderr: 'historical verifier said bwrap ENOENT' }),
false,
);
assert.equal(
sandboxUnavailable({
status: 1,
stderr: 'bwrap: Creating new namespace failed: Operation not permitted',
sandboxLauncher: 'bwrap',
sandboxEntered: false,
}),
true,
);
assert.equal(
sandboxUnavailable({
status: 1,
stderr: 'bwrap: Creating new namespace failed: Operation not permitted',
}),
false,
);
assert.equal(
sandboxUnavailable({
status: 1,
stderr: 'bwrap: Creating new namespace failed: Operation not permitted',
sandboxLauncher: 'bwrap',
sandboxEntered: true,
}),
false,
);
});
test('prospective history reads each commit own manifest rather than the current tree', async () => {
@@ -261,13 +289,13 @@ test('PR verification states the RM-60 boundary without executing an intermediat
});
assert.deepEqual(result.failures, []);
assert.ok(
result.observations.some(
(observation) => /DOES:.*current tree.*DOES NOT:.*isolated.*RM-60.*RM-59/i.test(observation),
result.observations.some((observation) =>
/DOES:.*current tree.*DOES NOT:.*isolated.*RM-60.*RM-59/i.test(observation),
),
);
assert.ok(
result.observations.some(
(observation) => /INTERMEDIATE REPLAY DEFERRED.*RM-60.*no success is inferred/i.test(observation),
result.observations.some((observation) =>
/INTERMEDIATE REPLAY DEFERRED.*RM-60.*no success is inferred/i.test(observation),
),
);
assert.ok(result.observations.every((observation) => !/INTERMEDIATE INERT/.test(observation)));
scripts/gate-verify.mjs
+77 -11
View File
@@ -29,12 +29,14 @@ function parseArgs(argv) {
root: process.cwd(),
manifest: 'gates/gates.manifest.json',
skipHistory: false,
structureOnly: false,
};
for (let index = 0; index < argv.length; index += 1) {
const value = argv[index];
if (value === '--root') options.root = path.resolve(argv[++index]);
else if (value === '--manifest') options.manifest = argv[++index];
else if (value === '--skip-history') options.skipHistory = true;
else if (value === '--structure-only') options.structureOnly = true;
else throw new Error(`unknown option: ${value}`);
}
return options;
@@ -307,7 +309,39 @@ function validateClosedSchema(manifest, failures) {
failures,
);
rejectDuplicateIds(manifest.criteria, 'criterion', failures);
for (const criterion of manifest.criteria ?? []) {
rejectUnknownKeys(
criterion,
new Set([
'id',
'originalText',
'currentText',
'claimType',
'source',
'meaningChanges',
'caseRefs',
]),
`criterion ${criterion.id}`,
failures,
);
if (!Array.isArray(criterion.caseRefs) || criterion.caseRefs.length === 0) {
failures.push(`${criterion.id}: no declared exercising cases`);
} else if (new Set(criterion.caseRefs).size !== criterion.caseRefs.length) {
failures.push(`${criterion.id}: duplicate declared exercising case`);
}
}
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
for (const claim of manifest.proseClaims ?? []) {
rejectUnknownKeys(
claim,
new Set(['id', 'criterionId', 'caseRef']),
`prose claim ${claim.id}`,
failures,
);
if (typeof claim.caseRef !== 'string' || claim.caseRef.length === 0) {
failures.push(`GATE-CLAIM:${claim.id} has no declared exercising case`);
}
}
rejectDuplicateIds(manifest.compatibilityScenarios, 'compatibility scenario', failures);
for (const scenario of manifest.compatibilityScenarios ?? []) {
rejectUnknownKeys(
@@ -390,7 +424,9 @@ function validateClosedSchema(manifest, failures) {
failures.push(`${gate.id}/${gateCase.id}: case invocation must be non-empty`);
}
if (gateCase.mustFail === true && !gateCase.reasonPattern?.trim()) {
failures.push(`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`);
failures.push(
`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`,
);
}
}
}
@@ -401,6 +437,7 @@ function validateStructure(manifest, failures) {
const criteria = new Map((manifest.criteria ?? []).map((criterion) => [criterion.id, criterion]));
const boundCriteria = new Set();
const negativeBoundCriteria = new Set();
const observedCaseRefs = new Map();
for (const gate of manifest.gates ?? []) {
const negativeCases = (gate.cases ?? []).filter((gateCase) => gateCase.mustFail === true);
@@ -413,12 +450,12 @@ function validateStructure(manifest, failures) {
failures.push(`${gate.id}/${gateCase.id}: unknown criterion ${criterionId}`);
}
boundCriteria.add(criterionId);
const caseRef = `${gate.id}/${gateCase.id}`;
if (!observedCaseRefs.has(criterionId)) observedCaseRefs.set(criterionId, new Set());
observedCaseRefs.get(criterionId).add(caseRef);
if (gateCase.mustFail === true) negativeBoundCriteria.add(criterionId);
}
if (
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
!gateCase.defect?.owner
) {
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && !gateCase.defect?.owner) {
failures.push(`${gate.id}/${gateCase.id}: behavior delta requires a tracked owner`);
}
}
@@ -427,6 +464,18 @@ function validateStructure(manifest, failures) {
for (const claim of manifest.proseClaims ?? []) {
if (!criteria.has(claim.criterionId)) {
failures.push(`GATE-CLAIM:${claim.id} references unknown criterion ${claim.criterionId}`);
continue;
}
const found = findGateCase(manifest, claim.caseRef ?? '');
if (!found) {
failures.push(`GATE-CLAIM:${claim.id} references missing exercising case ${claim.caseRef}`);
} else if (
found.gateCase.mustFail !== true ||
!found.gateCase.criterionIds?.includes(claim.criterionId)
) {
failures.push(
`GATE-CLAIM:${claim.id} exercising case ${claim.caseRef} does not exercise criterion ${claim.criterionId}`,
);
}
}
@@ -435,6 +484,20 @@ function validateStructure(manifest, failures) {
else if (!negativeBoundCriteria.has(criterion.id)) {
failures.push(`${criterion.id}: no must-fail case exercises this criterion`);
}
const declaredRefs = new Set(criterion.caseRefs ?? []);
const actualRefs = observedCaseRefs.get(criterion.id) ?? new Set();
for (const caseRef of declaredRefs) {
const found = findGateCase(manifest, caseRef);
if (!found) failures.push(`${criterion.id}: declared exercising case ${caseRef} is missing`);
else if (!actualRefs.has(caseRef)) {
failures.push(`${criterion.id}: declared exercising case ${caseRef} is not bound`);
}
}
for (const caseRef of actualRefs) {
if (!declaredRefs.has(caseRef)) {
failures.push(`${criterion.id}: bound to undeclared exercising case ${caseRef}`);
}
}
if (
criterion.originalText !== criterion.currentText &&
(!Array.isArray(criterion.meaningChanges) || criterion.meaningChanges.length === 0)
@@ -634,7 +697,9 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
for (const caseRef of scenario.caseRefs ?? []) {
const found = findGateCase(manifest, caseRef);
if (!found) {
failures.push(`${scenario.id}: compatibility construction references missing case ${caseRef}`);
failures.push(
`${scenario.id}: compatibility construction references missing case ${caseRef}`,
);
} else {
referenced.push({ caseRef, ...found });
}
@@ -671,7 +736,10 @@ async function runCompatibilityScenario(root, manifest, scenario, failures, obse
}
await applyFixture(sandbox, fixture);
}
for (const source of [...referenced.map(({ gateCase }) => gateCase.environment), scenario.environment]) {
for (const source of [
...referenced.map(({ gateCase }) => gateCase.environment),
scenario.environment,
]) {
for (const [key, value] of Object.entries(source ?? {})) {
if (environment[key] !== undefined && environment[key] !== value) {
failures.push(`${scenario.id}: incompatible environment values for ${key}`);
@@ -703,6 +771,7 @@ export async function verifyRegistry(options) {
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
validateStructure(manifest, failures);
if (options.structureOnly) return { failures, manifest, observations };
await validateClaims(options.root, manifest, failures);
await validateDiscovery(options.root, manifest, failures);
@@ -719,10 +788,7 @@ export async function verifyRegistry(options) {
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
}
if (
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
gateCase.defect?.owner
) {
if (!structuredValuesEqual(gateCase.required, gateCase.actual) && gateCase.defect?.owner) {
observations.push(
`DEFECT (owner: ${gateCase.defect.owner}) ${gate.id}/${gateCase.id}: required ${JSON.stringify(gateCase.required)}, actual ${JSON.stringify(gateCase.actual)}`,
);
scripts/gate-verify.test.mjs
+71 -3
View File
@@ -30,6 +30,7 @@ function baseManifest() {
claimType: 'integrity',
source: 'fixture',
meaningChanges: [],
caseRefs: ['meta-fixture/rejects-bad-input'],
},
],
compatibilityScenarios: [],
@@ -72,10 +73,18 @@ async function writeManifest(root, manifest) {
await writeFile(path.join(root, 'gates', 'gates.manifest.json'), `${JSON.stringify(manifest)}\n`);
}
function verify(root) {
function verify(root, extraArgs = []) {
return spawnSync(
process.execPath,
[verifier, '--root', root, '--manifest', 'gates/gates.manifest.json', '--skip-history'],
[
verifier,
'--root',
root,
'--manifest',
'gates/gates.manifest.json',
'--skip-history',
...extraArgs,
],
{ cwd: root, encoding: 'utf8', env: { ...process.env, HOME: os.homedir() } },
);
}
@@ -260,6 +269,61 @@ test('a required-versus-actual delta without a tracked owner is rejected', async
assert.match(output(result), /meta-fixture.*delta.*tracked owner/i);
});
test('moving criterion bindings to unrelated cases is rejected', async () => {
const root = await fixture('semantic-misbinding');
await writeGate(root);
const manifest = baseManifest();
manifest.criteria.push({
id: 'META-CRIT-2',
originalText: 'The fixture reports the second rejection reason.',
currentText: 'The fixture reports the second rejection reason.',
claimType: 'integrity',
source: 'fixture',
meaningChanges: [],
caseRefs: ['meta-fixture/rejects-second-input'],
});
manifest.gates[0].cases.push({
...manifest.gates[0].cases[0],
id: 'rejects-second-input',
criterionIds: ['META-CRIT-1'],
});
manifest.gates[0].cases[0].criterionIds = ['META-CRIT-2'];
await writeManifest(root, manifest);
const result = verify(root);
assert.notEqual(result.status, 0);
assert.match(output(result), /META-CRIT-1.*rejects-bad-input.*not bound/i);
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
});
test('moving meaning and prose criteria to an unrelated type error is rejected', async () => {
const root = await fixture('real-manifest-misbinding');
const manifest = JSON.parse(
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
);
for (const gate of manifest.gates) {
for (const gateCase of gate.cases) {
gateCase.criterionIds = gateCase.criterionIds.filter(
(id) => !['RM02-MEANING-PROVENANCE', 'RM02-PROSE-CONTROL'].includes(id),
);
}
}
const typeError = manifest.gates
.find((gate) => gate.id === 'quality-typecheck')
.cases.find((gateCase) => gateCase.id === 'type-error');
typeError.criterionIds.push('RM02-MEANING-PROVENANCE', 'RM02-PROSE-CONTROL');
await writeManifest(root, manifest);
const result = verify(root, ['--structure-only']);
assert.notEqual(result.status, 0);
assert.match(output(result), /RM02-MEANING-PROVENANCE.*missing-meaning-provenance.*not bound/i);
assert.match(output(result), /RM02-PROSE-CONTROL.*prose-claim-misbinding.*not bound/i);
assert.match(
output(result),
/RM02-(?:MEANING-PROVENANCE|PROSE-CONTROL).*undeclared exercising case quality-typecheck\/type-error/i,
);
});
test('a criterion with no bound case is rejected', async () => {
const root = await fixture('unbound-criterion');
await writeGate(root);
@@ -335,6 +399,7 @@ test('compatibility scenarios execute referenced conditions as one construction'
reasonPattern: 'SECOND_REASON',
environment: { SECOND_REASON: 'SECOND_REASON' },
});
manifest.criteria[0].caseRefs.push('meta-fixture/second-condition');
manifest.gates[0].cases[0].fixture = {
writeFiles: [{ path: 'conditions/first', content: 'present\n' }],
};
@@ -427,7 +492,10 @@ test('deployment drift meta-control fails if the shared comparator is made inert
const root = await fixture('deployment-comparator-inert');
await writeGate(root);
await mkdir(path.join(root, 'deployed'), { recursive: true });
await copyFile(path.join(root, 'gates', 'meta-fixture.sh'), path.join(root, 'deployed', 'meta-fixture.sh'));
await copyFile(
path.join(root, 'gates', 'meta-fixture.sh'),
path.join(root, 'deployed', 'meta-fixture.sh'),
);
const manifest = baseManifest();
manifest.gates[0].deployment = { kind: 'file', path: 'deployed/meta-fixture.sh' };
await writeManifest(root, manifest);