784 lines
28 KiB
JavaScript
784 lines
28 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { chmod, copyFile, mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import test from 'node:test';
|
|
|
|
const verifier = path.join(process.cwd(), 'scripts', 'gate-verify.mjs');
|
|
const fixtureRunner = path.join(
|
|
process.cwd(),
|
|
'scripts',
|
|
'test-support',
|
|
'gate-verify-fixture-runner.mjs',
|
|
);
|
|
const fixtureBase = path.join(process.cwd(), '.mosaic-test-work', `gate-verify-${process.pid}`);
|
|
|
|
async function fixture(name = 'case') {
|
|
const root = path.join(fixtureBase, name);
|
|
await rm(root, { recursive: true, force: true });
|
|
await mkdir(path.join(root, 'gates'), { recursive: true });
|
|
return root;
|
|
}
|
|
|
|
function baseManifest() {
|
|
return {
|
|
schemaVersion: 1,
|
|
gateRoots: ['gates'],
|
|
governingClaimFiles: [],
|
|
coverageBoundary: { included: ['meta fixture'], excluded: [], trackedBy: 'RM-54' },
|
|
criteria: [
|
|
{
|
|
id: 'META-CRIT-1',
|
|
originalText: 'The fixture rejects its bad input.',
|
|
currentText: 'The fixture rejects its bad input.',
|
|
claimType: 'integrity',
|
|
source: 'fixture',
|
|
meaningChanges: [],
|
|
caseRefs: ['meta-fixture/rejects-bad-input'],
|
|
},
|
|
],
|
|
compatibilityScenarios: [],
|
|
proseClaims: [],
|
|
gates: [
|
|
{
|
|
id: 'meta-fixture',
|
|
source: 'gates/meta-fixture.sh',
|
|
evidenceSubject: 'meta-fixture',
|
|
invocation: ['gates/meta-fixture.sh'],
|
|
deployment: { kind: 'none', reason: 'test fixture only' },
|
|
inertMutation: {
|
|
file: 'gates/meta-fixture.sh',
|
|
find: 'exit 7',
|
|
replace: 'exit 0',
|
|
expected: { exitCode: 0 },
|
|
},
|
|
cases: [
|
|
{
|
|
id: 'rejects-bad-input',
|
|
criterionIds: ['META-CRIT-1'],
|
|
mustFail: true,
|
|
invocation: ['gates/meta-fixture.sh'],
|
|
required: { exitCode: 7 },
|
|
actual: { exitCode: 7 },
|
|
reasonPattern: 'META_REJECT',
|
|
},
|
|
],
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
async function writeGate(root, contents = '#!/bin/sh\necho META_REJECT >&2\nexit 7\n') {
|
|
const target = path.join(root, 'gates', 'meta-fixture.sh');
|
|
await writeFile(target, contents);
|
|
await chmod(target, 0o755);
|
|
}
|
|
|
|
async function writeManifest(root, manifest) {
|
|
await writeFile(path.join(root, 'gates', 'gates.manifest.json'), `${JSON.stringify(manifest)}\n`);
|
|
}
|
|
|
|
function verify(root, extraArgs = []) {
|
|
return spawnSync(
|
|
process.execPath,
|
|
[fixtureRunner, '--root', root, '--manifest', 'gates/gates.manifest.json', ...extraArgs],
|
|
{ cwd: root, encoding: 'utf8', env: { ...process.env, HOME: os.homedir() } },
|
|
);
|
|
}
|
|
|
|
function verifyProductionStructure(root, extraArgs = []) {
|
|
return spawnSync(
|
|
process.execPath,
|
|
[
|
|
verifier,
|
|
'--root',
|
|
root,
|
|
'--manifest',
|
|
'gates/gates.manifest.json',
|
|
'--skip-history',
|
|
'--structure-only',
|
|
...extraArgs,
|
|
],
|
|
{ cwd: root, encoding: 'utf8', env: { ...process.env, HOME: os.homedir() } },
|
|
);
|
|
}
|
|
|
|
function output(result) {
|
|
return `${result.stdout}\n${result.stderr}`;
|
|
}
|
|
|
|
test.after(async () => {
|
|
await rm(fixtureBase, { recursive: true, force: true });
|
|
});
|
|
|
|
test('universally quantified registry checks reject empty populations before evaluation', async () => {
|
|
const root = await fixture('empty-registry-populations');
|
|
await mkdir(path.join(root, 'empty-gate-root'), { recursive: true });
|
|
const manifest = baseManifest();
|
|
manifest.gateRoots = ['empty-gate-root'];
|
|
manifest.criteria = [];
|
|
manifest.proseClaims = [];
|
|
manifest.compatibilityScenarios = [];
|
|
manifest.gates = [];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verifyProductionStructure(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /criteria population.*non-empty.*anchored/i);
|
|
assert.match(output(result), /gates population.*non-empty.*anchored/i);
|
|
assert.match(output(result), /proseClaims population.*non-empty.*anchored/i);
|
|
assert.match(output(result), /compatibilityScenarios population.*non-empty.*anchored/i);
|
|
});
|
|
|
|
test('production verifier exposes no fixture-profile population bypass', async () => {
|
|
const root = await fixture('no-production-fixture-profile');
|
|
const result = verifyProductionStructure(root, ['--fixture-profile']);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /unknown option: --fixture-profile/i);
|
|
});
|
|
|
|
test('anchored gate inventory and population criteria cannot shrink together', async () => {
|
|
const source = JSON.parse(
|
|
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
|
|
);
|
|
const root = await fixture('shrunken-gate-population');
|
|
source.gates = source.gates.filter((gate) => gate.id !== 'hook-pre-push');
|
|
for (const criterion of source.criteria) {
|
|
if (criterion.gateRefs) {
|
|
criterion.gateRefs = criterion.gateRefs.filter((gateId) => gateId !== 'hook-pre-push');
|
|
}
|
|
criterion.caseRefs = criterion.caseRefs.filter(
|
|
(caseRef) => !caseRef.startsWith('hook-pre-push/'),
|
|
);
|
|
}
|
|
await writeManifest(root, source);
|
|
|
|
const result = verifyProductionStructure(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /gates population is not anchored.*hook-pre-push/i);
|
|
});
|
|
|
|
test('general population criteria cannot delete their gateRefs binding', async () => {
|
|
const requiredCriteria = [
|
|
'RM02-EVIDENCE-SUBJECT-BINDING',
|
|
'RM02-TYPE-STRICT-SCHEMA',
|
|
'RM02-NONEMPTY-ANCHORED-QUANTIFICATION',
|
|
];
|
|
for (const criterionId of requiredCriteria) {
|
|
const source = JSON.parse(
|
|
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
|
|
);
|
|
const root = await fixture(`missing-gate-refs-${criterionId}`);
|
|
delete source.criteria.find((criterion) => criterion.id === criterionId).gateRefs;
|
|
await writeManifest(root, source);
|
|
const result = verifyProductionStructure(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(
|
|
output(result),
|
|
new RegExp(`${criterionId}.*gateRefs.*required`, 'i'),
|
|
criterionId,
|
|
);
|
|
}
|
|
});
|
|
|
|
test('general population criteria must span every registered gate', async () => {
|
|
const source = JSON.parse(
|
|
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
|
|
);
|
|
const root = await fixture('incomplete-gate-refs');
|
|
source.criteria.find((criterion) => criterion.id === 'RM02-EVIDENCE-SUBJECT-BINDING').gateRefs =
|
|
source.criteria
|
|
.find((criterion) => criterion.id === 'RM02-EVIDENCE-SUBJECT-BINDING')
|
|
.gateRefs.filter((gateId) => gateId !== 'quality-lint');
|
|
await writeManifest(root, source);
|
|
|
|
const result = verifyProductionStructure(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(
|
|
output(result),
|
|
/RM02-EVIDENCE-SUBJECT-BINDING.*gate population binding is missing quality-lint/i,
|
|
);
|
|
});
|
|
|
|
test('an externally inerted failure branch makes verification nonzero and names the gate', async () => {
|
|
const root = await fixture('external-inert');
|
|
await writeGate(root, '#!/bin/sh\nexit 0\n');
|
|
await writeManifest(root, baseManifest());
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture/);
|
|
});
|
|
|
|
test('the verifier applies the declared inert mutation and observes its own control red', async () => {
|
|
const root = await fixture('internal-meta');
|
|
await writeGate(root);
|
|
await writeManifest(root, baseManifest());
|
|
|
|
const result = verify(root);
|
|
assert.equal(result.status, 0, output(result));
|
|
assert.match(output(result), /META-NEGATIVE-CONTROL.*meta-fixture.*observed red/i);
|
|
});
|
|
|
|
test('a mutation crash is rejected instead of counted as an observed-red control', async () => {
|
|
const root = await fixture('mutation-crash');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].inertMutation.replace = 'this is not valid shell (';
|
|
manifest.gates[0].inertMutation.expected = { exitCode: 0 };
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*mutation.*unexpected outcome/i);
|
|
assert.doesNotMatch(output(result), /META-NEGATIVE-CONTROL.*observed red/i);
|
|
});
|
|
|
|
test('a stale declared mutation case id is rejected instead of falling back', async () => {
|
|
const root = await fixture('stale-case-id');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].inertMutation.caseId = 'case-that-does-not-exist';
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*case-that-does-not-exist.*not found/i);
|
|
});
|
|
|
|
test('duplicate stable ids and unsupported schema versions are rejected', async () => {
|
|
const root = await fixture('closed-schema');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.schemaVersion = 99;
|
|
manifest.criteria.push({ ...manifest.criteria[0] });
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /unsupported schemaVersion 99/i);
|
|
assert.match(output(result), /duplicate criterion id META-CRIT-1/i);
|
|
});
|
|
|
|
test('misspelled nested assertion fields are rejected instead of becoming optional', async () => {
|
|
const root = await fixture('nested-schema-typo');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].required.outputPatern = 'META_REJECT';
|
|
manifest.gates[0].cases[0].actual.outputPatern = 'META_REJECT';
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /required.*unknown field outputPatern/i);
|
|
assert.match(output(result), /actual.*unknown field outputPatern/i);
|
|
});
|
|
|
|
test('present outcome patterns cannot be empty assertion bypasses', async () => {
|
|
const root = await fixture('nested-schema-empty-patterns');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].required.outputPattern = '';
|
|
manifest.gates[0].cases[0].actual.notOutputPattern = ' ';
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root, ['--structure-only']);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /required.outputPattern: expected a non-empty pattern/i);
|
|
assert.match(output(result), /actual.notOutputPattern: expected a non-empty pattern/i);
|
|
});
|
|
|
|
test('nested discriminator and comparison fields reject wrong types', async () => {
|
|
const root = await fixture('nested-schema-types');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].mustFail = 'true';
|
|
manifest.gates[0].cases[0].required.exitCode = '7';
|
|
manifest.gates[0].cases[0].actual.outputPattern = 7;
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root, ['--structure-only']);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /mustFail: expected a boolean/i);
|
|
assert.match(output(result), /required.exitCode: expected an integer/i);
|
|
assert.match(output(result), /actual.outputPattern: expected a string/i);
|
|
});
|
|
|
|
test('recursive closed-schema guards reject unknown fields in every nested assertion object', async () => {
|
|
const source = JSON.parse(
|
|
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
|
|
);
|
|
const checkout = source.gates.find((gate) => gate.id === 'checkout-preflight');
|
|
const stale = checkout.cases.find((gateCase) => gateCase.id === 'stale-build-lock');
|
|
const queue = source.gates.find((gate) => gate.id === 'ci-queue-wait');
|
|
const queueCase = queue.cases.find((gateCase) => gateCase.id === 'terminal-success');
|
|
const targets = [
|
|
['coverageBoundary', (manifest) => manifest.coverageBoundary],
|
|
['mergeAssertions', (manifest) => manifest.mergeAssertions],
|
|
[
|
|
'meaningChanges',
|
|
(manifest) =>
|
|
manifest.criteria.find((criterion) => criterion.meaningChanges.length).meaningChanges[0],
|
|
],
|
|
['proseClaims', (manifest) => manifest.proseClaims[0]],
|
|
['compatibility expected', (manifest) => manifest.compatibilityScenarios[0].expected],
|
|
[
|
|
'deployment',
|
|
(manifest) => manifest.gates.find((gate) => gate.id === 'ci-queue-wait').deployment,
|
|
],
|
|
['inertMutation', (manifest) => manifest.gates[0].inertMutation],
|
|
['inert expected', (manifest) => manifest.gates[0].inertMutation.expected],
|
|
['required', (manifest) => manifest.gates[0].cases[0].required],
|
|
['actual', (manifest) => manifest.gates[0].cases[0].actual],
|
|
[
|
|
'fixture',
|
|
(manifest) =>
|
|
manifest.gates
|
|
.find((gate) => gate.id === 'checkout-preflight')
|
|
.cases.find((gateCase) => gateCase.id === 'stale-build-lock').fixture,
|
|
],
|
|
[
|
|
'write entry',
|
|
(manifest) =>
|
|
manifest.gates
|
|
.find((gate) => gate.id === 'checkout-preflight')
|
|
.cases.find((gateCase) => gateCase.id === 'stale-build-lock').fixture.writeFiles[0],
|
|
],
|
|
[
|
|
'replace entry',
|
|
(manifest) =>
|
|
manifest.gates
|
|
.find((gate) => gate.id === 'checkout-preflight')
|
|
.cases.find((gateCase) => gateCase.id === 'criterion-misbinding').fixture.replaceFiles[0],
|
|
],
|
|
[
|
|
'defect',
|
|
(manifest) =>
|
|
manifest.gates
|
|
.find((gate) => gate.id === 'ci-queue-wait')
|
|
.cases.find((gateCase) => gateCase.id === 'terminal-success').defect,
|
|
],
|
|
];
|
|
assert.ok(stale.fixture && queueCase.defect);
|
|
for (const [name, select] of targets) {
|
|
const root = await fixture(`recursive-${name.replaceAll(' ', '-')}`);
|
|
const manifest = structuredClone(source);
|
|
select(manifest).unexpectedNestedField = true;
|
|
await writeManifest(root, manifest);
|
|
const result = verify(root, ['--structure-only']);
|
|
assert.notEqual(result.status, 0, `${name}: ${output(result)}`);
|
|
assert.match(output(result), /unknown field unexpectedNestedField/i, name);
|
|
}
|
|
});
|
|
|
|
test('manifest-controlled fixture paths cannot escape the sandbox', async () => {
|
|
const root = await fixture('path-traversal');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].fixture = {
|
|
writeFiles: [{ path: '../../escaped-by-manifest', content: 'bad' }],
|
|
};
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /path escapes sandbox/i);
|
|
});
|
|
|
|
test('fixture writes reject a final symlink and preserve its outside target', async () => {
|
|
const root = await fixture('final-symlink');
|
|
await writeGate(root);
|
|
const outside = path.join(fixtureBase, 'outside-sentinel');
|
|
await writeFile(outside, 'preserve-me\n');
|
|
const linked = path.join(root, 'linked-sentinel');
|
|
await symlink(outside, linked);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].fixture = {
|
|
writeFiles: [{ path: 'linked-sentinel', content: 'overwritten\n' }],
|
|
};
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /fixture write.*symbolic link/i);
|
|
assert.equal(await readFile(outside, 'utf8'), 'preserve-me\n');
|
|
});
|
|
|
|
test('an executable below a gate root without an entry is rejected', async () => {
|
|
const root = await fixture('unregistered');
|
|
await writeGate(root);
|
|
const extra = path.join(root, 'gates', 'forgotten.sh');
|
|
await writeFile(extra, '#!/bin/sh\nexit 1\n');
|
|
await chmod(extra, 0o755);
|
|
await writeManifest(root, baseManifest());
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /unregistered gate.*forgotten\.sh/i);
|
|
});
|
|
|
|
test('a must-fail case without a reason diagnostic is rejected', async () => {
|
|
const root = await fixture('missing-reason');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].reasonPattern = '';
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*rejects-bad-input.*reasonPattern/i);
|
|
});
|
|
|
|
test('a gate with zero must-fail cases is rejected', async () => {
|
|
const root = await fixture('no-negative');
|
|
await writeGate(root, '#!/bin/sh\nexit 0\n');
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].inertMutation = {
|
|
file: 'gates/meta-fixture.sh',
|
|
find: 'exit 0',
|
|
replace: 'exit 1',
|
|
};
|
|
manifest.gates[0].cases = [
|
|
{
|
|
id: 'positive',
|
|
criterionIds: ['META-CRIT-1'],
|
|
mustFail: false,
|
|
invocation: ['gates/meta-fixture.sh'],
|
|
required: { exitCode: 0 },
|
|
actual: { exitCode: 0 },
|
|
reasonPattern: '',
|
|
},
|
|
];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*no negative control/i);
|
|
});
|
|
|
|
test('reordered but equivalent outcome fields do not create a false behavior delta', async () => {
|
|
const root = await fixture('reordered-outcomes');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases[0].required = { exitCode: 7, outputPattern: 'META_REJECT' };
|
|
manifest.gates[0].cases[0].actual = { outputPattern: 'META_REJECT', exitCode: 7 };
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.equal(result.status, 0, output(result));
|
|
assert.doesNotMatch(output(result), /behavior delta requires|DEFECT \(owner:/);
|
|
});
|
|
|
|
test('a required-versus-actual delta without a tracked owner is rejected', async () => {
|
|
const root = await fixture('ownerless-delta');
|
|
await writeGate(root, '#!/bin/sh\nexit 0\n');
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].inertMutation.find = 'exit 0';
|
|
manifest.gates[0].inertMutation.replace = 'exit 7';
|
|
manifest.gates[0].cases[0].actual.exitCode = 0;
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*delta.*tracked owner/i);
|
|
});
|
|
|
|
test('moving criterion bindings to unrelated cases is rejected', async () => {
|
|
const root = await fixture('semantic-misbinding');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.criteria.push({
|
|
id: 'META-CRIT-2',
|
|
originalText: 'The fixture reports the second rejection reason.',
|
|
currentText: 'The fixture reports the second rejection reason.',
|
|
claimType: 'integrity',
|
|
source: 'fixture',
|
|
meaningChanges: [],
|
|
caseRefs: ['meta-fixture/rejects-second-input'],
|
|
});
|
|
manifest.gates[0].cases.push({
|
|
...manifest.gates[0].cases[0],
|
|
id: 'rejects-second-input',
|
|
criterionIds: ['META-CRIT-1'],
|
|
});
|
|
manifest.gates[0].cases[0].criterionIds = ['META-CRIT-2'];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /META-CRIT-1.*rejects-bad-input.*not bound/i);
|
|
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
|
|
});
|
|
|
|
test('canonical verification preserves binding diagnostics when a case fixture is also stale', async () => {
|
|
const root = await fixture('misbinding-plus-stale-fixture');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.criteria.push({
|
|
id: 'META-CRIT-2',
|
|
originalText: 'The second case rejects its own bad input.',
|
|
currentText: 'The second case rejects its own bad input.',
|
|
claimType: 'integrity',
|
|
source: 'fixture',
|
|
meaningChanges: [],
|
|
caseRefs: ['meta-fixture/rejects-second-input'],
|
|
});
|
|
manifest.gates[0].cases.push({
|
|
...manifest.gates[0].cases[0],
|
|
id: 'rejects-second-input',
|
|
criterionIds: ['META-CRIT-1'],
|
|
});
|
|
manifest.gates[0].cases[0].criterionIds = ['META-CRIT-2'];
|
|
manifest.gates[0].cases[0].fixture = {
|
|
replaceFiles: [
|
|
{
|
|
path: 'gates/meta-fixture.sh',
|
|
find: 'text that is not present',
|
|
replace: 'irrelevant',
|
|
},
|
|
],
|
|
};
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /META-CRIT-1.*rejects-bad-input.*not bound/i);
|
|
assert.match(output(result), /META-CRIT-2.*rejects-second-input.*not bound/i);
|
|
assert.match(output(result), /fixture replace.*stale or ambiguous/i);
|
|
});
|
|
|
|
test('moving meaning and prose criteria to an unrelated type error is rejected', async () => {
|
|
const root = await fixture('real-manifest-misbinding');
|
|
const manifest = JSON.parse(
|
|
await readFile(path.join(process.cwd(), 'gates', 'gates.manifest.json'), 'utf8'),
|
|
);
|
|
for (const gate of manifest.gates) {
|
|
for (const gateCase of gate.cases) {
|
|
gateCase.criterionIds = gateCase.criterionIds.filter(
|
|
(id) => !['RM02-MEANING-PROVENANCE', 'RM02-PROSE-CONTROL'].includes(id),
|
|
);
|
|
}
|
|
}
|
|
const typeError = manifest.gates
|
|
.find((gate) => gate.id === 'quality-typecheck')
|
|
.cases.find((gateCase) => gateCase.id === 'type-error');
|
|
typeError.criterionIds.push('RM02-MEANING-PROVENANCE', 'RM02-PROSE-CONTROL');
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root, ['--structure-only']);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /RM02-MEANING-PROVENANCE.*missing-meaning-provenance.*not bound/i);
|
|
assert.match(output(result), /RM02-PROSE-CONTROL.*prose-claim-misbinding.*not bound/i);
|
|
assert.match(
|
|
output(result),
|
|
/RM02-(?:MEANING-PROVENANCE|PROSE-CONTROL).*undeclared exercising case quality-typecheck\/type-error/i,
|
|
);
|
|
});
|
|
|
|
test('a criterion with no bound case is rejected', async () => {
|
|
const root = await fixture('unbound-criterion');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.criteria.push({
|
|
id: 'ORPHAN',
|
|
originalText: 'This criterion is not exercised.',
|
|
currentText: 'This criterion is not exercised.',
|
|
claimType: 'quality',
|
|
source: 'fixture',
|
|
meaningChanges: [],
|
|
});
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /ORPHAN.*no bound case/i);
|
|
});
|
|
|
|
test('an unbound governing prose marker is rejected', async () => {
|
|
const root = await fixture('unbound-prose');
|
|
await writeGate(root);
|
|
await mkdir(path.join(root, 'docs'), { recursive: true });
|
|
await writeFile(path.join(root, 'docs', 'governing.md'), 'GATE-CLAIM:UNBOUND\n');
|
|
const manifest = baseManifest();
|
|
manifest.governingClaimFiles = ['docs/governing.md'];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /GATE-CLAIM:UNBOUND.*unbound/i);
|
|
});
|
|
|
|
test('directly contradictory modeled scenarios are rejected', async () => {
|
|
const root = await fixture('conflict');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.compatibilityScenarios = [
|
|
{
|
|
id: 'ONE',
|
|
construction: 'same-input',
|
|
caseRefs: ['meta-fixture/rejects-bad-input'],
|
|
invocation: ['sh', '-c', 'exit 0'],
|
|
expected: { exitCode: 0 },
|
|
},
|
|
{
|
|
id: 'TWO',
|
|
construction: 'same-input',
|
|
caseRefs: ['meta-fixture/rejects-bad-input'],
|
|
invocation: ['sh', '-c', 'exit 1'],
|
|
expected: { exitCode: 1 },
|
|
},
|
|
];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /ONE.*TWO.*conflict/i);
|
|
});
|
|
|
|
test('compatibility scenarios execute referenced conditions as one construction', async () => {
|
|
const root = await fixture('combined-compatibility');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].cases.push({
|
|
id: 'second-condition',
|
|
criterionIds: ['META-CRIT-1'],
|
|
mustFail: true,
|
|
invocation: ['sh', '-c', 'echo "$SECOND_REASON" >&2; exit 7'],
|
|
fixture: { writeFiles: [{ path: 'conditions/second', content: 'present\n' }] },
|
|
required: { exitCode: 7 },
|
|
actual: { exitCode: 7 },
|
|
reasonPattern: 'SECOND_REASON',
|
|
environment: { SECOND_REASON: 'SECOND_REASON' },
|
|
});
|
|
manifest.criteria[0].caseRefs.push('meta-fixture/second-condition');
|
|
manifest.gates[0].cases[0].fixture = {
|
|
writeFiles: [{ path: 'conditions/first', content: 'present\n' }],
|
|
};
|
|
manifest.compatibilityScenarios = [
|
|
{
|
|
id: 'BOTH-CONDITIONS',
|
|
construction: 'both-fixtures',
|
|
caseRefs: ['meta-fixture/rejects-bad-input', 'meta-fixture/second-condition'],
|
|
invocation: ['sh', '-c', 'test -f conditions/first && test -f conditions/second'],
|
|
expected: { exitCode: 0 },
|
|
},
|
|
];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.equal(result.status, 0, output(result));
|
|
assert.match(output(result), /COMPATIBILITY BOTH-CONDITIONS: observed expected outcome/i);
|
|
});
|
|
|
|
test('a restated criterion without provenance is rejected', async () => {
|
|
const root = await fixture('provenance');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.criteria[0].currentText = 'The fixture rejects only malformed input.';
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /META-CRIT-1.*meaning-change provenance/i);
|
|
});
|
|
|
|
test('a security criterion bound only to a positive case is unregistered in substance', async () => {
|
|
const root = await fixture('positive-only-criterion');
|
|
await writeGate(root);
|
|
const manifest = baseManifest();
|
|
manifest.criteria.push({
|
|
id: 'POSITIVE-ONLY',
|
|
originalText: 'A security property.',
|
|
currentText: 'A security property.',
|
|
claimType: 'security',
|
|
source: 'fixture',
|
|
meaningChanges: [],
|
|
});
|
|
manifest.gates[0].cases.push({
|
|
id: 'positive-only',
|
|
criterionIds: ['POSITIVE-ONLY'],
|
|
mustFail: false,
|
|
invocation: ['gates/meta-fixture.sh'],
|
|
required: { exitCode: 7 },
|
|
actual: { exitCode: 7 },
|
|
reasonPattern: '',
|
|
});
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /POSITIVE-ONLY.*no must-fail case/i);
|
|
});
|
|
|
|
test('a registered prose claim whose marker is absent is rejected', async () => {
|
|
const root = await fixture('missing-prose-marker');
|
|
await writeGate(root);
|
|
await mkdir(path.join(root, 'docs'), { recursive: true });
|
|
await writeFile(path.join(root, 'docs', 'governing.md'), 'No marker here.\n');
|
|
const manifest = baseManifest();
|
|
manifest.governingClaimFiles = ['docs/governing.md'];
|
|
manifest.proseClaims = [{ id: 'MISSING-MARKER', criterionId: 'META-CRIT-1' }];
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /GATE-CLAIM:MISSING-MARKER.*missing/i);
|
|
});
|
|
|
|
test('source-versus-deployed byte drift is rejected and names the gate', async () => {
|
|
const root = await fixture('deployment-drift');
|
|
await writeGate(root);
|
|
await mkdir(path.join(root, 'deployed'), { recursive: true });
|
|
await writeFile(path.join(root, 'deployed', 'meta-fixture.sh'), '#!/bin/sh\nexit 0\n');
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].deployment = { kind: 'file', path: 'deployed/meta-fixture.sh' };
|
|
await writeManifest(root, manifest);
|
|
|
|
const result = verify(root);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*source.*deployed.*drift/i);
|
|
});
|
|
|
|
test('deployment drift meta-control fails if the shared comparator is made inert', async () => {
|
|
const root = await fixture('deployment-comparator-inert');
|
|
await writeGate(root);
|
|
await mkdir(path.join(root, 'deployed'), { recursive: true });
|
|
await copyFile(
|
|
path.join(root, 'gates', 'meta-fixture.sh'),
|
|
path.join(root, 'deployed', 'meta-fixture.sh'),
|
|
);
|
|
const manifest = baseManifest();
|
|
manifest.gates[0].deployment = { kind: 'file', path: 'deployed/meta-fixture.sh' };
|
|
await writeManifest(root, manifest);
|
|
|
|
const alteredScripts = path.join(root, 'verifier-scripts');
|
|
await mkdir(alteredScripts, { recursive: true });
|
|
const verifierSource = await readFile(verifier, 'utf8');
|
|
const inertSource = verifierSource.replace(
|
|
'return source.equals(deployed);',
|
|
'return true; // deliberate test-only inert comparator',
|
|
);
|
|
assert.notEqual(inertSource, verifierSource, 'shared deployment comparator mutation went stale');
|
|
await writeFile(path.join(alteredScripts, 'gate-verify.mjs'), inertSource);
|
|
await copyFile(
|
|
path.join(process.cwd(), 'scripts', 'gate-history.mjs'),
|
|
path.join(alteredScripts, 'gate-history.mjs'),
|
|
);
|
|
|
|
const result = spawnSync(
|
|
process.execPath,
|
|
[
|
|
path.join(alteredScripts, 'gate-verify.mjs'),
|
|
'--root',
|
|
root,
|
|
'--manifest',
|
|
'gates/gates.manifest.json',
|
|
'--skip-history',
|
|
],
|
|
{ cwd: root, encoding: 'utf8', env: { ...process.env, HOME: os.homedir() } },
|
|
);
|
|
assert.notEqual(result.status, 0);
|
|
assert.match(output(result), /meta-fixture.*drift negative control was ineffective/i);
|
|
});
|