748 lines
26 KiB
JavaScript
748 lines
26 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { constants } from 'node:fs';
|
|
import { createHash } from 'node:crypto';
|
|
import {
|
|
access,
|
|
chmod,
|
|
copyFile,
|
|
lstat,
|
|
mkdir,
|
|
mkdtemp,
|
|
open,
|
|
readFile,
|
|
readdir,
|
|
readlink,
|
|
rm,
|
|
symlink,
|
|
writeFile,
|
|
} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
|
|
import { verifyHistory } from './gate-history.mjs';
|
|
|
|
const COPY_SKIP = new Set(['.git', '.mosaic-test-work', '.next', '.turbo', 'coverage', 'dist']);
|
|
|
|
function parseArgs(argv) {
|
|
const options = {
|
|
root: process.cwd(),
|
|
manifest: 'gates/gates.manifest.json',
|
|
skipHistory: false,
|
|
};
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const value = argv[index];
|
|
if (value === '--root') options.root = path.resolve(argv[++index]);
|
|
else if (value === '--manifest') options.manifest = argv[++index];
|
|
else if (value === '--skip-history') options.skipHistory = true;
|
|
else throw new Error(`unknown option: ${value}`);
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function runInvocation(root, invocation, extraEnvironment = {}) {
|
|
if (!Array.isArray(invocation) || invocation.length === 0) {
|
|
return { status: null, stdout: '', stderr: 'missing invocation' };
|
|
}
|
|
return spawnSync(invocation[0], invocation.slice(1), {
|
|
cwd: root,
|
|
encoding: 'utf8',
|
|
env: { ...process.env, GATE_VERIFY: '1', ...extraEnvironment },
|
|
timeout: 300_000,
|
|
});
|
|
}
|
|
|
|
async function sandboxPath(root, relativePath, label) {
|
|
if (typeof relativePath !== 'string' || path.isAbsolute(relativePath)) {
|
|
throw new Error(`${label}: path escapes sandbox (${String(relativePath)})`);
|
|
}
|
|
const resolvedRoot = path.resolve(root);
|
|
const target = path.resolve(resolvedRoot, relativePath);
|
|
if (target !== resolvedRoot && !target.startsWith(`${resolvedRoot}${path.sep}`)) {
|
|
throw new Error(`${label}: path escapes sandbox (${relativePath})`);
|
|
}
|
|
const parts = path.relative(resolvedRoot, path.dirname(target)).split(path.sep).filter(Boolean);
|
|
let current = resolvedRoot;
|
|
for (const part of parts) {
|
|
current = path.join(current, part);
|
|
try {
|
|
if ((await lstat(current)).isSymbolicLink()) {
|
|
throw new Error(`${label}: path crosses symbolic link (${relativePath})`);
|
|
}
|
|
} catch (error) {
|
|
if (error.code === 'ENOENT') break;
|
|
throw error;
|
|
}
|
|
}
|
|
return target;
|
|
}
|
|
|
|
function expand(value, root) {
|
|
return String(value)
|
|
.replaceAll('${ROOT}', root)
|
|
.replaceAll('${HOME}', process.env.HOME ?? '')
|
|
.replaceAll('${PATH}', process.env.PATH ?? '');
|
|
}
|
|
|
|
function normalizedJson(value) {
|
|
if (Array.isArray(value)) return value.map(normalizedJson);
|
|
if (value && typeof value === 'object') {
|
|
return Object.fromEntries(
|
|
Object.keys(value)
|
|
.sort()
|
|
.map((key) => [key, normalizedJson(value[key])]),
|
|
);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function structuredValuesEqual(left, right) {
|
|
return JSON.stringify(normalizedJson(left)) === JSON.stringify(normalizedJson(right));
|
|
}
|
|
|
|
function outcomeMatches(outcome, result) {
|
|
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
|
return (
|
|
result.status === outcome?.exitCode &&
|
|
(!outcome?.outputPattern || new RegExp(outcome.outputPattern, 'm').test(combined)) &&
|
|
(!outcome?.notOutputPattern || !new RegExp(outcome.notOutputPattern, 'm').test(combined))
|
|
);
|
|
}
|
|
|
|
function resultMatches(gateCase, result) {
|
|
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
|
return (
|
|
outcomeMatches(gateCase.actual, result) &&
|
|
(!gateCase.reasonPattern || new RegExp(gateCase.reasonPattern, 'm').test(combined))
|
|
);
|
|
}
|
|
|
|
async function safeSandboxWrite(root, relativePath, contents, label) {
|
|
const target = await sandboxPath(root, relativePath, label);
|
|
await mkdir(path.dirname(target), { recursive: true });
|
|
try {
|
|
if ((await lstat(target)).isSymbolicLink()) {
|
|
throw new Error(`${label}: target is a symbolic link (${relativePath})`);
|
|
}
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
}
|
|
let handle;
|
|
try {
|
|
handle = await open(
|
|
target,
|
|
constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW,
|
|
0o666,
|
|
);
|
|
await handle.writeFile(contents);
|
|
} catch (error) {
|
|
if (error.code === 'ELOOP') {
|
|
throw new Error(`${label}: target is a symbolic link (${relativePath})`);
|
|
}
|
|
throw error;
|
|
} finally {
|
|
await handle?.close();
|
|
}
|
|
return target;
|
|
}
|
|
|
|
async function applyFixture(root, fixture = {}) {
|
|
for (const entry of fixture.writeFiles ?? []) {
|
|
const target = await safeSandboxWrite(root, entry.path, entry.content, 'fixture write');
|
|
if (entry.mode !== undefined) await chmod(target, entry.mode);
|
|
}
|
|
for (const relativePath of fixture.removePaths ?? []) {
|
|
await rm(await sandboxPath(root, relativePath, 'fixture remove'), {
|
|
recursive: true,
|
|
force: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
async function copySandbox(root, destination, selectedPaths) {
|
|
if (!selectedPaths?.length) {
|
|
await copyTree(root, destination);
|
|
return;
|
|
}
|
|
await mkdir(destination, { recursive: true });
|
|
for (const relativePath of selectedPaths) {
|
|
await copyTree(
|
|
await sandboxPath(root, relativePath, 'sandbox copy source'),
|
|
await sandboxPath(destination, relativePath, 'sandbox copy destination'),
|
|
);
|
|
}
|
|
}
|
|
|
|
async function runCase(root, gate, gateCase) {
|
|
let caseRoot = root;
|
|
if (gateCase.fixture) {
|
|
caseRoot = await mkdtemp(path.join(path.dirname(root), `.gate-case-${gate.id}-`));
|
|
await copySandbox(root, caseRoot, gateCase.fixture.copyPaths);
|
|
await applyFixture(caseRoot, gateCase.fixture);
|
|
}
|
|
try {
|
|
const environment = Object.fromEntries(
|
|
Object.entries(gateCase.environment ?? {}).map(([key, value]) => [
|
|
key,
|
|
expand(value, caseRoot),
|
|
]),
|
|
);
|
|
return runInvocation(caseRoot, gateCase.invocation ?? gate.invocation, environment);
|
|
} finally {
|
|
if (caseRoot !== root) await rm(caseRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function copyTree(source, destination) {
|
|
const stats = await lstat(source);
|
|
if (stats.isSymbolicLink()) {
|
|
await symlink(await readlink(source), destination);
|
|
return;
|
|
}
|
|
if (stats.isFile()) {
|
|
await mkdir(path.dirname(destination), { recursive: true });
|
|
await copyFile(source, destination);
|
|
await chmod(destination, stats.mode);
|
|
return;
|
|
}
|
|
if (!stats.isDirectory()) return;
|
|
await mkdir(destination, { recursive: true });
|
|
for (const child of await readdir(source, { withFileTypes: true })) {
|
|
if (COPY_SKIP.has(child.name)) continue;
|
|
const childSource = path.join(source, child.name);
|
|
const childDestination = path.join(destination, child.name);
|
|
if (child.name === 'node_modules') {
|
|
await symlink(childSource, childDestination, 'dir');
|
|
continue;
|
|
}
|
|
await copyTree(childSource, childDestination);
|
|
}
|
|
}
|
|
|
|
async function executableFiles(root, relativeRoot) {
|
|
const base = await sandboxPath(root, relativeRoot, 'gate root');
|
|
const found = [];
|
|
async function walk(current) {
|
|
for (const child of await readdir(current, { withFileTypes: true })) {
|
|
const target = path.join(current, child.name);
|
|
if (child.isDirectory()) await walk(target);
|
|
else if (child.isFile()) {
|
|
try {
|
|
await access(target, constants.X_OK);
|
|
found.push(path.relative(root, target).split(path.sep).join('/'));
|
|
} catch {
|
|
// Non-executable files are not gates for discovery purposes.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
try {
|
|
await walk(base);
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
}
|
|
return found;
|
|
}
|
|
|
|
function rejectUnknownKeys(value, allowed, label, failures) {
|
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
failures.push(`${label}: expected an object`);
|
|
return;
|
|
}
|
|
for (const key of Object.keys(value)) {
|
|
if (!allowed.has(key)) failures.push(`${label}: unknown field ${key}`);
|
|
}
|
|
}
|
|
|
|
function rejectDuplicateIds(values, label, failures) {
|
|
const seen = new Set();
|
|
for (const value of values ?? []) {
|
|
if (typeof value?.id !== 'string' || value.id.length === 0) {
|
|
failures.push(`${label}: missing stable id`);
|
|
} else if (seen.has(value.id)) {
|
|
failures.push(`duplicate ${label} id ${value.id}`);
|
|
} else {
|
|
seen.add(value.id);
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateClosedSchema(manifest, failures) {
|
|
if (manifest.schemaVersion !== 1)
|
|
failures.push(`unsupported schemaVersion ${String(manifest.schemaVersion)}`);
|
|
rejectUnknownKeys(
|
|
manifest,
|
|
new Set([
|
|
'schemaVersion',
|
|
'activationCommit',
|
|
'gateRoots',
|
|
'governingClaimFiles',
|
|
'coverageBoundary',
|
|
'criteria',
|
|
'proseClaims',
|
|
'compatibilityScenarios',
|
|
'mergeAssertions',
|
|
'gates',
|
|
]),
|
|
'manifest',
|
|
failures,
|
|
);
|
|
rejectDuplicateIds(manifest.criteria, 'criterion', failures);
|
|
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
|
|
rejectDuplicateIds(manifest.compatibilityScenarios, 'compatibility scenario', failures);
|
|
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
|
rejectUnknownKeys(
|
|
scenario,
|
|
new Set([
|
|
'id',
|
|
'construction',
|
|
'caseRefs',
|
|
'invocation',
|
|
'expected',
|
|
'environment',
|
|
'fixture',
|
|
]),
|
|
`compatibility scenario ${scenario.id}`,
|
|
failures,
|
|
);
|
|
if (!Array.isArray(scenario.caseRefs) || scenario.caseRefs.length === 0) {
|
|
failures.push(`${scenario.id}: compatibility construction has no referenced conditions`);
|
|
}
|
|
if (!Array.isArray(scenario.invocation) || scenario.invocation.length === 0) {
|
|
failures.push(`${scenario.id}: compatibility construction invocation is missing`);
|
|
}
|
|
if (typeof scenario.expected?.exitCode !== 'number') {
|
|
failures.push(`${scenario.id}: compatibility construction exact expected exit is missing`);
|
|
}
|
|
}
|
|
rejectDuplicateIds(manifest.gates, 'gate', failures);
|
|
for (const gate of manifest.gates ?? []) {
|
|
rejectUnknownKeys(
|
|
gate,
|
|
new Set([
|
|
'id',
|
|
'source',
|
|
'invocation',
|
|
'deployment',
|
|
'inertMutation',
|
|
'cases',
|
|
'discoveryAliases',
|
|
]),
|
|
`gate ${gate.id}`,
|
|
failures,
|
|
);
|
|
rejectDuplicateIds(gate.cases, `case in gate ${gate.id}`, failures);
|
|
if (!Array.isArray(gate.invocation) || gate.invocation.length === 0) {
|
|
failures.push(`${gate.id}: exact invocation is missing`);
|
|
}
|
|
if (!['none', 'file'].includes(gate.deployment?.kind)) {
|
|
failures.push(`${gate.id}: unsupported deployment kind ${String(gate.deployment?.kind)}`);
|
|
}
|
|
for (const gateCase of gate.cases ?? []) {
|
|
rejectUnknownKeys(
|
|
gateCase,
|
|
new Set([
|
|
'id',
|
|
'criterionIds',
|
|
'mustFail',
|
|
'invocation',
|
|
'required',
|
|
'actual',
|
|
'reasonPattern',
|
|
'environment',
|
|
'fixture',
|
|
'defect',
|
|
]),
|
|
`${gate.id}/${gateCase.id}`,
|
|
failures,
|
|
);
|
|
if (
|
|
typeof gateCase.required?.exitCode !== 'number' ||
|
|
typeof gateCase.actual?.exitCode !== 'number'
|
|
) {
|
|
failures.push(
|
|
`${gate.id}/${gateCase.id}: required and actual exact exit codes are mandatory`,
|
|
);
|
|
}
|
|
if (
|
|
gateCase.invocation &&
|
|
(!Array.isArray(gateCase.invocation) || gateCase.invocation.length === 0)
|
|
) {
|
|
failures.push(`${gate.id}/${gateCase.id}: case invocation must be non-empty`);
|
|
}
|
|
if (gateCase.mustFail === true && !gateCase.reasonPattern?.trim()) {
|
|
failures.push(`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateStructure(manifest, failures) {
|
|
validateClosedSchema(manifest, failures);
|
|
const criteria = new Map((manifest.criteria ?? []).map((criterion) => [criterion.id, criterion]));
|
|
const boundCriteria = new Set();
|
|
const negativeBoundCriteria = new Set();
|
|
|
|
for (const gate of manifest.gates ?? []) {
|
|
const negativeCases = (gate.cases ?? []).filter((gateCase) => gateCase.mustFail === true);
|
|
if (negativeCases.length === 0) failures.push(`${gate.id}: no negative control`);
|
|
if (!gate.deployment?.kind) failures.push(`${gate.id}: deployment identity is not declared`);
|
|
|
|
for (const gateCase of gate.cases ?? []) {
|
|
for (const criterionId of gateCase.criterionIds ?? []) {
|
|
if (!criteria.has(criterionId)) {
|
|
failures.push(`${gate.id}/${gateCase.id}: unknown criterion ${criterionId}`);
|
|
}
|
|
boundCriteria.add(criterionId);
|
|
if (gateCase.mustFail === true) negativeBoundCriteria.add(criterionId);
|
|
}
|
|
if (
|
|
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
|
!gateCase.defect?.owner
|
|
) {
|
|
failures.push(`${gate.id}/${gateCase.id}: behavior delta requires a tracked owner`);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const claim of manifest.proseClaims ?? []) {
|
|
if (!criteria.has(claim.criterionId)) {
|
|
failures.push(`GATE-CLAIM:${claim.id} references unknown criterion ${claim.criterionId}`);
|
|
}
|
|
}
|
|
|
|
for (const criterion of criteria.values()) {
|
|
if (!boundCriteria.has(criterion.id)) failures.push(`${criterion.id}: no bound case`);
|
|
else if (!negativeBoundCriteria.has(criterion.id)) {
|
|
failures.push(`${criterion.id}: no must-fail case exercises this criterion`);
|
|
}
|
|
if (
|
|
criterion.originalText !== criterion.currentText &&
|
|
(!Array.isArray(criterion.meaningChanges) || criterion.meaningChanges.length === 0)
|
|
) {
|
|
failures.push(`${criterion.id}: missing meaning-change provenance`);
|
|
}
|
|
}
|
|
|
|
const byConstruction = new Map();
|
|
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
|
const previous = byConstruction.get(scenario.construction);
|
|
if (previous && !structuredValuesEqual(previous.expected, scenario.expected)) {
|
|
failures.push(`${previous.id} and ${scenario.id}: modeled compatibility conflict`);
|
|
} else {
|
|
byConstruction.set(scenario.construction, scenario);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function validateClaims(root, manifest, failures) {
|
|
const registered = new Set((manifest.proseClaims ?? []).map((claim) => claim.id));
|
|
const observed = new Set();
|
|
for (const relativeFile of manifest.governingClaimFiles ?? []) {
|
|
const contents = await readFile(
|
|
await sandboxPath(root, relativeFile, 'governing claim file'),
|
|
'utf8',
|
|
);
|
|
for (const match of contents.matchAll(/GATE-CLAIM:([A-Z0-9][A-Z0-9-]*)/g)) {
|
|
observed.add(match[1]);
|
|
if (!registered.has(match[1])) {
|
|
failures.push(`GATE-CLAIM:${match[1]} is unbound in ${relativeFile}`);
|
|
}
|
|
}
|
|
}
|
|
for (const claimId of registered) {
|
|
if (!observed.has(claimId)) failures.push(`GATE-CLAIM:${claimId} marker is missing`);
|
|
}
|
|
}
|
|
|
|
async function validateDiscovery(root, manifest, failures) {
|
|
const registered = new Set(
|
|
(manifest.gates ?? []).flatMap((gate) => [gate.source, ...(gate.discoveryAliases ?? [])]),
|
|
);
|
|
for (const gateRoot of manifest.gateRoots ?? []) {
|
|
for (const executable of await executableFiles(root, gateRoot)) {
|
|
if (!registered.has(executable)) failures.push(`unregistered gate: ${executable}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function deploymentFilesEqual(sourcePath, deployedPath) {
|
|
const [source, deployed] = await Promise.all([readFile(sourcePath), readFile(deployedPath)]);
|
|
return source.equals(deployed);
|
|
}
|
|
|
|
async function validateDeployment(root, gate, failures, observations) {
|
|
if (gate.deployment?.kind !== 'file') return;
|
|
const sourcePath = await sandboxPath(
|
|
root,
|
|
gate.deployment.source ?? gate.source,
|
|
`${gate.id} deployment source`,
|
|
);
|
|
const deployedPath = path.isAbsolute(expand(gate.deployment.path, root))
|
|
? expand(gate.deployment.path, root)
|
|
: path.resolve(root, expand(gate.deployment.path, root));
|
|
const source = await readFile(sourcePath);
|
|
let deployed;
|
|
try {
|
|
deployed = await readFile(deployedPath);
|
|
} catch (error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
const observedHash = createHash('sha256').update(source).digest('hex');
|
|
if (
|
|
!gate.deployment.unavailableOwner ||
|
|
!gate.deployment.observedSha256 ||
|
|
gate.deployment.observedSha256 !== observedHash
|
|
) {
|
|
failures.push(
|
|
`${gate.id}: deployed counterpart is unavailable and no current tracked observation matches source`,
|
|
);
|
|
return;
|
|
}
|
|
observations.push(
|
|
`DEPLOYED IDENTITY UNAVAILABLE (owner: ${gate.deployment.unavailableOwner}) ${gate.id}: ${deployedPath} is outside this runner; source matches pinned observation ${observedHash}, live equality is not inferred`,
|
|
);
|
|
return;
|
|
}
|
|
if (!(await deploymentFilesEqual(sourcePath, deployedPath))) {
|
|
failures.push(`${gate.id}: source versus deployed copy drift`);
|
|
return;
|
|
}
|
|
|
|
const controlRoot = await mkdtemp(
|
|
path.join(path.dirname(root), `.gate-deployment-control-${gate.id}-`),
|
|
);
|
|
try {
|
|
const alteredPath = path.join(controlRoot, 'deployed-copy');
|
|
await writeFile(
|
|
alteredPath,
|
|
Buffer.concat([deployed, Buffer.from('\n# gate deployment drift control\n')]),
|
|
);
|
|
if (await deploymentFilesEqual(sourcePath, alteredPath)) {
|
|
failures.push(`${gate.id}: deployed-copy drift negative control was ineffective`);
|
|
} else {
|
|
observations.push(`DEPLOYMENT-NEGATIVE-CONTROL ${gate.id}: observed red`);
|
|
}
|
|
} finally {
|
|
await rm(controlRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function validateMutation(root, gate, failures, observations) {
|
|
const mutation = gate.inertMutation;
|
|
if (
|
|
!mutation?.file ||
|
|
typeof mutation.find !== 'string' ||
|
|
typeof mutation.replace !== 'string' ||
|
|
typeof mutation.expected?.exitCode !== 'number'
|
|
) {
|
|
failures.push(`${gate.id}: declared inert mutation is incomplete`);
|
|
return;
|
|
}
|
|
const mutationPath = await sandboxPath(root, mutation.file, `${gate.id} mutation source`);
|
|
let source;
|
|
try {
|
|
source = await readFile(mutationPath, 'utf8');
|
|
} catch (error) {
|
|
failures.push(`${gate.id}: mutation source unreadable: ${error.message}`);
|
|
return;
|
|
}
|
|
const occurrences = source.split(mutation.find).length - 1;
|
|
if (occurrences !== 1) {
|
|
failures.push(
|
|
`${gate.id}: declared inert mutation is stale or ambiguous (${occurrences} matches)`,
|
|
);
|
|
return;
|
|
}
|
|
const targetCase = mutation.caseId
|
|
? (gate.cases ?? []).find((gateCase) => gateCase.id === mutation.caseId)
|
|
: (gate.cases ?? []).find((gateCase) => gateCase.mustFail === true);
|
|
if (!targetCase) {
|
|
failures.push(
|
|
mutation.caseId
|
|
? `${gate.id}: declared mutation case ${mutation.caseId} was not found`
|
|
: `${gate.id}: declared mutation has no must-fail case`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
const sandbox = await mkdtemp(path.join(path.dirname(root), `.gate-sandbox-${gate.id}-`));
|
|
try {
|
|
await copySandbox(root, sandbox, mutation.sandboxFiles);
|
|
await safeSandboxWrite(
|
|
sandbox,
|
|
mutation.file,
|
|
source.replace(mutation.find, mutation.replace),
|
|
`${gate.id} sandbox mutation write`,
|
|
);
|
|
await applyFixture(sandbox, targetCase.fixture);
|
|
const environment = Object.fromEntries(
|
|
Object.entries(targetCase.environment ?? {}).map(([key, value]) => [
|
|
key,
|
|
expand(value, sandbox),
|
|
]),
|
|
);
|
|
const result = runInvocation(sandbox, targetCase.invocation ?? gate.invocation, environment);
|
|
if (result.error || result.signal) {
|
|
failures.push(
|
|
`${gate.id}: mutation execution crashed${result.signal ? ` with ${result.signal}` : ''}${result.error ? `: ${result.error.message}` : ''}`,
|
|
);
|
|
} else if (!outcomeMatches(mutation.expected, result)) {
|
|
failures.push(
|
|
`${gate.id}: mutation produced unexpected outcome ${String(result.status)}; expected ${JSON.stringify(mutation.expected)}`,
|
|
);
|
|
} else if (resultMatches(targetCase, result)) {
|
|
failures.push(`${gate.id}: declared inert mutation was ineffective`);
|
|
} else {
|
|
observations.push(`META-NEGATIVE-CONTROL ${gate.id}: observed red`);
|
|
}
|
|
} finally {
|
|
await rm(sandbox, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function findGateCase(manifest, caseRef) {
|
|
const separator = caseRef.indexOf('/');
|
|
if (separator < 1) return undefined;
|
|
const gateId = caseRef.slice(0, separator);
|
|
const caseId = caseRef.slice(separator + 1);
|
|
const gate = (manifest.gates ?? []).find((candidate) => candidate.id === gateId);
|
|
const gateCase = (gate?.cases ?? []).find((candidate) => candidate.id === caseId);
|
|
return gate && gateCase ? { gate, gateCase } : undefined;
|
|
}
|
|
|
|
async function runCompatibilityScenario(root, manifest, scenario, failures, observations) {
|
|
const referenced = [];
|
|
for (const caseRef of scenario.caseRefs ?? []) {
|
|
const found = findGateCase(manifest, caseRef);
|
|
if (!found) {
|
|
failures.push(`${scenario.id}: compatibility construction references missing case ${caseRef}`);
|
|
} else {
|
|
referenced.push({ caseRef, ...found });
|
|
}
|
|
}
|
|
if (referenced.length !== (scenario.caseRefs ?? []).length) return;
|
|
|
|
const sandbox = await mkdtemp(path.join(path.dirname(root), `.gate-compat-${scenario.id}-`));
|
|
try {
|
|
await copySandbox(root, sandbox);
|
|
const environment = {};
|
|
const writeSignatures = new Map();
|
|
const removedPaths = new Set();
|
|
const fixtures = [...referenced.map(({ gateCase }) => gateCase.fixture), scenario.fixture];
|
|
for (const fixture of fixtures.filter(Boolean)) {
|
|
for (const entry of fixture.writeFiles ?? []) {
|
|
const signature = JSON.stringify({ content: entry.content, mode: entry.mode });
|
|
const previous = writeSignatures.get(entry.path);
|
|
if (previous !== undefined && previous !== signature) {
|
|
failures.push(`${scenario.id}: incompatible fixture writes for ${entry.path}`);
|
|
return;
|
|
}
|
|
if (removedPaths.has(entry.path)) {
|
|
failures.push(`${scenario.id}: fixture both writes and removes ${entry.path}`);
|
|
return;
|
|
}
|
|
writeSignatures.set(entry.path, signature);
|
|
}
|
|
for (const removed of fixture.removePaths ?? []) {
|
|
if (writeSignatures.has(removed)) {
|
|
failures.push(`${scenario.id}: fixture both writes and removes ${removed}`);
|
|
return;
|
|
}
|
|
removedPaths.add(removed);
|
|
}
|
|
await applyFixture(sandbox, fixture);
|
|
}
|
|
for (const source of [...referenced.map(({ gateCase }) => gateCase.environment), scenario.environment]) {
|
|
for (const [key, value] of Object.entries(source ?? {})) {
|
|
if (environment[key] !== undefined && environment[key] !== value) {
|
|
failures.push(`${scenario.id}: incompatible environment values for ${key}`);
|
|
return;
|
|
}
|
|
environment[key] = value;
|
|
}
|
|
}
|
|
const expandedEnvironment = Object.fromEntries(
|
|
Object.entries(environment).map(([key, value]) => [key, expand(value, sandbox)]),
|
|
);
|
|
const result = runInvocation(sandbox, scenario.invocation, expandedEnvironment);
|
|
if (result.error || result.signal || !outcomeMatches(scenario.expected, result)) {
|
|
failures.push(
|
|
`${scenario.id}: combined compatibility construction ${scenario.construction} observed ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''}; expected ${JSON.stringify(scenario.expected)}`,
|
|
);
|
|
} else {
|
|
observations.push(`COMPATIBILITY ${scenario.id}: observed expected outcome`);
|
|
}
|
|
} finally {
|
|
await rm(sandbox, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
export async function verifyRegistry(options) {
|
|
const failures = [];
|
|
const observations = [];
|
|
const manifestPath = path.resolve(options.root, options.manifest);
|
|
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
|
|
|
validateStructure(manifest, failures);
|
|
await validateClaims(options.root, manifest, failures);
|
|
await validateDiscovery(options.root, manifest, failures);
|
|
|
|
for (const gate of manifest.gates ?? []) {
|
|
await validateDeployment(options.root, gate, failures, observations);
|
|
for (const gateCase of gate.cases ?? []) {
|
|
const result = await runCase(options.root, gate, gateCase);
|
|
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
|
if (!outcomeMatches(gateCase.actual, result)) {
|
|
failures.push(
|
|
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
|
|
);
|
|
}
|
|
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
|
|
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
|
|
}
|
|
if (
|
|
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
|
gateCase.defect?.owner
|
|
) {
|
|
observations.push(
|
|
`DEFECT (owner: ${gateCase.defect.owner}) ${gate.id}/${gateCase.id}: required ${JSON.stringify(gateCase.required)}, actual ${JSON.stringify(gateCase.actual)}`,
|
|
);
|
|
}
|
|
}
|
|
await validateMutation(options.root, gate, failures, observations);
|
|
}
|
|
|
|
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
|
await runCompatibilityScenario(options.root, manifest, scenario, failures, observations);
|
|
}
|
|
|
|
return { failures, manifest, observations };
|
|
}
|
|
|
|
async function main() {
|
|
try {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const { failures, observations, manifest } = await verifyRegistry(options);
|
|
if (!options.skipHistory && failures.length === 0) {
|
|
const history = await verifyHistory({ root: options.root, manifest });
|
|
failures.push(...history.failures);
|
|
observations.push(...history.observations);
|
|
}
|
|
for (const observation of observations) process.stdout.write(`${observation}\n`);
|
|
if (failures.length > 0) {
|
|
for (const failure of failures) process.stderr.write(`GATE VERIFY FAILED: ${failure}\n`);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
const defects = observations.filter((line) => line.startsWith('DEFECT ')).length;
|
|
process.stdout.write(
|
|
`registry observations matched; open behavior deltas: ${defects}; required-behavior conformance is not asserted while deltas remain\n`,
|
|
);
|
|
} catch (error) {
|
|
process.stderr.write(`GATE VERIFY FAILED: ${error.message}\n`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
if (import.meta.url === `file://${process.argv[1]}`) await main();
|