This commit is contained in:
@@ -0,0 +1,747 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { constants } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
access,
|
||||
chmod,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
open,
|
||||
readFile,
|
||||
readdir,
|
||||
readlink,
|
||||
rm,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
import { verifyHistory } from './gate-history.mjs';
|
||||
|
||||
const COPY_SKIP = new Set(['.git', '.mosaic-test-work', '.next', '.turbo', 'coverage', 'dist']);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
root: process.cwd(),
|
||||
manifest: 'gates/gates.manifest.json',
|
||||
skipHistory: false,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const value = argv[index];
|
||||
if (value === '--root') options.root = path.resolve(argv[++index]);
|
||||
else if (value === '--manifest') options.manifest = argv[++index];
|
||||
else if (value === '--skip-history') options.skipHistory = true;
|
||||
else throw new Error(`unknown option: ${value}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function runInvocation(root, invocation, extraEnvironment = {}) {
|
||||
if (!Array.isArray(invocation) || invocation.length === 0) {
|
||||
return { status: null, stdout: '', stderr: 'missing invocation' };
|
||||
}
|
||||
return spawnSync(invocation[0], invocation.slice(1), {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, GATE_VERIFY: '1', ...extraEnvironment },
|
||||
timeout: 300_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function sandboxPath(root, relativePath, label) {
|
||||
if (typeof relativePath !== 'string' || path.isAbsolute(relativePath)) {
|
||||
throw new Error(`${label}: path escapes sandbox (${String(relativePath)})`);
|
||||
}
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const target = path.resolve(resolvedRoot, relativePath);
|
||||
if (target !== resolvedRoot && !target.startsWith(`${resolvedRoot}${path.sep}`)) {
|
||||
throw new Error(`${label}: path escapes sandbox (${relativePath})`);
|
||||
}
|
||||
const parts = path.relative(resolvedRoot, path.dirname(target)).split(path.sep).filter(Boolean);
|
||||
let current = resolvedRoot;
|
||||
for (const part of parts) {
|
||||
current = path.join(current, part);
|
||||
try {
|
||||
if ((await lstat(current)).isSymbolicLink()) {
|
||||
throw new Error(`${label}: path crosses symbolic link (${relativePath})`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') break;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
function expand(value, root) {
|
||||
return String(value)
|
||||
.replaceAll('${ROOT}', root)
|
||||
.replaceAll('${HOME}', process.env.HOME ?? '')
|
||||
.replaceAll('${PATH}', process.env.PATH ?? '');
|
||||
}
|
||||
|
||||
function normalizedJson(value) {
|
||||
if (Array.isArray(value)) return value.map(normalizedJson);
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, normalizedJson(value[key])]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function structuredValuesEqual(left, right) {
|
||||
return JSON.stringify(normalizedJson(left)) === JSON.stringify(normalizedJson(right));
|
||||
}
|
||||
|
||||
function outcomeMatches(outcome, result) {
|
||||
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
||||
return (
|
||||
result.status === outcome?.exitCode &&
|
||||
(!outcome?.outputPattern || new RegExp(outcome.outputPattern, 'm').test(combined)) &&
|
||||
(!outcome?.notOutputPattern || !new RegExp(outcome.notOutputPattern, 'm').test(combined))
|
||||
);
|
||||
}
|
||||
|
||||
function resultMatches(gateCase, result) {
|
||||
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
||||
return (
|
||||
outcomeMatches(gateCase.actual, result) &&
|
||||
(!gateCase.reasonPattern || new RegExp(gateCase.reasonPattern, 'm').test(combined))
|
||||
);
|
||||
}
|
||||
|
||||
async function safeSandboxWrite(root, relativePath, contents, label) {
|
||||
const target = await sandboxPath(root, relativePath, label);
|
||||
await mkdir(path.dirname(target), { recursive: true });
|
||||
try {
|
||||
if ((await lstat(target)).isSymbolicLink()) {
|
||||
throw new Error(`${label}: target is a symbolic link (${relativePath})`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
}
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(
|
||||
target,
|
||||
constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW,
|
||||
0o666,
|
||||
);
|
||||
await handle.writeFile(contents);
|
||||
} catch (error) {
|
||||
if (error.code === 'ELOOP') {
|
||||
throw new Error(`${label}: target is a symbolic link (${relativePath})`);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await handle?.close();
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
async function applyFixture(root, fixture = {}) {
|
||||
for (const entry of fixture.writeFiles ?? []) {
|
||||
const target = await safeSandboxWrite(root, entry.path, entry.content, 'fixture write');
|
||||
if (entry.mode !== undefined) await chmod(target, entry.mode);
|
||||
}
|
||||
for (const relativePath of fixture.removePaths ?? []) {
|
||||
await rm(await sandboxPath(root, relativePath, 'fixture remove'), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function copySandbox(root, destination, selectedPaths) {
|
||||
if (!selectedPaths?.length) {
|
||||
await copyTree(root, destination);
|
||||
return;
|
||||
}
|
||||
await mkdir(destination, { recursive: true });
|
||||
for (const relativePath of selectedPaths) {
|
||||
await copyTree(
|
||||
await sandboxPath(root, relativePath, 'sandbox copy source'),
|
||||
await sandboxPath(destination, relativePath, 'sandbox copy destination'),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runCase(root, gate, gateCase) {
|
||||
let caseRoot = root;
|
||||
if (gateCase.fixture) {
|
||||
caseRoot = await mkdtemp(path.join(path.dirname(root), `.gate-case-${gate.id}-`));
|
||||
await copySandbox(root, caseRoot, gateCase.fixture.copyPaths);
|
||||
await applyFixture(caseRoot, gateCase.fixture);
|
||||
}
|
||||
try {
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(gateCase.environment ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
expand(value, caseRoot),
|
||||
]),
|
||||
);
|
||||
return runInvocation(caseRoot, gateCase.invocation ?? gate.invocation, environment);
|
||||
} finally {
|
||||
if (caseRoot !== root) await rm(caseRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function copyTree(source, destination) {
|
||||
const stats = await lstat(source);
|
||||
if (stats.isSymbolicLink()) {
|
||||
await symlink(await readlink(source), destination);
|
||||
return;
|
||||
}
|
||||
if (stats.isFile()) {
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
await copyFile(source, destination);
|
||||
await chmod(destination, stats.mode);
|
||||
return;
|
||||
}
|
||||
if (!stats.isDirectory()) return;
|
||||
await mkdir(destination, { recursive: true });
|
||||
for (const child of await readdir(source, { withFileTypes: true })) {
|
||||
if (COPY_SKIP.has(child.name)) continue;
|
||||
const childSource = path.join(source, child.name);
|
||||
const childDestination = path.join(destination, child.name);
|
||||
if (child.name === 'node_modules') {
|
||||
await symlink(childSource, childDestination, 'dir');
|
||||
continue;
|
||||
}
|
||||
await copyTree(childSource, childDestination);
|
||||
}
|
||||
}
|
||||
|
||||
async function executableFiles(root, relativeRoot) {
|
||||
const base = await sandboxPath(root, relativeRoot, 'gate root');
|
||||
const found = [];
|
||||
async function walk(current) {
|
||||
for (const child of await readdir(current, { withFileTypes: true })) {
|
||||
const target = path.join(current, child.name);
|
||||
if (child.isDirectory()) await walk(target);
|
||||
else if (child.isFile()) {
|
||||
try {
|
||||
await access(target, constants.X_OK);
|
||||
found.push(path.relative(root, target).split(path.sep).join('/'));
|
||||
} catch {
|
||||
// Non-executable files are not gates for discovery purposes.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
await walk(base);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function rejectUnknownKeys(value, allowed, label, failures) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
failures.push(`${label}: expected an object`);
|
||||
return;
|
||||
}
|
||||
for (const key of Object.keys(value)) {
|
||||
if (!allowed.has(key)) failures.push(`${label}: unknown field ${key}`);
|
||||
}
|
||||
}
|
||||
|
||||
function rejectDuplicateIds(values, label, failures) {
|
||||
const seen = new Set();
|
||||
for (const value of values ?? []) {
|
||||
if (typeof value?.id !== 'string' || value.id.length === 0) {
|
||||
failures.push(`${label}: missing stable id`);
|
||||
} else if (seen.has(value.id)) {
|
||||
failures.push(`duplicate ${label} id ${value.id}`);
|
||||
} else {
|
||||
seen.add(value.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateClosedSchema(manifest, failures) {
|
||||
if (manifest.schemaVersion !== 1)
|
||||
failures.push(`unsupported schemaVersion ${String(manifest.schemaVersion)}`);
|
||||
rejectUnknownKeys(
|
||||
manifest,
|
||||
new Set([
|
||||
'schemaVersion',
|
||||
'activationCommit',
|
||||
'gateRoots',
|
||||
'governingClaimFiles',
|
||||
'coverageBoundary',
|
||||
'criteria',
|
||||
'proseClaims',
|
||||
'compatibilityScenarios',
|
||||
'mergeAssertions',
|
||||
'gates',
|
||||
]),
|
||||
'manifest',
|
||||
failures,
|
||||
);
|
||||
rejectDuplicateIds(manifest.criteria, 'criterion', failures);
|
||||
rejectDuplicateIds(manifest.proseClaims, 'prose claim', failures);
|
||||
rejectDuplicateIds(manifest.compatibilityScenarios, 'compatibility scenario', failures);
|
||||
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
||||
rejectUnknownKeys(
|
||||
scenario,
|
||||
new Set([
|
||||
'id',
|
||||
'construction',
|
||||
'caseRefs',
|
||||
'invocation',
|
||||
'expected',
|
||||
'environment',
|
||||
'fixture',
|
||||
]),
|
||||
`compatibility scenario ${scenario.id}`,
|
||||
failures,
|
||||
);
|
||||
if (!Array.isArray(scenario.caseRefs) || scenario.caseRefs.length === 0) {
|
||||
failures.push(`${scenario.id}: compatibility construction has no referenced conditions`);
|
||||
}
|
||||
if (!Array.isArray(scenario.invocation) || scenario.invocation.length === 0) {
|
||||
failures.push(`${scenario.id}: compatibility construction invocation is missing`);
|
||||
}
|
||||
if (typeof scenario.expected?.exitCode !== 'number') {
|
||||
failures.push(`${scenario.id}: compatibility construction exact expected exit is missing`);
|
||||
}
|
||||
}
|
||||
rejectDuplicateIds(manifest.gates, 'gate', failures);
|
||||
for (const gate of manifest.gates ?? []) {
|
||||
rejectUnknownKeys(
|
||||
gate,
|
||||
new Set([
|
||||
'id',
|
||||
'source',
|
||||
'invocation',
|
||||
'deployment',
|
||||
'inertMutation',
|
||||
'cases',
|
||||
'discoveryAliases',
|
||||
]),
|
||||
`gate ${gate.id}`,
|
||||
failures,
|
||||
);
|
||||
rejectDuplicateIds(gate.cases, `case in gate ${gate.id}`, failures);
|
||||
if (!Array.isArray(gate.invocation) || gate.invocation.length === 0) {
|
||||
failures.push(`${gate.id}: exact invocation is missing`);
|
||||
}
|
||||
if (!['none', 'file'].includes(gate.deployment?.kind)) {
|
||||
failures.push(`${gate.id}: unsupported deployment kind ${String(gate.deployment?.kind)}`);
|
||||
}
|
||||
for (const gateCase of gate.cases ?? []) {
|
||||
rejectUnknownKeys(
|
||||
gateCase,
|
||||
new Set([
|
||||
'id',
|
||||
'criterionIds',
|
||||
'mustFail',
|
||||
'invocation',
|
||||
'required',
|
||||
'actual',
|
||||
'reasonPattern',
|
||||
'environment',
|
||||
'fixture',
|
||||
'defect',
|
||||
]),
|
||||
`${gate.id}/${gateCase.id}`,
|
||||
failures,
|
||||
);
|
||||
if (
|
||||
typeof gateCase.required?.exitCode !== 'number' ||
|
||||
typeof gateCase.actual?.exitCode !== 'number'
|
||||
) {
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: required and actual exact exit codes are mandatory`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
gateCase.invocation &&
|
||||
(!Array.isArray(gateCase.invocation) || gateCase.invocation.length === 0)
|
||||
) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: case invocation must be non-empty`);
|
||||
}
|
||||
if (gateCase.mustFail === true && !gateCase.reasonPattern?.trim()) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: must-fail case requires a non-empty reasonPattern`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateStructure(manifest, failures) {
|
||||
validateClosedSchema(manifest, failures);
|
||||
const criteria = new Map((manifest.criteria ?? []).map((criterion) => [criterion.id, criterion]));
|
||||
const boundCriteria = new Set();
|
||||
const negativeBoundCriteria = new Set();
|
||||
|
||||
for (const gate of manifest.gates ?? []) {
|
||||
const negativeCases = (gate.cases ?? []).filter((gateCase) => gateCase.mustFail === true);
|
||||
if (negativeCases.length === 0) failures.push(`${gate.id}: no negative control`);
|
||||
if (!gate.deployment?.kind) failures.push(`${gate.id}: deployment identity is not declared`);
|
||||
|
||||
for (const gateCase of gate.cases ?? []) {
|
||||
for (const criterionId of gateCase.criterionIds ?? []) {
|
||||
if (!criteria.has(criterionId)) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: unknown criterion ${criterionId}`);
|
||||
}
|
||||
boundCriteria.add(criterionId);
|
||||
if (gateCase.mustFail === true) negativeBoundCriteria.add(criterionId);
|
||||
}
|
||||
if (
|
||||
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
||||
!gateCase.defect?.owner
|
||||
) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: behavior delta requires a tracked owner`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const claim of manifest.proseClaims ?? []) {
|
||||
if (!criteria.has(claim.criterionId)) {
|
||||
failures.push(`GATE-CLAIM:${claim.id} references unknown criterion ${claim.criterionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const criterion of criteria.values()) {
|
||||
if (!boundCriteria.has(criterion.id)) failures.push(`${criterion.id}: no bound case`);
|
||||
else if (!negativeBoundCriteria.has(criterion.id)) {
|
||||
failures.push(`${criterion.id}: no must-fail case exercises this criterion`);
|
||||
}
|
||||
if (
|
||||
criterion.originalText !== criterion.currentText &&
|
||||
(!Array.isArray(criterion.meaningChanges) || criterion.meaningChanges.length === 0)
|
||||
) {
|
||||
failures.push(`${criterion.id}: missing meaning-change provenance`);
|
||||
}
|
||||
}
|
||||
|
||||
const byConstruction = new Map();
|
||||
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
||||
const previous = byConstruction.get(scenario.construction);
|
||||
if (previous && !structuredValuesEqual(previous.expected, scenario.expected)) {
|
||||
failures.push(`${previous.id} and ${scenario.id}: modeled compatibility conflict`);
|
||||
} else {
|
||||
byConstruction.set(scenario.construction, scenario);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function validateClaims(root, manifest, failures) {
|
||||
const registered = new Set((manifest.proseClaims ?? []).map((claim) => claim.id));
|
||||
const observed = new Set();
|
||||
for (const relativeFile of manifest.governingClaimFiles ?? []) {
|
||||
const contents = await readFile(
|
||||
await sandboxPath(root, relativeFile, 'governing claim file'),
|
||||
'utf8',
|
||||
);
|
||||
for (const match of contents.matchAll(/GATE-CLAIM:([A-Z0-9][A-Z0-9-]*)/g)) {
|
||||
observed.add(match[1]);
|
||||
if (!registered.has(match[1])) {
|
||||
failures.push(`GATE-CLAIM:${match[1]} is unbound in ${relativeFile}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const claimId of registered) {
|
||||
if (!observed.has(claimId)) failures.push(`GATE-CLAIM:${claimId} marker is missing`);
|
||||
}
|
||||
}
|
||||
|
||||
async function validateDiscovery(root, manifest, failures) {
|
||||
const registered = new Set(
|
||||
(manifest.gates ?? []).flatMap((gate) => [gate.source, ...(gate.discoveryAliases ?? [])]),
|
||||
);
|
||||
for (const gateRoot of manifest.gateRoots ?? []) {
|
||||
for (const executable of await executableFiles(root, gateRoot)) {
|
||||
if (!registered.has(executable)) failures.push(`unregistered gate: ${executable}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function deploymentFilesEqual(sourcePath, deployedPath) {
|
||||
const [source, deployed] = await Promise.all([readFile(sourcePath), readFile(deployedPath)]);
|
||||
return source.equals(deployed);
|
||||
}
|
||||
|
||||
async function validateDeployment(root, gate, failures, observations) {
|
||||
if (gate.deployment?.kind !== 'file') return;
|
||||
const sourcePath = await sandboxPath(
|
||||
root,
|
||||
gate.deployment.source ?? gate.source,
|
||||
`${gate.id} deployment source`,
|
||||
);
|
||||
const deployedPath = path.isAbsolute(expand(gate.deployment.path, root))
|
||||
? expand(gate.deployment.path, root)
|
||||
: path.resolve(root, expand(gate.deployment.path, root));
|
||||
const source = await readFile(sourcePath);
|
||||
let deployed;
|
||||
try {
|
||||
deployed = await readFile(deployedPath);
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
const observedHash = createHash('sha256').update(source).digest('hex');
|
||||
if (
|
||||
!gate.deployment.unavailableOwner ||
|
||||
!gate.deployment.observedSha256 ||
|
||||
gate.deployment.observedSha256 !== observedHash
|
||||
) {
|
||||
failures.push(
|
||||
`${gate.id}: deployed counterpart is unavailable and no current tracked observation matches source`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
observations.push(
|
||||
`DEPLOYED IDENTITY UNAVAILABLE (owner: ${gate.deployment.unavailableOwner}) ${gate.id}: ${deployedPath} is outside this runner; source matches pinned observation ${observedHash}, live equality is not inferred`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!(await deploymentFilesEqual(sourcePath, deployedPath))) {
|
||||
failures.push(`${gate.id}: source versus deployed copy drift`);
|
||||
return;
|
||||
}
|
||||
|
||||
const controlRoot = await mkdtemp(
|
||||
path.join(path.dirname(root), `.gate-deployment-control-${gate.id}-`),
|
||||
);
|
||||
try {
|
||||
const alteredPath = path.join(controlRoot, 'deployed-copy');
|
||||
await writeFile(
|
||||
alteredPath,
|
||||
Buffer.concat([deployed, Buffer.from('\n# gate deployment drift control\n')]),
|
||||
);
|
||||
if (await deploymentFilesEqual(sourcePath, alteredPath)) {
|
||||
failures.push(`${gate.id}: deployed-copy drift negative control was ineffective`);
|
||||
} else {
|
||||
observations.push(`DEPLOYMENT-NEGATIVE-CONTROL ${gate.id}: observed red`);
|
||||
}
|
||||
} finally {
|
||||
await rm(controlRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function validateMutation(root, gate, failures, observations) {
|
||||
const mutation = gate.inertMutation;
|
||||
if (
|
||||
!mutation?.file ||
|
||||
typeof mutation.find !== 'string' ||
|
||||
typeof mutation.replace !== 'string' ||
|
||||
typeof mutation.expected?.exitCode !== 'number'
|
||||
) {
|
||||
failures.push(`${gate.id}: declared inert mutation is incomplete`);
|
||||
return;
|
||||
}
|
||||
const mutationPath = await sandboxPath(root, mutation.file, `${gate.id} mutation source`);
|
||||
let source;
|
||||
try {
|
||||
source = await readFile(mutationPath, 'utf8');
|
||||
} catch (error) {
|
||||
failures.push(`${gate.id}: mutation source unreadable: ${error.message}`);
|
||||
return;
|
||||
}
|
||||
const occurrences = source.split(mutation.find).length - 1;
|
||||
if (occurrences !== 1) {
|
||||
failures.push(
|
||||
`${gate.id}: declared inert mutation is stale or ambiguous (${occurrences} matches)`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const targetCase = mutation.caseId
|
||||
? (gate.cases ?? []).find((gateCase) => gateCase.id === mutation.caseId)
|
||||
: (gate.cases ?? []).find((gateCase) => gateCase.mustFail === true);
|
||||
if (!targetCase) {
|
||||
failures.push(
|
||||
mutation.caseId
|
||||
? `${gate.id}: declared mutation case ${mutation.caseId} was not found`
|
||||
: `${gate.id}: declared mutation has no must-fail case`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const sandbox = await mkdtemp(path.join(path.dirname(root), `.gate-sandbox-${gate.id}-`));
|
||||
try {
|
||||
await copySandbox(root, sandbox, mutation.sandboxFiles);
|
||||
await safeSandboxWrite(
|
||||
sandbox,
|
||||
mutation.file,
|
||||
source.replace(mutation.find, mutation.replace),
|
||||
`${gate.id} sandbox mutation write`,
|
||||
);
|
||||
await applyFixture(sandbox, targetCase.fixture);
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(targetCase.environment ?? {}).map(([key, value]) => [
|
||||
key,
|
||||
expand(value, sandbox),
|
||||
]),
|
||||
);
|
||||
const result = runInvocation(sandbox, targetCase.invocation ?? gate.invocation, environment);
|
||||
if (result.error || result.signal) {
|
||||
failures.push(
|
||||
`${gate.id}: mutation execution crashed${result.signal ? ` with ${result.signal}` : ''}${result.error ? `: ${result.error.message}` : ''}`,
|
||||
);
|
||||
} else if (!outcomeMatches(mutation.expected, result)) {
|
||||
failures.push(
|
||||
`${gate.id}: mutation produced unexpected outcome ${String(result.status)}; expected ${JSON.stringify(mutation.expected)}`,
|
||||
);
|
||||
} else if (resultMatches(targetCase, result)) {
|
||||
failures.push(`${gate.id}: declared inert mutation was ineffective`);
|
||||
} else {
|
||||
observations.push(`META-NEGATIVE-CONTROL ${gate.id}: observed red`);
|
||||
}
|
||||
} finally {
|
||||
await rm(sandbox, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function findGateCase(manifest, caseRef) {
|
||||
const separator = caseRef.indexOf('/');
|
||||
if (separator < 1) return undefined;
|
||||
const gateId = caseRef.slice(0, separator);
|
||||
const caseId = caseRef.slice(separator + 1);
|
||||
const gate = (manifest.gates ?? []).find((candidate) => candidate.id === gateId);
|
||||
const gateCase = (gate?.cases ?? []).find((candidate) => candidate.id === caseId);
|
||||
return gate && gateCase ? { gate, gateCase } : undefined;
|
||||
}
|
||||
|
||||
async function runCompatibilityScenario(root, manifest, scenario, failures, observations) {
|
||||
const referenced = [];
|
||||
for (const caseRef of scenario.caseRefs ?? []) {
|
||||
const found = findGateCase(manifest, caseRef);
|
||||
if (!found) {
|
||||
failures.push(`${scenario.id}: compatibility construction references missing case ${caseRef}`);
|
||||
} else {
|
||||
referenced.push({ caseRef, ...found });
|
||||
}
|
||||
}
|
||||
if (referenced.length !== (scenario.caseRefs ?? []).length) return;
|
||||
|
||||
const sandbox = await mkdtemp(path.join(path.dirname(root), `.gate-compat-${scenario.id}-`));
|
||||
try {
|
||||
await copySandbox(root, sandbox);
|
||||
const environment = {};
|
||||
const writeSignatures = new Map();
|
||||
const removedPaths = new Set();
|
||||
const fixtures = [...referenced.map(({ gateCase }) => gateCase.fixture), scenario.fixture];
|
||||
for (const fixture of fixtures.filter(Boolean)) {
|
||||
for (const entry of fixture.writeFiles ?? []) {
|
||||
const signature = JSON.stringify({ content: entry.content, mode: entry.mode });
|
||||
const previous = writeSignatures.get(entry.path);
|
||||
if (previous !== undefined && previous !== signature) {
|
||||
failures.push(`${scenario.id}: incompatible fixture writes for ${entry.path}`);
|
||||
return;
|
||||
}
|
||||
if (removedPaths.has(entry.path)) {
|
||||
failures.push(`${scenario.id}: fixture both writes and removes ${entry.path}`);
|
||||
return;
|
||||
}
|
||||
writeSignatures.set(entry.path, signature);
|
||||
}
|
||||
for (const removed of fixture.removePaths ?? []) {
|
||||
if (writeSignatures.has(removed)) {
|
||||
failures.push(`${scenario.id}: fixture both writes and removes ${removed}`);
|
||||
return;
|
||||
}
|
||||
removedPaths.add(removed);
|
||||
}
|
||||
await applyFixture(sandbox, fixture);
|
||||
}
|
||||
for (const source of [...referenced.map(({ gateCase }) => gateCase.environment), scenario.environment]) {
|
||||
for (const [key, value] of Object.entries(source ?? {})) {
|
||||
if (environment[key] !== undefined && environment[key] !== value) {
|
||||
failures.push(`${scenario.id}: incompatible environment values for ${key}`);
|
||||
return;
|
||||
}
|
||||
environment[key] = value;
|
||||
}
|
||||
}
|
||||
const expandedEnvironment = Object.fromEntries(
|
||||
Object.entries(environment).map(([key, value]) => [key, expand(value, sandbox)]),
|
||||
);
|
||||
const result = runInvocation(sandbox, scenario.invocation, expandedEnvironment);
|
||||
if (result.error || result.signal || !outcomeMatches(scenario.expected, result)) {
|
||||
failures.push(
|
||||
`${scenario.id}: combined compatibility construction ${scenario.construction} observed ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''}; expected ${JSON.stringify(scenario.expected)}`,
|
||||
);
|
||||
} else {
|
||||
observations.push(`COMPATIBILITY ${scenario.id}: observed expected outcome`);
|
||||
}
|
||||
} finally {
|
||||
await rm(sandbox, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyRegistry(options) {
|
||||
const failures = [];
|
||||
const observations = [];
|
||||
const manifestPath = path.resolve(options.root, options.manifest);
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||||
|
||||
validateStructure(manifest, failures);
|
||||
await validateClaims(options.root, manifest, failures);
|
||||
await validateDiscovery(options.root, manifest, failures);
|
||||
|
||||
for (const gate of manifest.gates ?? []) {
|
||||
await validateDeployment(options.root, gate, failures, observations);
|
||||
for (const gateCase of gate.cases ?? []) {
|
||||
const result = await runCase(options.root, gate, gateCase);
|
||||
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
||||
if (!outcomeMatches(gateCase.actual, result)) {
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
|
||||
);
|
||||
}
|
||||
if (gateCase.reasonPattern && !new RegExp(gateCase.reasonPattern, 'm').test(combined)) {
|
||||
failures.push(`${gate.id}/${gateCase.id}: did not fail for its stated reason`);
|
||||
}
|
||||
if (
|
||||
!structuredValuesEqual(gateCase.required, gateCase.actual) &&
|
||||
gateCase.defect?.owner
|
||||
) {
|
||||
observations.push(
|
||||
`DEFECT (owner: ${gateCase.defect.owner}) ${gate.id}/${gateCase.id}: required ${JSON.stringify(gateCase.required)}, actual ${JSON.stringify(gateCase.actual)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await validateMutation(options.root, gate, failures, observations);
|
||||
}
|
||||
|
||||
for (const scenario of manifest.compatibilityScenarios ?? []) {
|
||||
await runCompatibilityScenario(options.root, manifest, scenario, failures, observations);
|
||||
}
|
||||
|
||||
return { failures, manifest, observations };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const { failures, observations, manifest } = await verifyRegistry(options);
|
||||
if (!options.skipHistory && failures.length === 0) {
|
||||
const history = await verifyHistory({ root: options.root, manifest });
|
||||
failures.push(...history.failures);
|
||||
observations.push(...history.observations);
|
||||
}
|
||||
for (const observation of observations) process.stdout.write(`${observation}\n`);
|
||||
if (failures.length > 0) {
|
||||
for (const failure of failures) process.stderr.write(`GATE VERIFY FAILED: ${failure}\n`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const defects = observations.filter((line) => line.startsWith('DEFECT ')).length;
|
||||
process.stdout.write(
|
||||
`registry observations matched; open behavior deltas: ${defects}; required-behavior conformance is not asserted while deltas remain\n`,
|
||||
);
|
||||
} catch (error) {
|
||||
process.stderr.write(`GATE VERIFY FAILED: ${error.message}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) await main();
|
||||
Reference in New Issue
Block a user