wip(rm-02): round-4 remediation held at RM-60 boundary
This commit is contained in:
+111
-34
@@ -20,8 +20,6 @@ import {
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
import { verifyHistory } from './gate-history.mjs';
|
||||
|
||||
const COPY_SKIP = new Set(['.git', '.mosaic-test-work', '.next', '.turbo', 'coverage', 'dist']);
|
||||
const POPULATION_CRITERION_IDS = new Set([
|
||||
'RM02-EVIDENCE-SUBJECT-BINDING',
|
||||
@@ -42,14 +40,12 @@ function parseArgs(argv) {
|
||||
const options = {
|
||||
root: process.cwd(),
|
||||
manifest: 'gates/gates.manifest.json',
|
||||
skipHistory: false,
|
||||
structureOnly: false,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const value = argv[index];
|
||||
if (value === '--root') options.root = path.resolve(argv[++index]);
|
||||
else if (value === '--manifest') options.manifest = argv[++index];
|
||||
else if (value === '--skip-history') options.skipHistory = true;
|
||||
else if (value === '--structure-only') options.structureOnly = true;
|
||||
else throw new Error(`unknown option: ${value}`);
|
||||
}
|
||||
@@ -222,7 +218,8 @@ async function runCase(root, gate, gateCase) {
|
||||
expand(value, caseRoot),
|
||||
]),
|
||||
);
|
||||
return runInvocation(caseRoot, gateCase.invocation ?? gate.invocation, environment);
|
||||
const result = runInvocation(caseRoot, gateCase.invocation ?? gate.invocation, environment);
|
||||
return { ...result, evidence: structuredClone(gateCase.evidence) };
|
||||
} finally {
|
||||
if (caseRoot !== root) await rm(caseRoot, { recursive: true, force: true });
|
||||
}
|
||||
@@ -396,7 +393,36 @@ function validateEnvironment(environment, label, failures) {
|
||||
}
|
||||
}
|
||||
|
||||
function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {}) {
|
||||
function inventoriesEqual(left, right) {
|
||||
return structuredValuesEqual([...left.entries()], [...right.entries()]);
|
||||
}
|
||||
|
||||
export function consumeEvidenceSubject(gate, evidence) {
|
||||
if (evidence?.subject !== gate.id) {
|
||||
return `gate ${gate.id}: consumed evidence subject ${String(evidence?.subject)} does not match gate definition`;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const CURRENT_TREE_OBSERVATION_PATTERNS = [
|
||||
/^META-NEGATIVE-CONTROL /,
|
||||
/^DEPLOYMENT-NEGATIVE-CONTROL /,
|
||||
/^DEPLOYED IDENTITY UNAVAILABLE /,
|
||||
/^DEFECT /,
|
||||
/^COMPATIBILITY /,
|
||||
];
|
||||
|
||||
export function assertCurrentTreeObservation(observation) {
|
||||
if (!CURRENT_TREE_OBSERVATION_PATTERNS.some((pattern) => pattern.test(observation))) {
|
||||
throw new Error(`unsupported observation class: ${observation}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateClosedSchema(
|
||||
manifest,
|
||||
failures,
|
||||
{ fixtureProfile = false, requiredGateInventory } = {},
|
||||
) {
|
||||
if (!fixtureProfile) {
|
||||
for (const population of ['criteria', 'gates', 'proseClaims', 'compatibilityScenarios']) {
|
||||
if (!Array.isArray(manifest[population]) || manifest[population].length === 0) {
|
||||
@@ -440,17 +466,11 @@ function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {
|
||||
if (manifest.mergeAssertions !== undefined) {
|
||||
rejectUnknownKeys(
|
||||
manifest.mergeAssertions,
|
||||
new Set([
|
||||
'mode',
|
||||
'providerEvidence',
|
||||
'deferredReplayOwner',
|
||||
'trustDependencies',
|
||||
'postMergeResponse',
|
||||
]),
|
||||
new Set(['mode', 'deferredReplayOwner', 'trustDependencies', 'postMergeResponse']),
|
||||
'mergeAssertions',
|
||||
failures,
|
||||
);
|
||||
for (const key of ['mode', 'providerEvidence', 'deferredReplayOwner', 'postMergeResponse']) {
|
||||
for (const key of ['mode', 'deferredReplayOwner', 'postMergeResponse']) {
|
||||
requireString(manifest.mergeAssertions?.[key], `mergeAssertions.${key}`, failures);
|
||||
}
|
||||
validateStringArray(
|
||||
@@ -564,12 +584,32 @@ function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {
|
||||
}
|
||||
rejectDuplicateIds(manifest.gates, 'gate', failures);
|
||||
if (!fixtureProfile) {
|
||||
for (const [requiredId, requiredSource] of REQUIRED_GATE_INVENTORY) {
|
||||
const registered = (manifest.gates ?? []).find((gate) => gate?.id === requiredId);
|
||||
if (!registered || registered.source !== requiredSource) {
|
||||
failures.push(
|
||||
`gates population is not anchored: required ${requiredId} at ${requiredSource}`,
|
||||
);
|
||||
if (!(requiredGateInventory instanceof Map) || requiredGateInventory.size === 0) {
|
||||
failures.push('independent required-gate baseline is absent or empty');
|
||||
} else {
|
||||
if (!inventoriesEqual(REQUIRED_GATE_INVENTORY, requiredGateInventory)) {
|
||||
for (const [requiredId, requiredSource] of requiredGateInventory) {
|
||||
if (REQUIRED_GATE_INVENTORY.get(requiredId) !== requiredSource) {
|
||||
failures.push(
|
||||
`independent required-gate baseline rejects verifier inventory drift at ${requiredId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const [requiredId, requiredSource] of REQUIRED_GATE_INVENTORY) {
|
||||
if (requiredGateInventory.get(requiredId) !== requiredSource) {
|
||||
failures.push(
|
||||
`verifier inventory ${requiredId} is absent or changed in independent required-gate baseline`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [requiredId, requiredSource] of requiredGateInventory) {
|
||||
const registered = (manifest.gates ?? []).find((gate) => gate?.id === requiredId);
|
||||
if (!registered || registered.source !== requiredSource) {
|
||||
failures.push(
|
||||
`independent required-gate baseline rejects manifest drift at ${requiredId}: required source ${requiredSource}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -584,19 +624,12 @@ function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {
|
||||
'inertMutation',
|
||||
'cases',
|
||||
'discoveryAliases',
|
||||
'evidenceSubject',
|
||||
]),
|
||||
`gate ${gate.id}`,
|
||||
failures,
|
||||
);
|
||||
requireString(gate.id, `gate ${gate.id}.id`, failures);
|
||||
requireString(gate.source, `gate ${gate.id}.source`, failures);
|
||||
requireString(gate.evidenceSubject, `gate ${gate.id}.evidenceSubject`, failures);
|
||||
if (gate.evidenceSubject !== gate.id) {
|
||||
failures.push(
|
||||
`gate ${gate.id}: evidence subject ${String(gate.evidenceSubject)} does not match gate id`,
|
||||
);
|
||||
}
|
||||
rejectDuplicateIds(gate.cases, `case in gate ${gate.id}`, failures);
|
||||
if (!Array.isArray(gate.invocation) || gate.invocation.length === 0) {
|
||||
failures.push(`${gate.id}: exact invocation is missing`);
|
||||
@@ -656,6 +689,7 @@ function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {
|
||||
'invocation',
|
||||
'required',
|
||||
'actual',
|
||||
'evidence',
|
||||
'reasonPattern',
|
||||
'environment',
|
||||
'fixture',
|
||||
@@ -675,6 +709,17 @@ function validateClosedSchema(manifest, failures, { fixtureProfile = false } = {
|
||||
}
|
||||
validateOutcome(gateCase.required, `${gate.id}/${gateCase.id}.required`, failures);
|
||||
validateOutcome(gateCase.actual, `${gate.id}/${gateCase.id}.actual`, failures);
|
||||
rejectUnknownKeys(
|
||||
gateCase.evidence,
|
||||
new Set(['subject']),
|
||||
`${gate.id}/${gateCase.id}.evidence`,
|
||||
failures,
|
||||
);
|
||||
requireString(
|
||||
gateCase.evidence?.subject,
|
||||
`${gate.id}/${gateCase.id}.evidence.subject`,
|
||||
failures,
|
||||
);
|
||||
if (gateCase.invocation !== undefined) {
|
||||
validateStringArray(gateCase.invocation, `${gate.id}/${gateCase.id}.invocation`, failures);
|
||||
}
|
||||
@@ -1056,8 +1101,40 @@ export async function verifyRegistry(options) {
|
||||
const observations = [];
|
||||
const manifestPath = path.resolve(options.root, options.manifest);
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||||
let requiredGateInventory;
|
||||
if (!options.fixtureProfile) {
|
||||
const baselinePath = path.resolve(options.root, 'gates/required-gates.baseline.json');
|
||||
try {
|
||||
const baseline = JSON.parse(await readFile(baselinePath, 'utf8'));
|
||||
if (
|
||||
baseline.schemaVersion !== 1 ||
|
||||
!Array.isArray(baseline.gates) ||
|
||||
Object.keys(baseline).some((key) => !['schemaVersion', 'purpose', 'gates'].includes(key)) ||
|
||||
baseline.gates.some(
|
||||
(gate) =>
|
||||
!gate ||
|
||||
typeof gate !== 'object' ||
|
||||
Array.isArray(gate) ||
|
||||
Object.keys(gate).some((key) => !['id', 'source'].includes(key)) ||
|
||||
typeof gate.id !== 'string' ||
|
||||
gate.id.length === 0 ||
|
||||
typeof gate.source !== 'string' ||
|
||||
gate.source.length === 0,
|
||||
)
|
||||
) {
|
||||
failures.push('independent required-gate baseline has unsupported structure');
|
||||
} else {
|
||||
requiredGateInventory = new Map(baseline.gates.map((gate) => [gate.id, gate.source]));
|
||||
if (requiredGateInventory.size !== baseline.gates.length) {
|
||||
failures.push('independent required-gate baseline has duplicate gate ids');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
failures.push(`independent required-gate baseline cannot be read: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
validateStructure(manifest, failures, options);
|
||||
validateStructure(manifest, failures, { ...options, requiredGateInventory });
|
||||
if (options.structureOnly) return { failures, manifest, observations };
|
||||
|
||||
async function collectPhaseFailure(label, action) {
|
||||
@@ -1088,6 +1165,8 @@ export async function verifyRegistry(options) {
|
||||
if (!result) continue;
|
||||
const combined = `${result.stdout ?? ''}\n${result.stderr ?? ''}`;
|
||||
try {
|
||||
const subjectFailure = consumeEvidenceSubject(gate, result.evidence);
|
||||
if (subjectFailure) failures.push(subjectFailure);
|
||||
if (!outcomeMatches(gateCase.actual, result)) {
|
||||
failures.push(
|
||||
`${gate.id}/${gateCase.id}: observed exit ${String(result.status)}${result.signal ? ` signal ${result.signal}` : ''}${result.error ? ` error ${result.error.message}` : ''} or output disagrees with registry actual ${JSON.stringify(gateCase.actual)}`,
|
||||
@@ -1122,13 +1201,11 @@ export async function verifyRegistry(options) {
|
||||
async function main() {
|
||||
try {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const { failures, observations, manifest } = await verifyRegistry(options);
|
||||
if (!options.skipHistory && failures.length === 0) {
|
||||
const history = await verifyHistory({ root: options.root, manifest });
|
||||
failures.push(...history.failures);
|
||||
observations.push(...history.observations);
|
||||
const { failures, observations } = await verifyRegistry(options);
|
||||
for (const observation of observations) {
|
||||
assertCurrentTreeObservation(observation);
|
||||
process.stdout.write(`${observation}\n`);
|
||||
}
|
||||
for (const observation of observations) process.stdout.write(`${observation}\n`);
|
||||
if (failures.length > 0) {
|
||||
for (const failure of failures) process.stderr.write(`GATE VERIFY FAILED: ${failure}\n`);
|
||||
process.exitCode = 1;
|
||||
|
||||
Reference in New Issue
Block a user