feat(quality-rails): typed evaluator core — statuses, digested definitions, per-subject sets (#1275)
This commit is contained in:
@@ -0,0 +1,67 @@
|
|||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import type { ChildProcess } from 'node:child_process';
|
||||||
|
|
||||||
|
import type { AdapterOutcome, AdapterRequest, ProcessAdapter } from './types.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default thin process adapter (spawn-based). Runs a command to completion with
|
||||||
|
* a hard timeout and reports exit code + captured output — it owns NO verdict
|
||||||
|
* logic. Interpreting the outcome is always the check implementation's job.
|
||||||
|
*/
|
||||||
|
export function createSpawnProcessAdapter(): ProcessAdapter {
|
||||||
|
return {
|
||||||
|
run(request: AdapterRequest): Promise<AdapterOutcome> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
let child: ChildProcess;
|
||||||
|
try {
|
||||||
|
child = spawn(request.file, request.args, {
|
||||||
|
cwd: request.cwd,
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
resolve({
|
||||||
|
ok: false,
|
||||||
|
kind: 'spawn-error',
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stdout = '';
|
||||||
|
let stderr = '';
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
child.kill('SIGKILL');
|
||||||
|
resolve({
|
||||||
|
ok: false,
|
||||||
|
kind: 'timeout',
|
||||||
|
message: `process timed out after ${request.timeoutMs}ms: ${request.file}`,
|
||||||
|
});
|
||||||
|
}, request.timeoutMs);
|
||||||
|
|
||||||
|
const settle = (outcome: AdapterOutcome): void => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
clearTimeout(timer);
|
||||||
|
resolve(outcome);
|
||||||
|
};
|
||||||
|
|
||||||
|
child.stdout?.on('data', (chunk: Buffer) => {
|
||||||
|
stdout += chunk.toString('utf8');
|
||||||
|
});
|
||||||
|
child.stderr?.on('data', (chunk: Buffer) => {
|
||||||
|
stderr += chunk.toString('utf8');
|
||||||
|
});
|
||||||
|
child.on('error', (error: Error) => {
|
||||||
|
settle({ ok: false, kind: 'spawn-error', message: error.message });
|
||||||
|
});
|
||||||
|
child.on('close', (code: number | null) => {
|
||||||
|
settle({ ok: true, exitCode: code, stdout, stderr });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
import { constants } from 'node:fs';
|
||||||
|
import { access } from 'node:fs/promises';
|
||||||
|
import { isAbsolute, resolve } from 'node:path';
|
||||||
|
|
||||||
|
import { digestOfPolicy, digestOfSpec } from './digest.js';
|
||||||
|
import type {
|
||||||
|
CheckContext,
|
||||||
|
CheckDefinition,
|
||||||
|
CheckDefinitionSpec,
|
||||||
|
CheckOutcome,
|
||||||
|
CheckSetPolicy,
|
||||||
|
CheckSetPolicySpec,
|
||||||
|
SubjectKind,
|
||||||
|
} from './types.js';
|
||||||
|
|
||||||
|
// Check definitions for the RI-N4 evaluator (card RI-3-002). Each definition is
|
||||||
|
// DATA with a version and a content digest (see digest.ts); the executable
|
||||||
|
// half is attached via defineCheck. Check-set SELECTION is per subject kind
|
||||||
|
// (probe-inventory gap 7): this monorepo does not match the node template's
|
||||||
|
// file list, so the QC-19 definition carries a distinct file set for the
|
||||||
|
// `monorepo` subject kind and the policy selects checks per kind.
|
||||||
|
|
||||||
|
export function defineCheck(
|
||||||
|
spec: CheckDefinitionSpec,
|
||||||
|
evaluate: (ctx: CheckContext) => Promise<CheckOutcome>,
|
||||||
|
): CheckDefinition {
|
||||||
|
return { ...spec, definitionDigest: digestOfSpec(spec), evaluate };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fileExists(filePath: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await access(filePath, constants.F_OK);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── QC-19: downstream rails presence ────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Typed absorption of the former presence-only `quality-rails check` loop in
|
||||||
|
// cli.ts. The scaffold-kind file lists below are carried over VERBATIM so the
|
||||||
|
// evaluator's typed verdicts are parity-equivalent with the presence loop on
|
||||||
|
// the same fixture; the `monorepo` list is new (per-subject check sets).
|
||||||
|
|
||||||
|
const qc19Spec: CheckDefinitionSpec = {
|
||||||
|
id: 'qc-19-rails-files-present',
|
||||||
|
version: '1.0.0',
|
||||||
|
canonicalCheck: 'QC-19',
|
||||||
|
description:
|
||||||
|
'The subject still carries its quality-rails files. Typed absorption of the former presence-only check loop; presence is necessary, not sufficient (RI-N4).',
|
||||||
|
appliesTo: ['node', 'python', 'rust', 'monorepo', 'unknown'],
|
||||||
|
params: {
|
||||||
|
expectedFilesByKind: {
|
||||||
|
node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
|
||||||
|
python: ['pyproject.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
|
||||||
|
rust: ['rustfmt.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
|
||||||
|
monorepo: [
|
||||||
|
'.husky/pre-commit',
|
||||||
|
'.husky/pre-push',
|
||||||
|
'eslint.config.mjs',
|
||||||
|
'.prettierrc',
|
||||||
|
'.lintstagedrc',
|
||||||
|
],
|
||||||
|
unknown: ['.githooks/pre-commit', 'PR-CHECKLIST.md'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
async function evaluateQc19(ctx: CheckContext): Promise<CheckOutcome> {
|
||||||
|
const byKind = ctx.params['expectedFilesByKind'] as Record<string, readonly string[]> | undefined;
|
||||||
|
if (byKind === undefined) {
|
||||||
|
return { status: 'error', reason: 'definition params missing expectedFilesByKind' };
|
||||||
|
}
|
||||||
|
const expected = byKind[ctx.subject.kind];
|
||||||
|
if (expected === undefined) {
|
||||||
|
// Fail-closed: an undefined file set for a declared subject kind is a
|
||||||
|
// definition gap, never a green outcome.
|
||||||
|
return {
|
||||||
|
status: 'blocked',
|
||||||
|
reason: `no expected-file set defined for subject kind '${ctx.subject.kind}'`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const missing: string[] = [];
|
||||||
|
for (const relativePath of expected) {
|
||||||
|
if (!(await fileExists(resolve(ctx.subject.path, relativePath)))) {
|
||||||
|
missing.push(relativePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (missing.length > 0) {
|
||||||
|
return {
|
||||||
|
status: 'failed',
|
||||||
|
reason: `missing rails files (${ctx.subject.kind}): ${missing.join(', ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { status: 'passed' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── QC-20: downstream enforcement verification (behavioral probe) ──────────
|
||||||
|
//
|
||||||
|
// The planted-commit behavioral probe (framework tools/quality/scripts/verify.sh)
|
||||||
|
// stays a THIN SHELL ADAPTER: the TS evaluator invokes it and OWNS the verdict
|
||||||
|
// parsing (RI-N4: grep-on-output verdict logic moves into the typed evaluator).
|
||||||
|
// Probe contract (verify.sh): exit 0 ⇔ every sub-probe passed, exit 1 ⇔ at
|
||||||
|
// least one sub-probe failed; sub-probe verdicts appear as `PASS:` / `FAIL:`
|
||||||
|
// marker lines and the script always prints a `Verification Summary` section.
|
||||||
|
// Any deviation from that contract (other exit codes, unparseable output,
|
||||||
|
// missing probe, process failure, timeout) is `error`/`blocked` — never
|
||||||
|
// `passed`.
|
||||||
|
|
||||||
|
const qc20Spec: CheckDefinitionSpec = {
|
||||||
|
id: 'qc-20-enforcement-verify',
|
||||||
|
version: '1.0.0',
|
||||||
|
canonicalCheck: 'QC-20',
|
||||||
|
description:
|
||||||
|
'The behavioral planted-commit probe runs against the subject and every sub-probe blocks as intended. The shell probe is a thin adapter; verdict parsing is owned by this evaluator.',
|
||||||
|
appliesTo: ['node', 'python', 'rust', 'unknown'],
|
||||||
|
params: {
|
||||||
|
command: 'bash',
|
||||||
|
timeoutMs: 120_000,
|
||||||
|
passMarker: 'PASS:',
|
||||||
|
failMarker: 'FAIL:',
|
||||||
|
summaryMarker: 'Verification Summary',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function linesWith(text: string, marker: string): string[] {
|
||||||
|
return text
|
||||||
|
.split('\n')
|
||||||
|
.map((line) => line.trim())
|
||||||
|
.filter((line) => line.includes(marker));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function evaluateQc20(ctx: CheckContext): Promise<CheckOutcome> {
|
||||||
|
const rawProbePath = ctx.inputs['probePath'];
|
||||||
|
if (typeof rawProbePath !== 'string' || rawProbePath.trim().length === 0) {
|
||||||
|
return {
|
||||||
|
status: 'blocked',
|
||||||
|
reason:
|
||||||
|
'missing input: probePath — the behavioral probe script must be provided (e.g. the framework verify.sh)',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const probePath = isAbsolute(rawProbePath)
|
||||||
|
? rawProbePath
|
||||||
|
: resolve(ctx.subject.path, rawProbePath);
|
||||||
|
if (!(await fileExists(probePath))) {
|
||||||
|
return { status: 'blocked', reason: `probe script not found: ${probePath}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const command = typeof ctx.params['command'] === 'string' ? ctx.params['command'] : 'bash';
|
||||||
|
const timeoutMs = typeof ctx.params['timeoutMs'] === 'number' ? ctx.params['timeoutMs'] : 120_000;
|
||||||
|
const passMarker =
|
||||||
|
typeof ctx.params['passMarker'] === 'string' ? ctx.params['passMarker'] : 'PASS:';
|
||||||
|
const failMarker =
|
||||||
|
typeof ctx.params['failMarker'] === 'string' ? ctx.params['failMarker'] : 'FAIL:';
|
||||||
|
const summaryMarker =
|
||||||
|
typeof ctx.params['summaryMarker'] === 'string'
|
||||||
|
? ctx.params['summaryMarker']
|
||||||
|
: 'Verification Summary';
|
||||||
|
|
||||||
|
const outcome = await ctx.adapter.run({
|
||||||
|
file: command,
|
||||||
|
args: [probePath],
|
||||||
|
cwd: ctx.subject.path,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!outcome.ok) {
|
||||||
|
// Process error or timeout: the probe never produced a trustworthy result.
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
reason: `probe process ${outcome.kind}: ${outcome.message}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const output = `${outcome.stdout}\n${outcome.stderr}`;
|
||||||
|
const failLines = linesWith(output, failMarker);
|
||||||
|
const passLines = linesWith(output, passMarker);
|
||||||
|
|
||||||
|
if (outcome.exitCode === 0) {
|
||||||
|
// A green exit must be corroborated by a parseable green transcript:
|
||||||
|
// at least one pass marker, no fail markers, and the summary section.
|
||||||
|
if (passLines.length > 0 && failLines.length === 0 && output.includes(summaryMarker)) {
|
||||||
|
return { status: 'passed' };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
reason: `malformed probe output: exit 0 without a parseable pass transcript (${passLines.length} pass markers, ${failLines.length} fail markers, summary ${output.includes(summaryMarker) ? 'present' : 'absent'})`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outcome.exitCode === 1) {
|
||||||
|
if (failLines.length === 0) {
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
reason: 'malformed probe output: exit 1 without parseable FAIL markers',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: 'failed',
|
||||||
|
reason: `enforcement probe reported ${failLines.length} failing sub-probe(s): ${failLines.join(' | ')}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
status: 'error',
|
||||||
|
reason: `probe exited with unexpected code ${String(outcome.exitCode)} — outcome not interpretable`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Per-subject check-set policy ───────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Gap 7 of the probe inventory: check sets must be selected per subject, not
|
||||||
|
// one global list. Downstream scaffold kinds get the presence check plus the
|
||||||
|
// behavioral probe (QC-20 blocks until a probePath input is provided — an
|
||||||
|
// unverified subject can never evaluate green). The monorepo subject is this
|
||||||
|
// repository itself: its rails are the husky hooks + shared lint/format
|
||||||
|
// configs, covered by QC-19; the downstream planted-commit probe does not
|
||||||
|
// apply to it (this repo's own commit gates are QC-13/QC-14, outside this
|
||||||
|
// evaluator's owned checks).
|
||||||
|
|
||||||
|
const checkSetPolicySpec: CheckSetPolicySpec = {
|
||||||
|
version: '1.0.0',
|
||||||
|
byKind: {
|
||||||
|
node: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
|
||||||
|
python: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
|
||||||
|
rust: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
|
||||||
|
unknown: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
|
||||||
|
monorepo: ['qc-19-rails-files-present'],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export const CHECK_SET_POLICY: CheckSetPolicy = {
|
||||||
|
...checkSetPolicySpec,
|
||||||
|
policyDigest: digestOfPolicy(checkSetPolicySpec),
|
||||||
|
};
|
||||||
|
|
||||||
|
export const QC_19_RAILS_FILES_PRESENT = defineCheck(qc19Spec, evaluateQc19);
|
||||||
|
export const QC_20_ENFORCEMENT_VERIFY = defineCheck(qc20Spec, evaluateQc20);
|
||||||
|
|
||||||
|
/** Built-in check definitions, keyed by id. */
|
||||||
|
export function builtInDefinitions(): CheckDefinition[] {
|
||||||
|
return [QC_19_RAILS_FILES_PRESENT, QC_20_ENFORCEMENT_VERIFY];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function checkSetForKind(
|
||||||
|
kind: SubjectKind,
|
||||||
|
policy: CheckSetPolicy = CHECK_SET_POLICY,
|
||||||
|
): readonly string[] {
|
||||||
|
const selected = policy.byKind[kind];
|
||||||
|
if (selected === undefined) {
|
||||||
|
// Fail-closed selection: an unknown kind yields an EMPTY set only to the
|
||||||
|
// caller; the runner treats an empty result list as `blocked`, never green.
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return selected;
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
|
||||||
|
import type { CheckDefinitionSpec, CheckSetPolicySpec } from './types.js';
|
||||||
|
|
||||||
|
// Deterministic JSON: object keys sorted at every level so two specs with the
|
||||||
|
// same content always produce the same bytes (and thus the same digest).
|
||||||
|
export function canonicalJson(value: unknown): string {
|
||||||
|
if (value === null || typeof value !== 'object') {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
}
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return `[${value.map((entry) => canonicalJson(entry)).join(',')}]`;
|
||||||
|
}
|
||||||
|
const record = value as Record<string, unknown>;
|
||||||
|
const keys = Object.keys(record).sort();
|
||||||
|
return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** sha256 over the canonical JSON of `value`. */
|
||||||
|
export function digestContent(value: unknown): string {
|
||||||
|
return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Content digest of a check definition: covers the declarative spec (id,
|
||||||
|
* version, canonical check, description, applicability, params) — everything a
|
||||||
|
* reviewer reasons about — while excluding the executable function object.
|
||||||
|
* Changing any covered field changes the digest, so a recorded digest always
|
||||||
|
* identifies exactly which definition content produced a verdict.
|
||||||
|
*/
|
||||||
|
export function digestOfSpec(spec: CheckDefinitionSpec): string {
|
||||||
|
return digestContent({
|
||||||
|
id: spec.id,
|
||||||
|
version: spec.version,
|
||||||
|
canonicalCheck: spec.canonicalCheck,
|
||||||
|
description: spec.description,
|
||||||
|
appliesTo: spec.appliesTo,
|
||||||
|
params: spec.params,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Content digest of the per-subject check-set policy. */
|
||||||
|
export function digestOfPolicy(spec: CheckSetPolicySpec): string {
|
||||||
|
return digestContent(spec);
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
import { constants } from 'node:fs';
|
||||||
|
import { access, stat } from 'node:fs/promises';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
|
||||||
|
import { createSpawnProcessAdapter } from './adapter.js';
|
||||||
|
import { builtInDefinitions, CHECK_SET_POLICY, checkSetForKind } from './definitions.js';
|
||||||
|
import type {
|
||||||
|
AggregateState,
|
||||||
|
CheckResult,
|
||||||
|
CheckStatus,
|
||||||
|
EvaluateOptions,
|
||||||
|
EvaluationReport,
|
||||||
|
ProcessAdapter,
|
||||||
|
Subject,
|
||||||
|
SubjectKind,
|
||||||
|
} from './types.js';
|
||||||
|
import { detectProjectKind } from '../detect.js';
|
||||||
|
|
||||||
|
async function pathExists(targetPath: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
await access(targetPath, constants.F_OK);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function isDirectory(targetPath: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
return (await stat(targetPath)).isDirectory();
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Subject-kind detection for the evaluator. Extends the scaffold detection
|
||||||
|
* (detect.ts) with the `monorepo` kind: a pnpm workspace is this repository's
|
||||||
|
* own subject shape and carries a different rails file set (probe-inventory
|
||||||
|
* gap 7 — check sets are per subject, not one global file list).
|
||||||
|
*/
|
||||||
|
export async function detectSubjectKind(subjectPath: string): Promise<SubjectKind> {
|
||||||
|
if (await pathExists(join(subjectPath, 'pnpm-workspace.yaml'))) {
|
||||||
|
return 'monorepo';
|
||||||
|
}
|
||||||
|
const kind = await detectProjectKind(subjectPath);
|
||||||
|
return kind;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Aggregate state, MACP-style discipline: `passed` only when at least one
|
||||||
|
* check produced a verdict AND every verdict is `passed` or an explicitly
|
||||||
|
* qualified `not-applicable`. Precedence is fail-closed: error > blocked >
|
||||||
|
* failed > passed; an empty result list aggregates to `blocked`.
|
||||||
|
*/
|
||||||
|
export function aggregateState(results: readonly CheckResult[]): AggregateState {
|
||||||
|
if (results.length === 0) {
|
||||||
|
return 'blocked';
|
||||||
|
}
|
||||||
|
const has = (status: CheckStatus): boolean => results.some((result) => result.status === status);
|
||||||
|
if (has('error')) {
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
if (has('blocked')) {
|
||||||
|
return 'blocked';
|
||||||
|
}
|
||||||
|
if (has('failed')) {
|
||||||
|
return 'failed';
|
||||||
|
}
|
||||||
|
return 'passed';
|
||||||
|
}
|
||||||
|
|
||||||
|
function reasonFrom(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Evaluate one subject against a set of checks, producing typed verdicts.
|
||||||
|
*
|
||||||
|
* Fail-closed invariants (RI-N4):
|
||||||
|
* - unknown check id → `error` (never passed)
|
||||||
|
* - subject directory absent → every verdict `blocked`
|
||||||
|
* - check implementation threw → `error`
|
||||||
|
* - non-passed without a reason → `error` (no unqualified skips)
|
||||||
|
* - check not applicable → `not-applicable` WITH a reason
|
||||||
|
*/
|
||||||
|
export async function evaluateSubject(options: EvaluateOptions): Promise<EvaluationReport> {
|
||||||
|
const subjectPath = resolve(options.subjectPath);
|
||||||
|
const subject: Subject = {
|
||||||
|
path: subjectPath,
|
||||||
|
kind: await detectSubjectKind(subjectPath),
|
||||||
|
};
|
||||||
|
|
||||||
|
const definitions = options.definitions ?? builtInDefinitions();
|
||||||
|
const byId = new Map(definitions.map((definition) => [definition.id, definition]));
|
||||||
|
const requested = options.checkIds ?? checkSetForKind(subject.kind);
|
||||||
|
const adapter: ProcessAdapter = options.adapter ?? createSpawnProcessAdapter();
|
||||||
|
|
||||||
|
const results: CheckResult[] = [];
|
||||||
|
const definitionDigests: Record<string, string> = {};
|
||||||
|
|
||||||
|
for (const checkId of requested) {
|
||||||
|
const definition = byId.get(checkId);
|
||||||
|
if (definition === undefined) {
|
||||||
|
const known = definitions.map((entry) => entry.id).join(', ');
|
||||||
|
results.push({
|
||||||
|
status: 'error',
|
||||||
|
checkId,
|
||||||
|
checkVersion: 'unknown',
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: `unknown check id '${checkId}' — no registered definition (known: ${known})`,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
definitionDigests[checkId] = definition.definitionDigest;
|
||||||
|
|
||||||
|
if (!(await isDirectory(subjectPath))) {
|
||||||
|
results.push({
|
||||||
|
status: 'blocked',
|
||||||
|
checkId,
|
||||||
|
checkVersion: definition.version,
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: `subject directory does not exist: ${subjectPath}`,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!definition.appliesTo.includes(subject.kind)) {
|
||||||
|
results.push({
|
||||||
|
status: 'not-applicable',
|
||||||
|
checkId,
|
||||||
|
checkVersion: definition.version,
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: `check '${checkId}' does not apply to subject kind '${subject.kind}'`,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const inputs = options.inputs?.[checkId] ?? {};
|
||||||
|
const outcome = await definition.evaluate({
|
||||||
|
subject,
|
||||||
|
params: definition.params,
|
||||||
|
inputs,
|
||||||
|
adapter,
|
||||||
|
});
|
||||||
|
if (outcome.status !== 'passed' && (outcome.reason === undefined || outcome.reason === '')) {
|
||||||
|
results.push({
|
||||||
|
status: 'error',
|
||||||
|
checkId,
|
||||||
|
checkVersion: definition.version,
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: `check returned status '${outcome.status}' without a reason — treated as error`,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
results.push({
|
||||||
|
status: outcome.status,
|
||||||
|
checkId,
|
||||||
|
checkVersion: definition.version,
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: outcome.reason,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
results.push({
|
||||||
|
status: 'error',
|
||||||
|
checkId,
|
||||||
|
checkVersion: definition.version,
|
||||||
|
subject: subjectPath,
|
||||||
|
reason: `check implementation threw: ${reasonFrom(error)}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
subject,
|
||||||
|
results,
|
||||||
|
definitionDigests,
|
||||||
|
checkSetVersion: CHECK_SET_POLICY.version,
|
||||||
|
state: aggregateState(results),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
// Evaluator core types — RI-N4 (card RI-3-002, SDLC-D-037 second half).
|
||||||
|
//
|
||||||
|
// The quality-rails evaluator is the SOLE authoritative producer of check
|
||||||
|
// verdicts for the checks it owns. Every verdict is typed and fail-closed:
|
||||||
|
// missing implementations, missing inputs, unknown check ids, process errors,
|
||||||
|
// timeouts, and malformed adapter output can never become `passed` or an
|
||||||
|
// unqualified skip — they surface as `blocked` or `error` with a reason.
|
||||||
|
// (Vocabulary mirrors MACP's GateStatus discipline from packages/macp.)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Typed verdict for a single check execution.
|
||||||
|
*
|
||||||
|
* - `passed` — the check really ran and its condition held.
|
||||||
|
* - `failed` — the check really ran and its condition did NOT hold.
|
||||||
|
* - `blocked` — the check could not run at all (missing subject, missing
|
||||||
|
* input). Never a green outcome.
|
||||||
|
* - `error` — the check attempted to run but its outcome cannot be trusted
|
||||||
|
* (unknown check id, implementation threw, process error, timeout, malformed
|
||||||
|
* adapter output). Never a green outcome.
|
||||||
|
* - `not-applicable` — the check definition explicitly declares it does not
|
||||||
|
* apply to this subject (a qualified skip, always with a reason).
|
||||||
|
*/
|
||||||
|
export type CheckStatus = 'passed' | 'failed' | 'blocked' | 'error' | 'not-applicable';
|
||||||
|
|
||||||
|
/** Aggregate outcome, MACP-style: `passed` only when every result is green. */
|
||||||
|
export type AggregateState = 'passed' | 'failed' | 'blocked' | 'error';
|
||||||
|
|
||||||
|
/** Kinds of subjects the evaluator can assess. */
|
||||||
|
export type SubjectKind = 'node' | 'python' | 'rust' | 'monorepo' | 'unknown';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A single check verdict. This is the canonical result shape: `status`,
|
||||||
|
* `checkId`, `checkVersion`, `subject`, `reason`. `reason` is REQUIRED
|
||||||
|
* (enforced by the runner) for every status other than `passed`.
|
||||||
|
*/
|
||||||
|
export interface CheckResult {
|
||||||
|
status: CheckStatus;
|
||||||
|
checkId: string;
|
||||||
|
checkVersion: string;
|
||||||
|
subject: string;
|
||||||
|
reason?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The project being evaluated. */
|
||||||
|
export interface Subject {
|
||||||
|
/** Absolute path. */
|
||||||
|
path: string;
|
||||||
|
kind: SubjectKind;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The data half of a check definition. Definitions live as DATA with a version
|
||||||
|
* and a content digest (see `digestOfSpec`); the executable half is attached
|
||||||
|
* separately so the digest covers only reviewable, declarative content.
|
||||||
|
*/
|
||||||
|
export interface CheckDefinitionSpec {
|
||||||
|
/** Stable id, e.g. `qc-19-rails-files-present`. */
|
||||||
|
id: string;
|
||||||
|
/** Semver of this definition's data+semantics. */
|
||||||
|
version: string;
|
||||||
|
/** Canonical check id from docs/release-integrity/probe-inventory.md (QC-n). */
|
||||||
|
canonicalCheck: string;
|
||||||
|
description: string;
|
||||||
|
/** Subject kinds this check can assess (others yield `not-applicable`). */
|
||||||
|
appliesTo: readonly SubjectKind[];
|
||||||
|
/** Declarative parameters (file lists, markers, timeouts) — digest-covered. */
|
||||||
|
params: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A fully assembled check definition: spec + digest + implementation. */
|
||||||
|
export interface CheckDefinition extends CheckDefinitionSpec {
|
||||||
|
/** sha256 content digest of the spec (canonical JSON projection). */
|
||||||
|
definitionDigest: string;
|
||||||
|
evaluate(ctx: CheckContext): Promise<CheckOutcome>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What a check implementation returns; the runner stamps id/version/subject. */
|
||||||
|
export interface CheckOutcome {
|
||||||
|
status: CheckStatus;
|
||||||
|
reason?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Caller-provided inputs for one check invocation (e.g. the QC-20 probe path). */
|
||||||
|
export type CheckInputs = Record<string, unknown>;
|
||||||
|
|
||||||
|
/** Everything a check implementation may use. */
|
||||||
|
export interface CheckContext {
|
||||||
|
subject: Subject;
|
||||||
|
params: Record<string, unknown>;
|
||||||
|
inputs: CheckInputs;
|
||||||
|
adapter: ProcessAdapter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Outcome of running a shell probe through the thin process adapter. */
|
||||||
|
export type AdapterOutcome =
|
||||||
|
| { ok: true; exitCode: number | null; stdout: string; stderr: string }
|
||||||
|
| { ok: false; kind: 'spawn-error' | 'timeout'; message: string };
|
||||||
|
|
||||||
|
/** Request for the process adapter. */
|
||||||
|
export interface AdapterRequest {
|
||||||
|
file: string;
|
||||||
|
args: string[];
|
||||||
|
cwd: string;
|
||||||
|
timeoutMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Thin process adapter: runs a command, owns NO verdict logic. Verdict parsing
|
||||||
|
* always lives in the check implementation (TS), never in the shell probe.
|
||||||
|
*/
|
||||||
|
export interface ProcessAdapter {
|
||||||
|
run(request: AdapterRequest): Promise<AdapterOutcome>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Per-subject-kind check-set selection policy (versioned and digested). */
|
||||||
|
export interface CheckSetPolicySpec {
|
||||||
|
version: string;
|
||||||
|
byKind: Record<SubjectKind, readonly string[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CheckSetPolicy extends CheckSetPolicySpec {
|
||||||
|
/** sha256 content digest of the policy spec. */
|
||||||
|
policyDigest: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Full typed evaluation report for one subject. */
|
||||||
|
export interface EvaluationReport {
|
||||||
|
subject: Subject;
|
||||||
|
results: CheckResult[];
|
||||||
|
/** checkId → content digest of the definition that produced the verdicts. */
|
||||||
|
definitionDigests: Record<string, string>;
|
||||||
|
/** Version of the check-set policy used for subject selection. */
|
||||||
|
checkSetVersion: string;
|
||||||
|
state: AggregateState;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Options for `evaluateSubject`. */
|
||||||
|
export interface EvaluateOptions {
|
||||||
|
subjectPath: string;
|
||||||
|
/** Restrict to these check ids; defaults to the subject kind's check set. */
|
||||||
|
checkIds?: string[];
|
||||||
|
/** Per-check inputs, keyed by check id (e.g. `{ 'qc-20-enforcement-verify': { probePath } }`). */
|
||||||
|
inputs?: Record<string, CheckInputs>;
|
||||||
|
/** Replace the built-in definitions (tests / future batches). */
|
||||||
|
definitions?: CheckDefinition[];
|
||||||
|
/** Inject a process adapter (tests / instrumentation). */
|
||||||
|
adapter?: ProcessAdapter;
|
||||||
|
}
|
||||||
@@ -3,3 +3,35 @@ export * from './detect.js';
|
|||||||
export * from './scaffolder.js';
|
export * from './scaffolder.js';
|
||||||
export * from './templates.js';
|
export * from './templates.js';
|
||||||
export * from './types.js';
|
export * from './types.js';
|
||||||
|
|
||||||
|
// RI-N4 evaluator (card RI-3-002): the public, programmatic entry points.
|
||||||
|
export {
|
||||||
|
builtInDefinitions,
|
||||||
|
CHECK_SET_POLICY,
|
||||||
|
checkSetForKind,
|
||||||
|
defineCheck,
|
||||||
|
QC_19_RAILS_FILES_PRESENT,
|
||||||
|
QC_20_ENFORCEMENT_VERIFY,
|
||||||
|
} from './evaluator/definitions.js';
|
||||||
|
export { canonicalJson, digestContent, digestOfPolicy, digestOfSpec } from './evaluator/digest.js';
|
||||||
|
export { createSpawnProcessAdapter } from './evaluator/adapter.js';
|
||||||
|
export { aggregateState, detectSubjectKind, evaluateSubject } from './evaluator/runner.js';
|
||||||
|
export type {
|
||||||
|
AdapterOutcome,
|
||||||
|
AdapterRequest,
|
||||||
|
AggregateState,
|
||||||
|
CheckContext,
|
||||||
|
CheckDefinition,
|
||||||
|
CheckDefinitionSpec,
|
||||||
|
CheckInputs,
|
||||||
|
CheckOutcome,
|
||||||
|
CheckResult,
|
||||||
|
CheckSetPolicy,
|
||||||
|
CheckSetPolicySpec,
|
||||||
|
CheckStatus,
|
||||||
|
EvaluateOptions,
|
||||||
|
EvaluationReport,
|
||||||
|
ProcessAdapter,
|
||||||
|
Subject,
|
||||||
|
SubjectKind,
|
||||||
|
} from './evaluator/types.js';
|
||||||
|
|||||||
Reference in New Issue
Block a user