jason.woltje
4c8bf9ca4e
PR #1916 rule: a refusal names the field or file, never a config value or a host path. readJsonFile's three refusals and parseJsonFileText now name the file by its `what` label only. Callers whose label alone is ambiguous add a name that already passed a strict pattern (business id, role name, org id, turn id, project key) or a fixed path under the data root. Exit codes, refusal codes and check order are unchanged. organization: util, access, project, config-command, library-seats, org-service deleteOrg receipt, agent-levels regex; T1 files role.mjs and organization-config.mjs change only the readJsonFile label. cli: access lock, accounts owner, host state/lock, notifier journal, notify-config, run-evidence, dev-fixture, git-socket, org-bus, org-console. discord: turn record label names the turn id. Sentinel tests (sk-test-NOT-A-SECRET-1234 as a path component) cover each changed refusal and fail on BASE; old path pins updated.
168 lines
9.8 KiB
JavaScript
168 lines
9.8 KiB
JavaScript
// `mosaic run evidence`: a fabricated run in a scratch HOME, verified, then
|
|
// tampered with. Nothing here starts a bus.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash } from 'node:crypto';
|
|
import { appendFileSync, chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { reserveOrg } from '../../organization/src/org-registry.mjs';
|
|
import { launchLogFile, runsDir } from '../../seat/src/session.mjs';
|
|
import { DEV_OWNER } from '../src/dev-local-owner.mjs';
|
|
import { runCommand } from '../src/run-evidence.mjs';
|
|
import { io } from './helpers.mjs';
|
|
|
|
const HOME = mkdtempSync(join(tmpdir(), 'runev'));
|
|
process.env.HOME = HOME;
|
|
test.after(() => rmSync(HOME, { recursive: true, force: true }));
|
|
const sha = s => createHash('sha256').update(s).digest('hex');
|
|
const THREAD = '0b6f1c2e-3d4a-4b5c-8d6e-7f8091a2b3c4';
|
|
const OTHER = '11111111-2222-4333-8444-555555555555';
|
|
let n = 0;
|
|
|
|
// An instance with org alpha and one finished run for THREAD.
|
|
async function world({ run = 'r0123456789ab', under = HOME } = {}) {
|
|
const name = `ev${++n}`;
|
|
const env = { PATH: process.env.PATH, HOME, MOSAIC_CONFIG: join(HOME, '.config', `mosaic-${name}`, 'config.json') };
|
|
mkdirSync(dirname(env.MOSAIC_CONFIG), { recursive: true, mode: 0o700 });
|
|
const dataRoot = join(under, `data-${name}`); mkdirSync(dataRoot, { recursive: true, mode: 0o700 });
|
|
const org = await reserveOrg({ configDir: dirname(env.MOSAIC_CONFIG), dataRoot, instance: name, principal: DEV_OWNER }, 'alpha');
|
|
const runDir = join(runsDir(org.root, 'alpha'), run);
|
|
mkdirSync(join(runDir, 'bundle'), { recursive: true, mode: 0o700 });
|
|
mkdirSync(join(runDir, 'home/.pi/agent'), { recursive: true, mode: 0o700 });
|
|
mkdirSync(join(runDir, 'turns'), { mode: 0o700 });
|
|
const put = (p, text) => { writeFileSync(join(runDir, p), text, { mode: 0o600 }); return sha(text); };
|
|
put('session.json', JSON.stringify({ business: 'alpha', instance: name, thread: THREAD, run }));
|
|
const files = { 'prompt.md': put('bundle/prompt.md', '# prompt\n'), '../home/.pi/agent/models.json': put('home/.pi/agent/models.json', '{}\n') };
|
|
put('bundle/manifest.json', JSON.stringify({ bundleVersion: 1, run, files }));
|
|
put('turns/0001.stderr', '');
|
|
const log = launchLogFile(org.root, 'alpha');
|
|
const lines = [
|
|
{ at: '2026-11-07T00:00:00.000Z', event: 'launch', instance: name, thread: OTHER, session: 's0', run: 'rffffffffffff' },
|
|
{ at: '2026-11-07T00:00:01.000Z', event: 'launch', instance: name, thread: THREAD, session: 's1', run, model: 'fixture-deterministic', harness: 'pi', manifest: join(runDir, 'bundle/manifest.json') },
|
|
{ at: '2026-11-07T00:00:02.000Z', event: 'end', instance: name, thread: THREAD, session: 's1', run, reason: 'idle', exitCode: 0 },
|
|
].map(l => JSON.stringify(l));
|
|
writeFileSync(log, `${lines[0]}\nnot json\n${lines[1]}\n${lines[2]}\n`, { mode: 0o600 });
|
|
return { env, dataRoot, runDir, log, launchLine: lines[1], name };
|
|
}
|
|
|
|
async function evidence(w, args, deps = {}) {
|
|
const x = io(w.env);
|
|
let error = null;
|
|
try { await runCommand(['evidence', 'alpha', ...args], x, { system: { dataRoot: w.dataRoot }, ...deps }); } catch (e) { error = e; }
|
|
return { error, out: x.out.text, json: args.includes('--json') && x.out.text ? JSON.parse(x.out.text) : null };
|
|
}
|
|
|
|
test('a complete run verifies and names its org, run and launch line', async () => {
|
|
const w = await world();
|
|
const { error, json } = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.equal(error, null, error?.message);
|
|
assert.equal(json.verified, true);
|
|
assert.deepEqual([json.org.id, json.org.state, json.instance, json.launchLog.malformed], ['alpha', 'reserved', w.name, 1]);
|
|
assert.equal(json.runs.length, 1);
|
|
const [r] = json.runs;
|
|
assert.deepEqual([r.run, r.launch.file, r.launch.line, r.launch.sha256], ['r0123456789ab', w.log, 3, sha(w.launchLine)]);
|
|
assert.deepEqual(r.end.map(e => [e.line, e.reason]), [[4, 'idle']]);
|
|
assert.equal(r.result.path, join(w.runDir, 'session.json'));
|
|
assert.deepEqual(r.snapshot.files.map(f => [f.name, f.ok]), [['prompt.md', true], ['../home/.pi/agent/models.json', true]]);
|
|
assert.deepEqual(r.stderr.map(s => [s.path, s.sha256]), [[join(w.runDir, 'turns/0001.stderr'), sha('')]]);
|
|
const text = await evidence(w, ['--thread', THREAD]);
|
|
assert.equal(text.error, null);
|
|
assert.match(text.out, /not verified \(pass --verify\)/);
|
|
});
|
|
|
|
test('a changed bundle file fails verification', async () => {
|
|
const w = await world();
|
|
appendFileSync(join(w.runDir, 'bundle/prompt.md'), 'edited\n');
|
|
const { error, json } = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.equal(error?.exitCode, 1);
|
|
assert.equal(json.verified, false);
|
|
assert.deepEqual(json.runs[0].problems, ['bundle file prompt.md: digest differs from the manifest']);
|
|
const plain = await evidence(w, ['--thread', THREAD, '--json']);
|
|
assert.equal(plain.error, null, 'without --verify a digest is not checked');
|
|
});
|
|
|
|
test('missing stderr, a loose session file and a linked session file fail', async () => {
|
|
let w = await world();
|
|
unlinkSync(join(w.runDir, 'turns/0001.stderr'));
|
|
let r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.equal(r.error?.exitCode, 1);
|
|
assert.deepEqual(r.json.runs[0].problems, ['no turns/*.stderr']);
|
|
|
|
w = await world();
|
|
chmodSync(join(w.runDir, 'session.json'), 0o644);
|
|
r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.deepEqual(r.json.runs[0].problems, ['session.json: mode 644, expected 600']);
|
|
|
|
w = await world();
|
|
writeFileSync(join(w.dataRoot, 'decoy.json'), JSON.stringify({ thread: THREAD, run: 'r0123456789ab' }), { mode: 0o600 });
|
|
unlinkSync(join(w.runDir, 'session.json')); symlinkSync(join(w.dataRoot, 'decoy.json'), join(w.runDir, 'session.json'));
|
|
r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.equal(r.error?.exitCode, 1);
|
|
assert.match(r.json.runs[0].problems[0], /^session\.json: cannot open \(ELOOP\)$/);
|
|
|
|
w = await world();
|
|
rmSync(w.runDir, { recursive: true });
|
|
r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.deepEqual(r.json.runs[0].problems, ['run directory missing']);
|
|
});
|
|
|
|
test('a manifest file outside the run directory or a mismatched run fails', async () => {
|
|
const w = await world();
|
|
const manifest = join(w.runDir, 'bundle/manifest.json');
|
|
writeFileSync(join(w.dataRoot, 'outside'), 'x', { mode: 0o600 });
|
|
rmSync(manifest);
|
|
writeFileSync(manifest, JSON.stringify({ run: 'r0123456789ab', files: { '../../../../../outside': sha('x') } }), { mode: 0o600 });
|
|
let r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.deepEqual(r.json.runs[0].problems, ['bundle file ../../../../../outside: outside the run directory']);
|
|
rmSync(manifest);
|
|
writeFileSync(manifest, JSON.stringify({ run: 'rbbbbbbbbbbbb', files: {} }), { mode: 0o600 });
|
|
r = await evidence(w, ['--thread', THREAD, '--verify', '--json']);
|
|
assert.match(r.json.runs[0].problems[0], /names run rbbbbbbbbbbbb/);
|
|
});
|
|
|
|
test('no launch for the thread, an unknown org and a usage error', async () => {
|
|
const w = await world();
|
|
let r = await evidence(w, ['--thread', '99999999-2222-4333-8444-555555555555', '--json']);
|
|
assert.equal(r.error?.exitCode, 2);
|
|
assert.deepEqual(r.json.runs, []);
|
|
const x = io(w.env);
|
|
await assert.rejects(runCommand(['evidence', 'beta', '--thread', THREAD], x, { system: { dataRoot: w.dataRoot } }), e => e.exitCode === 2 && e.code === 'org-unknown');
|
|
await assert.rejects(runCommand(['evidence', 'alpha'], x, { system: { dataRoot: w.dataRoot } }), e => e.exitCode === 4);
|
|
await assert.rejects(runCommand(['evidence', 'alpha', '--thread', THREAD], io({ PATH: process.env.PATH, HOME }), { system: { dataRoot: w.dataRoot } }), e => e.exitCode === 3 && e.code === 'run-evidence-instance-refused');
|
|
});
|
|
|
|
// PR #1916 (org-redact5): a refusal names the launch log by label, never by host path.
|
|
const SENTINEL = 'sk-test-NOT-A-SECRET-1234';
|
|
const noPath = (e, w) => !e.message.includes(SENTINEL) && !e.message.includes(w.dataRoot) && !e.message.includes(w.log);
|
|
|
|
test('a launch log of the wrong mode refuses by label, never by host path', async () => {
|
|
const w = await world({ under: join(HOME, SENTINEL) });
|
|
chmodSync(w.log, 0o644);
|
|
const r = await evidence(w, ['--thread', THREAD, '--json']);
|
|
assert.equal(r.error?.exitCode, 3);
|
|
assert.equal(r.error.message, 'run-evidence-refused: launch log: mode 644, expected 600');
|
|
assert.ok(noPath(r.error, w), r.error.message);
|
|
});
|
|
|
|
test('no launch for the thread names the launch log by label, never by host path', async () => {
|
|
const w = await world({ under: join(HOME, SENTINEL) });
|
|
const r = await evidence(w, ['--thread', '99999999-2222-4333-8444-555555555555', '--json']);
|
|
assert.equal(r.error?.exitCode, 2);
|
|
assert.equal(r.error.message, 'no launch for thread 99999999-2222-4333-8444-555555555555 in the launch log');
|
|
assert.ok(noPath(r.error, w), r.error.message);
|
|
});
|
|
|
|
test('a thread name resolves through the bus with the org as business', async () => {
|
|
const w = await world();
|
|
const seen = [];
|
|
const callFor = p => { seen.push([p.values['--business'], p.flags.has('--tester')]); return async verb => verb === 'thread.list' ? [{ id: THREAD, name: 'chat', project: 'p1' }, { id: OTHER, name: 'other', project: 'p1' }] : [{ id: 'p1', name: 'work' }]; };
|
|
let r = await evidence(w, ['--thread', 'work/chat', '--verify', '--json', '--tester'], { callFor });
|
|
assert.equal(r.error, null, r.error?.message);
|
|
assert.equal(r.json.thread.id, THREAD);
|
|
r = await evidence(w, ['--thread', 'chat', '--verify', '--json'], { callFor });
|
|
assert.equal(r.error, null, r.error?.message);
|
|
assert.equal(r.json.thread.label, 'chat');
|
|
assert.deepEqual(seen, [['alpha', true], ['alpha', false]]);
|
|
});
|