salvage 2026-11-06: uncommitted work from src/mosaic-stack-rocko-124 (disk cleanup #824)
This commit is contained in:
1 parent
9098c33f7f
commit
ebfb0df1a4
37 files changed
+1367
-28
No files matched your search
@@ -26,7 +26,7 @@ do: a usage source reads what a session used after each turn, and a limit
|
||||
check answers whether a launch may start (`mayLaunch`) and whether a running
|
||||
session must stop at the end of its turn (`mustStop`). The arguments and
|
||||
return values of these functions are set by the rows that call them (row 124
|
||||
converts the first usage source). No usage-source or limit-check module ships yet.
|
||||
converts the first usage source). The transcript usage source ships with row 124; no limit-check module ships yet.
|
||||
|
||||
A `search-source` answers the broker's `web.search` (REQ-HARN-2, REQ-PLUG-2, decision 0b81b0c5): `search(settings, query, deps, { signal })` resolves `{ query, total, results: [{ title, url, snippet }] }` or throws a `WebRefusal`; `signal` is an `AbortSignal` the broker fires at its deadline, and a module must stop its request when it does. The kind switches on one module, and the module's name is the source: `searxng-url` queries the operator's SearXNG at the business var `web.searxng`; `searxng-bundled` queries the SearXNG the Stack installs, at a fixed loopback address (the install is REQ-INST-1's first component); `harness-builtin` is served by the harness itself: a granted claude-code role gets WebSearch and WebFetch, the broker serves no web verbs, and `search` only refuses. The choice and every change to it are the `plugins` setting, logged in `changes.jsonl` by the plug-in setter.
|
||||
|
||||
@@ -122,3 +122,12 @@ Limits of the checks:
|
||||
until the process restarts.
|
||||
- The loader is not wired into any host process yet. The rows that call
|
||||
the first two kinds (124, 125, 126) wire it.
|
||||
|
||||
## Transcript usage source (row 124)
|
||||
|
||||
`usage-source/transcript` now ships interface 1's `readTurn({harness, root})`.
|
||||
The bus brackets each actual harness turn, reads all sources through the loader,
|
||||
and appends baselines and outcomes. See that module's README for JSONL formats,
|
||||
security/resource bounds and the best-reading limitation. No limit check ships
|
||||
with this row. Enable transcript through the human plug-in settings writer;
|
||||
a restart/re-pin remains a separate authorized window.
|
||||
@@ -0,0 +1,138 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, symlinkSync, linkSync } from 'node:fs';
|
||||
import fs, { appendFileSync } from 'node:fs';
|
||||
import { syncBuiltinESMExports } from 'node:module';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { readTurn } from '../usage-source/transcript/index.mjs';
|
||||
function fixture(t, harness = 'pi') {
|
||||
const root = mkdtempSync(join(tmpdir(), 'usage-source-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dir = harness === 'pi' ? root : join(root, 'project');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const file = join(dir, 'session.jsonl');
|
||||
const pi = { type: 'message', id: 'assistant1', message: { role: 'assistant', usage: { input: 12, output: 3, cacheWrite: 4, cacheRead: 5 } } };
|
||||
const claude = { type: 'assistant', uuid: 'record1', message: { id: 'assistant1', role: 'assistant', usage: { input_tokens: 12, output_tokens: 3, cache_creation_input_tokens: 4, cache_read_input_tokens: 5 } } };
|
||||
const entry = harness === 'pi' ? pi : claude;
|
||||
const put = value => writeFileSync(file, JSON.stringify(value) + '\n');
|
||||
put(entry);
|
||||
return { root, file, entry, put, read: () => readTurn({ harness, root }) };
|
||||
}
|
||||
for (const harness of ['pi', 'claude-code']) test(`REQ-LIMIT-4 ${harness}: reports all token kinds and deduplicates assistant ids`, async t => {
|
||||
const f = fixture(t, harness);
|
||||
writeFileSync(f.file, [f.entry, f.entry].map(JSON.stringify).join('\n') + '\n');
|
||||
const result = await f.read();
|
||||
assert.equal(result.status, 'known');
|
||||
assert.deepEqual(Object.values(result.entries), [{ input: 12, output: 3, cacheWrite: 4, cacheRead: 5 }]);
|
||||
});
|
||||
for (const [name, change, code] of [
|
||||
['cut line', f => writeFileSync(f.file, '{"type":'), 'transcript-parse'],
|
||||
['invalid UTF-8', f => writeFileSync(f.file, Buffer.concat([Buffer.from('{"type":"user","text":"'), Buffer.from([0xff]), Buffer.from('"}\n')])), 'transcript-parse'],
|
||||
['missing input', f => { delete f.entry.message.usage.input; f.put(f.entry); }, 'transcript-usage'],
|
||||
['negative count', f => { f.entry.message.usage.output = -1; f.put(f.entry); }, 'transcript-usage'],
|
||||
['fractional count', f => { f.entry.message.usage.output = 0.5; f.put(f.entry); }, 'transcript-usage'],
|
||||
['unsafe integer', f => { f.entry.message.usage.output = Number.MAX_SAFE_INTEGER + 1; f.put(f.entry); }, 'transcript-usage'],
|
||||
['file size', f => writeFileSync(f.file, ' '.repeat(8 * 1024 * 1024 + 1)), 'transcript-size'],
|
||||
['line size', f => writeFileSync(f.file, JSON.stringify({ type: 'user', text: 'x'.repeat(1024 * 1024) }) + '\n'), 'transcript-line'],
|
||||
['symlink file', f => { const other = join(f.root, 'other'); writeFileSync(other, JSON.stringify(f.entry) + '\n'); rmSync(f.file); symlinkSync(other, f.file); }, 'transcript-path'],
|
||||
['hardlink file', f => linkSync(f.file, join(f.root, 'linked')), 'transcript-path'],
|
||||
]) test(`REQ-LIMIT-4 ${name} is unknown, never zero or content`, async t => {
|
||||
const f = fixture(t); change(f);
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code });
|
||||
});
|
||||
test('REQ-LIMIT-4 a symlink directory anywhere in the root refuses', async t => {
|
||||
const f = fixture(t), alias = join(f.root, 'alias');
|
||||
symlinkSync(f.root, alias);
|
||||
assert.deepEqual(await readTurn({ harness: 'pi', root: alias }), { status: 'unknown', code: 'transcript-path' });
|
||||
});
|
||||
test('REQ-LIMIT-4 bounded directory entries refuse before scanning an arbitrary tree', async t => {
|
||||
const f = fixture(t);
|
||||
for (let n = 0; n < 513; n++) writeFileSync(join(f.root, `ignored-${n}`), '');
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-entries' });
|
||||
});
|
||||
test('REQ-LIMIT-4 conflicting repeated usage refuses rather than double counting', async t => {
|
||||
const f = fixture(t), second = structuredClone(f.entry); second.message.usage.input++;
|
||||
writeFileSync(f.file, [f.entry, second].map(JSON.stringify).join('\n') + '\n');
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-usage' });
|
||||
});
|
||||
|
||||
|
||||
test('REQ-LIMIT-4 oversized metadata refuses before reading any transcript bytes', async t => {
|
||||
const f = fixture(t); writeFileSync(f.file, 'x'.repeat(8 * 1024 * 1024 + 1));
|
||||
let reads = 0;
|
||||
const original = fs.readSync;
|
||||
fs.readSync = (...args) => { reads++; return original(...args); };
|
||||
syncBuiltinESMExports();
|
||||
try { assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-size' }); assert.equal(reads, 0); }
|
||||
finally { fs.readSync = original; syncBuiltinESMExports(); }
|
||||
});
|
||||
test('REQ-LIMIT-4 a file growing after fstat cannot exceed the streaming byte budget', async t => {
|
||||
const f = fixture(t);
|
||||
let grew = false;
|
||||
const original = fs.readSync;
|
||||
fs.readSync = (...args) => {
|
||||
if (!grew) { grew = true; appendFileSync(f.file, 'x'.repeat(8 * 1024 * 1024)); }
|
||||
return original(...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
try { assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-size' }); }
|
||||
finally { fs.readSync = original; syncBuiltinESMExports(); }
|
||||
});
|
||||
test('REQ-LIMIT-4 a same-size edit during a read is unknown rather than a stable snapshot', async t => {
|
||||
const f = fixture(t);
|
||||
const original = fs.readSync;
|
||||
let edited = false;
|
||||
fs.readSync = (...args) => {
|
||||
if (!edited) { edited = true; const bytes = fs.readFileSync(f.file); fs.writeFileSync(f.file, bytes); }
|
||||
return original(...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
try { assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-changed' }); }
|
||||
finally { fs.readSync = original; syncBuiltinESMExports(); }
|
||||
});
|
||||
test('REQ-LIMIT-4 record count is bounded even when each record is tiny', async t => {
|
||||
const f = fixture(t); writeFileSync(f.file, '{"type":"user"}\n'.repeat(20001));
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-records' });
|
||||
});
|
||||
test('REQ-LIMIT-4 missing assistant usage, malformed JSON and conflicting identities are unknown', async t => {
|
||||
const f = fixture(t);
|
||||
for (const value of [{ type: 'message', id: 'a', message: { role: 'assistant' } }, { ...f.entry, id: '../bad' }]) {
|
||||
f.put(value); assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-usage' });
|
||||
}
|
||||
writeFileSync(f.file, 'not-json\n'); assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-parse' });
|
||||
});
|
||||
|
||||
test('REQ-LIMIT-4 caller roots, unsupported harnesses and non-object JSON never become zero usage', async t => {
|
||||
assert.deepEqual(await readTurn({ harness: 'pi', root: 'relative' }), { status: 'unknown', code: 'transcript-path' });
|
||||
const f = fixture(t);
|
||||
assert.deepEqual(await readTurn({ harness: 'other', root: f.root }), { status: 'unknown', code: 'transcript-harness' });
|
||||
for (const value of [null, [], 7]) { f.put(value); assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-parse' }); }
|
||||
});
|
||||
|
||||
|
||||
test('REQ-LIMIT-4 even a complete JSON record with no final newline is an unknown cut line', async t => {
|
||||
const f = fixture(t); writeFileSync(f.file, JSON.stringify(f.entry));
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-parse' });
|
||||
});
|
||||
|
||||
|
||||
test('REQ-LIMIT-4 final file opens are nonblocking before hostile file type is known', async t => {
|
||||
const f = fixture(t), original = fs.openSync;
|
||||
const flags = [];
|
||||
fs.openSync = (path, mode, ...args) => {
|
||||
if (String(path).endsWith('/session.jsonl')) flags.push(mode);
|
||||
return original(path, mode, ...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
try {
|
||||
assert.equal((await f.read()).status, 'known');
|
||||
assert.equal(flags.length, 1);
|
||||
assert.ok(flags[0] & fs.constants.O_NONBLOCK);
|
||||
} finally { fs.openSync = original; syncBuiltinESMExports(); }
|
||||
});
|
||||
test('REQ-LIMIT-4 invalid transcript entry names refuse before file lookup', async t => {
|
||||
const f = fixture(t);
|
||||
writeFileSync(join(f.root, 'bad name.jsonl'), JSON.stringify(f.entry) + '\n');
|
||||
assert.deepEqual(await f.read(), { status: 'unknown', code: 'transcript-path' });
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
# Transcript usage source
|
||||
|
||||
REQ-LIMIT-4, interface version 1. Enable with
|
||||
`"plugins": { "usage-source": ["transcript"] }` in the organization file,
|
||||
keeping its other plug-in kinds. Human changes use `mosaic plugins set` and
|
||||
the current revision. No agent edits the live organization file.
|
||||
|
||||
`readTurn({ harness, root })` returns `{status:'known', entries}` or
|
||||
`{status:'unknown', code}`. Entries map assistant message identities to
|
||||
`{input, output, cacheWrite, cacheRead}`. This is a snapshot; the bus stores a
|
||||
baseline and computes the turn's new entries. No known new entry means unknown,
|
||||
not zero. The bus attributes records to the bound run/instance/provider/model;
|
||||
the account is null. No token limit or enforcement ships here.
|
||||
|
||||
For Pi, root is the run's session directory, or a resumed thread's persistent
|
||||
session directory. JSONL `message` entries with `message.role === 'assistant'`
|
||||
have `id` and `message.usage` input/output/cacheWrite/cacheRead counters.
|
||||
For Claude Code, root is `home/.claude/projects`, or the thread's persistent
|
||||
`claude-projects` directory: one project-directory level, then session JSONL
|
||||
files. Assistant entries have message.id and message.usage input_tokens,
|
||||
output_tokens, cache_creation_input_tokens and cache_read_input_tokens. Claude's
|
||||
absent optional cache fields mean no reported cache usage; missing required
|
||||
input/output is unknown. Duplicate identical identities count once; conflicting
|
||||
counters refuse the snapshot.
|
||||
|
||||
The source follows no user symlink: root components and project directories
|
||||
are opened with directory descriptors and O_NOFOLLOW; final opens add
|
||||
O_NONBLOCK. Files must be regular and single-link. Both fstat size and bytes
|
||||
actually read are bounded, and edits during a read refuse the snapshot. Limits:
|
||||
8 MiB total bytes, 1 MiB per line, 512 directory entries and 20,000 JSON records
|
||||
per snapshot. These are parser resource bounds, not policy token ceilings.
|
||||
Invalid UTF-8, malformed/cut JSONL or invalid counters return unknown, never
|
||||
zero. Reads/logs return no transcript text or parser exception text.
|
||||
|
||||
A session can edit its own transcript. Counts are the stack's best reading,
|
||||
not proof. Resumed history present before a run's next turn is excluded by its
|
||||
baseline. Rewriting that baseline makes the turn unknown. Missing data or a
|
||||
source disabled/failed at a restart leaves its pending turn visibly unknown.
|
||||
@@ -0,0 +1,113 @@
|
||||
// REQ-LIMIT-4: hostile transcripts are a best reading, never proof of usage.
|
||||
// Linux dirfds anchor every lookup; no caller-provided symlink is followed.
|
||||
import { constants, openSync, closeSync, fstatSync, readSync, opendirSync } from 'node:fs';
|
||||
import { isAbsolute } from 'node:path';
|
||||
|
||||
export const interfaceVersion = 1;
|
||||
const MAX_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_LINE = 1024 * 1024;
|
||||
const MAX_ENTRIES = 512;
|
||||
const MAX_RECORDS = 20000;
|
||||
const ID = /^[A-Za-z0-9_-]{1,128}$/;
|
||||
const NAME = /^[A-Za-z0-9_.-]{1,240}$/;
|
||||
const DIR = constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW;
|
||||
const FILE = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
||||
const refuse = code => { throw Object.assign(new Error(code), { code }); };
|
||||
const at = (fd, name) => `/proc/self/fd/${fd}/${name}`;
|
||||
|
||||
function rootFd(root) {
|
||||
if (typeof root !== 'string' || !isAbsolute(root) || root.split('/').includes('..')) refuse('transcript-path');
|
||||
let fd = openSync('/', DIR);
|
||||
try {
|
||||
for (const part of root.split('/').filter(Boolean)) {
|
||||
const next = openSync(at(fd, part), DIR);
|
||||
closeSync(fd); fd = next;
|
||||
}
|
||||
return fd;
|
||||
} catch (e) { closeSync(fd); throw e; }
|
||||
}
|
||||
function readFile(parent, name, budget) {
|
||||
const fd = openSync(at(parent, name), FILE);
|
||||
try {
|
||||
const before = fstatSync(fd);
|
||||
if (!before.isFile() || before.nlink !== 1) refuse('transcript-path');
|
||||
if (before.size > MAX_BYTES - budget.bytes) refuse('transcript-size');
|
||||
const parts = [];
|
||||
let total = 0;
|
||||
const buffer = Buffer.alloc(65536);
|
||||
for (;;) {
|
||||
const n = readSync(fd, buffer, 0, Math.min(buffer.length, MAX_BYTES - budget.bytes + 1), null);
|
||||
if (!n) break;
|
||||
total += n; budget.bytes += n;
|
||||
if (budget.bytes > MAX_BYTES) refuse('transcript-size');
|
||||
parts.push(Buffer.from(buffer.subarray(0, n)));
|
||||
}
|
||||
const after = fstatSync(fd);
|
||||
if (before.size !== after.size || total !== after.size || before.mtimeMs !== after.mtimeMs ||
|
||||
before.ctimeMs !== after.ctimeMs) refuse('transcript-changed');
|
||||
return Buffer.concat(parts);
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
function counters(usage, harness) {
|
||||
if (!usage || typeof usage !== 'object' || Array.isArray(usage)) refuse('transcript-usage');
|
||||
const fields = harness === 'pi' ? ['input', 'output', 'cacheWrite', 'cacheRead'] :
|
||||
['input_tokens', 'output_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'];
|
||||
const values = fields.map((key, i) => harness === 'claude-code' && i > 1 ? (usage[key] ?? 0) : usage[key]);
|
||||
if (values.some(n => !Number.isSafeInteger(n) || n < 0)) refuse('transcript-usage');
|
||||
return Object.fromEntries(['input', 'output', 'cacheWrite', 'cacheRead'].map((key, i) => [key, values[i]]));
|
||||
}
|
||||
function parse(bytes, harness, file, entries, budget) {
|
||||
let text;
|
||||
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); }
|
||||
catch { refuse('transcript-parse'); }
|
||||
if (text && !text.endsWith('\n')) refuse('transcript-parse');
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
if (Buffer.byteLength(line) > MAX_LINE) refuse('transcript-line');
|
||||
if (++budget.records > MAX_RECORDS) refuse('transcript-records');
|
||||
let record;
|
||||
try { record = JSON.parse(line); } catch { refuse('transcript-parse'); }
|
||||
if (!record || typeof record !== 'object' || Array.isArray(record)) refuse('transcript-parse');
|
||||
if (harness === 'pi' ? record.type !== 'message' || record.message?.role !== 'assistant' : record.type !== 'assistant') continue;
|
||||
const id = harness === 'pi' ? record.id : record.message?.id;
|
||||
if (typeof id !== 'string' || !ID.test(id)) refuse('transcript-usage');
|
||||
const value = counters(record.message?.usage, harness);
|
||||
const key = `${file}:${id}`;
|
||||
if (Object.hasOwn(entries, key) && JSON.stringify(entries[key]) !== JSON.stringify(value)) refuse('transcript-usage');
|
||||
entries[key] = value;
|
||||
}
|
||||
}
|
||||
function walk(fd, harness, depth, prefix, entries, budget) {
|
||||
const directory = opendirSync(`/proc/self/fd/${fd}`);
|
||||
try {
|
||||
let entry;
|
||||
while ((entry = directory.readSync())) {
|
||||
if (++budget.entries > MAX_ENTRIES) refuse('transcript-entries');
|
||||
if (!NAME.test(entry.name) || entry.name === '.' || entry.name === '..') refuse('transcript-path');
|
||||
if (harness === 'claude-code' && depth === 0) {
|
||||
// Only the standard project directory / session.jsonl shape is admitted.
|
||||
if (entry.isFile()) continue;
|
||||
const child = openSync(at(fd, entry.name), DIR);
|
||||
try { walk(child, harness, 1, entry.name + '/', entries, budget); }
|
||||
finally { closeSync(child); }
|
||||
} else if (entry.name.endsWith('.jsonl')) {
|
||||
parse(readFile(fd, entry.name, budget), harness, prefix + entry.name, entries, budget);
|
||||
}
|
||||
}
|
||||
} finally { directory.closeSync(); }
|
||||
}
|
||||
export async function readTurn({ harness, root }) {
|
||||
let fd;
|
||||
try {
|
||||
if (!['pi', 'claude-code'].includes(harness)) refuse('transcript-harness');
|
||||
fd = rootFd(root);
|
||||
const entries = Object.create(null);
|
||||
walk(fd, harness, 0, '', entries, { bytes: 0, entries: 0, records: 0 });
|
||||
return { status: 'known', entries };
|
||||
} catch (e) {
|
||||
// A new run can have no transcript directory before its first turn.
|
||||
if (e.code === 'ENOENT' && fd === undefined) return { status: 'known', entries: {} };
|
||||
const code = typeof e.code === 'string' && e.code.startsWith('transcript-') ? e.code : 'transcript-path';
|
||||
return { status: 'unknown', code };
|
||||
} finally { if (fd !== undefined) closeSync(fd); }
|
||||
}
|
||||
Reference in new issue
Block a user