jason.woltjeandClaude Opus 5.5
771fc3d270
Pi 0.85.1 and Claude Code 2.1.289 (Agent SDK 0.3.289), 34 cases against a scripted local model in a loopback-only network namespace. MATRIX.md has the command, outcome, model-visible message and fail-closed column per case, and one rely-on line per case. Evidence under evidence/. Co-Authored-By: Claude Opus 5.5 <[email protected]>
20 lines
1.2 KiB
JavaScript
20 lines
1.2 KiB
JavaScript
// Row S0: one summary block per run directory: outcome, the requests the
|
|
// mock received, the tool results the model saw, and the first stderr lines.
|
|
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
|
const runs = process.argv[2];
|
|
const names = process.argv.length > 3 ? process.argv.slice(3) : readdirSync(runs).sort();
|
|
const read = (p) => (existsSync(p) ? readFileSync(p, "utf8") : "");
|
|
for (const n of names) {
|
|
const r = `${runs}/${n}`;
|
|
const reqs = read(`${r}/mock.jsonl`).split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((x) => x.method === "POST" && x.tools);
|
|
const withTools = reqs.filter((x) => x.tools.length);
|
|
const last = withTools.at(-1);
|
|
console.log(`### ${n}`);
|
|
console.log(read(`${r}/outcome.txt`).trim().split("\n").filter((l) => !l.startsWith("uid")).join(" | "));
|
|
console.log(`model requests: ${reqs.length} (with tools: ${withTools.length}); tools offered: ${JSON.stringify(withTools[0]?.tools ?? [])}`);
|
|
console.log(`model saw: ${last ? JSON.stringify(last.toolResults) : "(no request with tools)"}`);
|
|
const err = read(`${r}/stderr.txt`).trim().split("\n").filter(Boolean);
|
|
console.log(`stderr (${err.length} lines): ${err.slice(0, 4).join(" // ").slice(0, 600)}`);
|
|
console.log("");
|
|
}
|