jason.woltjeandClaude Opus 5.5
1abebe5321
Record for #1516 comment 26725: 34/34 cases reproduce, and a wrapped Claude command hook fails closed on crash, missing path, noexec and hang. Co-Authored-By: Claude Opus 5.5 <[email protected]>
17 lines
1.5 KiB
JavaScript
17 lines
1.5 KiB
JavaScript
// Compares two evidence trees case by case: exit, target, gate calls, tools offered, and each tool result's
|
|
// is_error and content, with run paths and the trailing system-reminder normalised away.
|
|
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
|
const [a, b, ...only] = process.argv.slice(2);
|
|
const read = (p) => (existsSync(p) ? readFileSync(p, "utf8") : "");
|
|
const norm = (s) => s.replace(/\n*<system-reminder>[\s\S]*<\/system-reminder>/, "").replace(/\/[^\s\]"]*?\/(runs\/[a-z0-9-]+\/work|slice1-probes|probes)\//g, "<P>/").replace(/~\/[^\s\]"]*?\/(runs\/[a-z0-9-]+\/work|probes)\//g, "<P>/").replace(/~<P>/g, "<P>");
|
|
const sig = (d) => {
|
|
const o = Object.fromEntries(read(`${d}/outcome.txt`).split(/\s+/).filter((x) => x.includes("=")).map((x) => x.split("=")));
|
|
const reqs = read(`${d}/mock.jsonl`).split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((x) => x.method === "POST");
|
|
return JSON.stringify({ exit: o.exit, target: o.target, loaded: o.gate_loaded, calls: o.gate_calls, reqs: reqs.length, tools: reqs[0]?.tools,
|
|
saw: reqs.flatMap((r) => r.toolResults ?? []).map((t) => [t.is_error, norm(typeof t.content === "string" ? t.content : JSON.stringify(t.content))]) });
|
|
};
|
|
const cases = only.length ? only : readdirSync(a, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name);
|
|
let same = 0;
|
|
for (const c of cases) { const x = sig(`${a}/${c}`), y = sig(`${b}/${c}`); if (x === y) same++; else console.log("DIFF", c, "\n ", x, "\n ", y); }
|
|
console.log(`${same}/${cases.length} identical`);
|