Pi 0.85.1 and Claude Code 2.1.289 (Agent SDK 0.3.289), 34 cases against a scripted local model in a loopback-only network namespace. MATRIX.md has the command, outcome, model-visible message and fail-closed column per case, and one rely-on line per case. Evidence under evidence/. Co-Authored-By: Claude Opus 5.5 <[email protected]>
89 lines
4.8 KiB
JavaScript
89 lines
4.8 KiB
JavaScript
// A scripted stand-in for the Anthropic Messages API, for row S0's probes.
|
|
// It never calls a real model. Each request that offers tools gets the next
|
|
// tool call from MOCK_SCRIPT; once every scripted call has a tool_result in
|
|
// the conversation, it answers with plain text and ends the turn. A request
|
|
// without tools (titles, summaries) gets a short text answer.
|
|
//
|
|
// Every request is logged to MOCK_LOG as one JSON line, including each
|
|
// tool_result the harness sent back: that is "the message the model saw".
|
|
//
|
|
// Env: MOCK_PORT (default 47123), MOCK_SCRIPT (JSON array of {name, input}),
|
|
// MOCK_LOG (path).
|
|
|
|
import { createServer } from "node:http";
|
|
import { appendFileSync } from "node:fs";
|
|
|
|
const port = Number(process.env.MOCK_PORT ?? 47123);
|
|
const script = JSON.parse(process.env.MOCK_SCRIPT ?? "[]");
|
|
const logPath = process.env.MOCK_LOG ?? "/dev/null";
|
|
const log = (v) => appendFileSync(logPath, JSON.stringify({ at: new Date().toISOString(), ...v }) + "\n");
|
|
|
|
const textOf = (c) => (typeof c === "string" ? c : Array.isArray(c) ? c.map((b) => b.text ?? (b.type === "image" ? "[image]" : "")).join("") : "");
|
|
|
|
function toolResults(messages) {
|
|
const out = [];
|
|
for (const m of messages ?? []) {
|
|
if (m.role !== "user" || !Array.isArray(m.content)) continue;
|
|
for (const b of m.content) if (b.type === "tool_result") out.push({ tool_use_id: b.tool_use_id, is_error: b.is_error ?? false, content: textOf(b.content) });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function reply(body) {
|
|
const tools = Array.isArray(body.tools) ? body.tools.map((t) => t.name) : [];
|
|
const results = toolResults(body.messages);
|
|
if (tools.length === 0) return { kind: "text", text: "probe" };
|
|
const step = script[results.length];
|
|
if (!step) return { kind: "text", text: `PROBE-DONE results=${JSON.stringify(results)}` };
|
|
return { kind: "tool", id: `toolu_probe_${results.length + 1}`, name: step.name, input: step.input };
|
|
}
|
|
|
|
function sse(res, model, r) {
|
|
const send = (event, data) => res.write(`event: ${event}\ndata: ${JSON.stringify({ type: event, ...data })}\n\n`);
|
|
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" });
|
|
send("message_start", { message: { id: `msg_probe_${Date.now()}`, type: "message", role: "assistant", model, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 10, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } } });
|
|
if (r.kind === "tool") {
|
|
send("content_block_start", { index: 0, content_block: { type: "tool_use", id: r.id, name: r.name, input: {} } });
|
|
send("content_block_delta", { index: 0, delta: { type: "input_json_delta", partial_json: JSON.stringify(r.input) } });
|
|
} else {
|
|
send("content_block_start", { index: 0, content_block: { type: "text", text: "" } });
|
|
send("content_block_delta", { index: 0, delta: { type: "text_delta", text: r.text } });
|
|
}
|
|
send("content_block_stop", { index: 0 });
|
|
send("message_delta", { delta: { stop_reason: r.kind === "tool" ? "tool_use" : "end_turn", stop_sequence: null }, usage: { output_tokens: 5 } });
|
|
send("message_stop", {});
|
|
res.end();
|
|
}
|
|
|
|
function json(res, model, r) {
|
|
const content = r.kind === "tool" ? [{ type: "tool_use", id: r.id, name: r.name, input: r.input }] : [{ type: "text", text: r.text }];
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
res.end(JSON.stringify({ id: `msg_probe_${Date.now()}`, type: "message", role: "assistant", model, content, stop_reason: r.kind === "tool" ? "tool_use" : "end_turn", stop_sequence: null, usage: { input_tokens: 10, output_tokens: 5 } }));
|
|
}
|
|
|
|
createServer((req, res) => {
|
|
let raw = "";
|
|
req.on("data", (c) => (raw += c));
|
|
req.on("end", () => {
|
|
if (req.method !== "POST" || !/\/messages(\?|$)/.test(req.url) || /count_tokens/.test(req.url)) {
|
|
log({ method: req.method, url: req.url, answered: req.url.includes("count_tokens") ? "count_tokens" : 404 });
|
|
if (req.url.includes("count_tokens")) {
|
|
res.writeHead(200, { "content-type": "application/json" });
|
|
return res.end(JSON.stringify({ input_tokens: 10 }));
|
|
}
|
|
res.writeHead(404, { "content-type": "application/json" });
|
|
return res.end(JSON.stringify({ type: "error", error: { type: "not_found_error", message: "probe mock" } }));
|
|
}
|
|
let body = {};
|
|
try {
|
|
body = JSON.parse(raw);
|
|
} catch {
|
|
// logged below with no fields
|
|
}
|
|
const r = reply(body);
|
|
log({ method: req.method, url: req.url, model: body.model, stream: body.stream === true, tools: Array.isArray(body.tools) ? body.tools.map((t) => t.name) : [], toolResults: toolResults(body.messages), answer: r });
|
|
if (body.stream) sse(res, body.model ?? "probe-model", r);
|
|
else json(res, body.model ?? "probe-model", r);
|
|
});
|
|
}).listen(port, "127.0.0.1", () => log({ listening: port, script }));
|