stack/agents/researcher/work/i1-proxy-test/analyze.mjs

#!/usr/bin/env node
// Row 96 (#1590) analysis. Usage: node analyze.mjs <out-dir>
//
// Reads what the test left in out/ and prints, per client run, what the client sent to the
// proxy (from the proxy's own request log: incoming headers and body) next to what the
// stand-in upstream actually received (from the upstream log). No network, no secrets:
// every key in these files is made up.
import { readFileSync, readdirSync, existsSync } from "node:fs";
import path from "node:path";

const out = process.argv[2];
if (!out) { console.error("usage: analyze.mjs <out-dir>"); process.exit(2); }
const read = (f) => readFileSync(path.join(out, f), "utf8");
const lines = (f) => (existsSync(path.join(out, f)) ? read(f).split("\n").filter(Boolean) : []);

// --- proxy request logs ---------------------------------------------------------------
function parseLog(text) {
  const sections = {};
  let name = null;
  for (const line of text.split("\n")) {
    const m = /^=== (.+) ===$/.exec(line);
    if (m) { name = m[1]; sections[name] = []; continue; }
    if (name) sections[name].push(line);
  }
  const headers = (arr) => {
    const h = {};
    for (const l of arr) {
      if (l === "Body:") break;
      const m = /^([A-Za-z0-9-]+): (.*)$/.exec(l);
      if (m) h[m[1].toLowerCase()] = m[2];
    }
    return h;
  };
  const bodyAfter = (arr, marker) => {
    const i = marker ? arr.indexOf(marker) : -1;
    const raw = (marker ? arr.slice(i + 1) : arr).find((l) => l.startsWith("{"));
    try { return raw ? JSON.parse(raw) : null; } catch { return null; }
  };
  const info = Object.fromEntries((sections["REQUEST INFO"] ?? []).map((l) => l.split(/: (.*)/s).slice(0, 2)).filter((p) => p[0]));
  const up = sections["API REQUEST 1"];
  const resp = sections["RESPONSE"] ?? [];
  return {
    ts: info.Timestamp,
    url: info.URL,
    incoming: { headers: headers(sections["HEADERS"] ?? []), body: bodyAfter(sections["REQUEST BODY"] ?? []) },
    sentUp: up ? { headers: headers(up.slice(up.indexOf("Headers:") + 1)), body: bodyAfter(up, "Body:"), auth: (up.find((l) => l.startsWith("Auth:")) ?? "") } : null,
    status: /^Status: (\d+)/.exec(resp[0] ?? "")?.[1] ?? null,
  };
}
const logDir = path.join(out, "proxy-logs");
const logs = existsSync(logDir)
  ? readdirSync(logDir).filter((f) => f.startsWith("v1-messages-")).map((f) => ({ f, ...parseLog(readFileSync(path.join(logDir, f), "utf8")) })).sort((a, b) => String(a.ts).localeCompare(String(b.ts)))
  : [];

// --- helpers --------------------------------------------------------------------------
const count = (n) => (Array.isArray(n) ? n.reduce((s, v) => s + count(v), 0) : n && typeof n === "object" ? Object.entries(n).reduce((s, [k, v]) => s + (k === "cache_control" ? 1 : 0) + count(v), 0) : 0);
const TRANSPORT = new Set(["host", "content-length", "connection", "accept-encoding", "accept", "transfer-encoding"]);
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
const keyLabel = (masked) => {
  const m = /(?:Bearer )?(sk-s\S*|stan\S*)/.exec(masked ?? "");
  const tail = m ? m[1].slice(-4) : null;
  const keys = Object.fromEntries(["A", "B"].map((k) => [k, existsSync(path.join(out, `key-${k}.txt`)) ? read(`key-${k}.txt`).trim() : null]));
  const hit = Object.entries(keys).find(([, v]) => v && tail && v.endsWith(tail));
  return hit ? `key ${hit[0]} (${masked})` : masked ? `${masked}` : "none";
};
const allUp = lines("upstream.all.jsonl");
const labels = readdirSync(out).filter((f) => f.endsWith(".upstream.jsonl")).map((f) => f.replace(/\.upstream\.jsonl$/, "")).sort();
const allLabels = readdirSync(out).filter((f) => f.endsWith(".rc")).map((f) => f.replace(/\.rc$/, "")).sort();

console.log(`proxy request logs parsed: ${logs.length}; upstream requests recorded: ${allUp.length}`);
console.log(`stand-in upstream accepted only its own made-up key; any other presented key would be marked ok:false below.\n`);

for (const label of allLabels) {
  const rc = read(`${label}.rc`).trim();
  const stdout = read(`${label}.stdout`).trim().replace(/\s+/g, " ").slice(0, 160);
  const stderr = existsSync(path.join(out, `${label}.stderr`)) ? read(`${label}.stderr`).trim().replace(/\s+/g, " ").slice(0, 200) : "";
  const slice = lines(`${label}.upstream.jsonl`).map((l) => JSON.parse(l));
  console.log(`## ${label}: exit ${rc}${rc === "124" ? " (killed by the harness timeout)" : ""}; upstream requests ${slice.length}`);
  console.log(`   client stdout: ${stdout || "(empty)"}`);
  if (stderr) console.log(`   client stderr: ${stderr}`);
  const qf = path.join(out, `usage-queue.${label}.json`);
  if (existsSync(qf)) {
    let events = [];
    try { events = JSON.parse(readFileSync(qf, "utf8")); } catch { /* not JSON */ }
    const mine = Array.isArray(events) ? events : [];
    console.log(`   usage queue after the run: ${mine.length} event(s)${mine.length ? "; " + mine.map((e) => `client key ${e.api_key ?? "?"} (${e.access_provider}) failed=${e.failed}${e.fail ? ` status=${e.fail.status_code}` : ""}`).slice(0, 3).join(" | ") : ""}`);
  }
  for (const [i, u] of slice.entries()) {
    const log = logs.find((l) => !l.claimed && l.sentUp && same(l.sentUp.body, u.body) && (l.claimed = true));
    const inc = log?.incoming;
    const uh = u.headers;
    const ih = inc?.headers ?? {};
    const dropped = Object.keys(ih).filter((h) => !TRANSPORT.has(h) && h !== "authorization" && h !== "x-api-key" && !(h in uh));
    const changed = Object.keys(ih).filter((h) => !TRANSPORT.has(h) && h !== "authorization" && h !== "x-api-key" && h in uh && ih[h] !== uh[h]);
    const added = Object.keys(uh).filter((h) => !TRANSPORT.has(h) && h !== "authorization" && h !== "x-api-key" && !(h in ih));
    console.log(`   upstream request ${i + 1}: ${u.method} ${u.url}  reply=${u.summary?.reply ?? "-"} stream=${u.summary?.stream} tools=${JSON.stringify(u.summary?.tools)} tool_result_in_history=${u.summary?.has_tool_result}`);
    if (!log) { console.log("     (no matching proxy request log)"); continue; }
    console.log(`     key the proxy saw   : ${keyLabel(ih.authorization ?? ih["x-api-key"])}  [header ${ih.authorization ? "Authorization: Bearer" : "x-api-key"}]`);
    console.log(`     key the upstream saw: ${u.auth.presented} via ${u.auth.header}, accepted=${u.auth.ok}; client key present in upstream headers: ${Object.values(uh).some((v) => /sk-standin-agent/.test(String(v)))}`);
    console.log(`     anthropic-version   : client ${ih["anthropic-version"] ?? "-"} -> upstream ${uh["anthropic-version"] ?? "-"}`);
    console.log(`     anthropic-beta      : ${ih["anthropic-beta"] === uh["anthropic-beta"] ? "identical" : "DIFFERENT"} (${(uh["anthropic-beta"] ?? "-").split(",").length} flags upstream; client ${(ih["anthropic-beta"] ?? "-").split(",").length})`);
    if (ih["anthropic-beta"] !== uh["anthropic-beta"]) console.log(`       client  : ${ih["anthropic-beta"] ?? "-"}\n       upstream: ${uh["anthropic-beta"] ?? "-"}`);
    console.log(`     cache_control blocks: client ${count(inc.body)} -> upstream ${u.summary?.cache_control_blocks}`);
    console.log(`     body                : ${same(inc.body, u.body) ? "byte-for-byte equal as JSON" : "CHANGED"}`);
    console.log(`     headers dropped: ${dropped.join(", ") || "none"}; changed: ${changed.map((h) => `${h} ${ih[h]} -> ${uh[h]}`).join("; ") || "none"}; added: ${added.join(", ") || "none"}`);
  }
}

// --- direct probes (curl to the proxy) and what the usage queue recorded for each ------
console.log("\n## direct probes (curl, one request each)");
for (const l of lines("probes.log")) {
  const label = l.split(" ")[0];
  const qf = path.join(out, `usage-queue.${label}.json`);
  let events = [];
  try { events = JSON.parse(readFileSync(qf, "utf8")); } catch { /* no queue file */ }
  const ev = Array.isArray(events) ? events : [];
  console.log(`   ${l.slice(0, 120)}\n      usage queue: ${ev.length} event(s)${ev.length ? "; client key " + ev.map((e) => `${e.api_key} failed=${e.failed}`).slice(0, 3).join(" | ") : ""}`);
}

// --- refusals the proxy logged (requests that never reached the upstream) ------------
const refused = logs.filter((l) => !l.sentUp);
console.log(`\nproxy request logs with no upstream call: ${refused.length}`);
const main = existsSync(path.join(logDir, "main.log")) ? readFileSync(path.join(logDir, "main.log"), "utf8") : "";
const status = {};
for (const m of main.matchAll(/gin_logger\.go:\d+\] (\d{3}) \|[^"]*"(\/[^"]*)"/g)) {
  const k = `${m[1]} ${m[2].split("?")[0]}`;
  status[k] = (status[k] ?? 0) + 1;
}
console.log("proxy HTTP status counts (main.log):", JSON.stringify(status));