stack/packages/conversation/tests/smoke.test.mjs

// CHAT-03 (#1507): the pinned Pi binary, started sealed in a scratch home with
// no credentials, answers the RPC commands the controller sends with the
// shapes the fake engine models. No prompt is sent, so no model is called.
//
// It also records pinned Pi's startup append (sdk.js 240–252): a session
// whose branch has no thinking_level_change entry gains one at every start,
// so the leaf moves after launch and the K8 load check fails closed on it.
//
// Set CHAT03_SMOKE_OUT=<file> to keep the exchange as evidence.

import { test, after } from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { PassThrough } from "node:stream";
import { LineSplitter, encodeLine } from "../src/framing.mjs";
import { PI_BIN, PI_VERSION, buildPiArgs, checkEnginePin, checkSeal } from "../src/pi-pin.mjs";
import { FakePi } from "./fake-pi.mjs";
import { REPO, assistantEntry, header, thinkingEntry, userEntry } from "./harness.mjs";

const scratch = mkdtempSync(join(tmpdir(), "chat03-smoke-"));
after(() => rmSync(scratch, { recursive: true, force: true }));

const COMMANDS = ["get_state", "get_commands", "clear_queue", "abort", "get_tree"];
const exchanges = {};

function session(name, entries, project = "proj") {
  const dir = join(scratch, project, ".pi", "state", "smoke", "sessions");
  mkdirSync(dir, { recursive: true });
  const file = join(dir, name);
  writeFileSync(file, [header(join(scratch, project)), ...entries].map((v) => JSON.stringify(v) + "\n").join(""));
  return file;
}

// A minimal environment: no inherited variables, so no provider key or token
// can reach Pi; HOME and the agent dir are empty scratch directories.
function scratchEnv(tag) {
  const home = join(scratch, `home-${tag}`);
  const agent = join(scratch, `agent-${tag}`);
  for (const d of [home, agent, join(home, ".config"), join(home, ".cache"), join(home, ".local", "share")]) mkdirSync(d, { recursive: true });
  return {
    home, agent,
    env: {
      PATH: process.env.PATH, HOME: home, PI_CODING_AGENT_DIR: agent, PI_OFFLINE: "1", PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0",
      XDG_CONFIG_HOME: join(home, ".config"), XDG_CACHE_HOME: join(home, ".cache"), XDG_DATA_HOME: join(home, ".local", "share"),
    },
  };
}

// Sends each command after the previous response arrives and records every
// line in order.
async function converse(input, output, { onExit = null } = {}) {
  const lines = [];
  const waiters = new Set();
  const split = new LineSplitter((l) => {
    lines.push(JSON.parse(l));
    for (const w of [...waiters]) w();
  });
  output.on("data", (c) => split.push(c));
  const reply = (id) => new Promise((resolve, reject) => {
    const t = setTimeout(() => reject(new Error(`no response to ${id}; lines ${JSON.stringify(lines).slice(0, 2000)}`)), 20000);
    const check = () => {
      const hit = lines.find((v) => v.type === "response" && v.id === id);
      if (hit) {
        waiters.delete(check);
        clearTimeout(t);
        resolve(hit);
      }
    };
    waiters.add(check);
    onExit?.(() => reject(new Error(`pi exited; lines ${JSON.stringify(lines).slice(0, 2000)}`)));
    check();
  });
  const responses = {};
  for (const type of COMMANDS) {
    const id = `smoke-${type}`;
    input.write(encodeLine({ id, type }));
    responses[type] = await reply(id);
  }
  return { lines, responses };
}

// `trusted` writes the agent dir's trust.json first, marking that project
// trusted the way an operator's `~/.pi/agent/trust.json` can; `after` is
// argv appended past the seal, which checkSeal would refuse.
async function realPi(tag, sessionFile, { project = "proj", trusted = null, after = [] } = {}) {
  const { home, agent, env } = scratchEnv(tag);
  assert.equal(existsSync(join(home, ".pi", "agent", "auth.json")), false);
  assert.deepEqual(readdirSync(agent), [], "the agent dir starts empty");
  if (trusted) writeFileSync(join(agent, "trust.json"), JSON.stringify({ [realpathSync(join(scratch, trusted))]: true }));
  const args = buildPiArgs({ sessionFile });
  checkSeal(args);
  const pi = spawn(process.execPath, [join(REPO, PI_BIN), ...args, ...after], { cwd: join(scratch, project), env, stdio: ["pipe", "pipe", "pipe"] });
  let stderr = "";
  pi.stderr.on("data", (c) => (stderr += c));
  const exited = new Promise((r) => pi.on("exit", (code, signal) => r({ code, signal })));
  try {
    const out = await converse(pi.stdin, pi.stdout, { onExit: (fn) => exited.then(fn) });
    return { ...out, stderr, agentFiles: readdirSync(agent).sort() };
  } finally {
    pi.stdin.end();
    pi.kill("SIGTERM");
    const t = setTimeout(() => pi.kill("SIGKILL"), 3000);
    await exited;
    clearTimeout(t);
  }
}

function fakePi(sessionFile) {
  const input = new PassThrough();
  const output = new PassThrough();
  new FakePi({ input, output, argv: buildPiArgs({ sessionFile }) });
  return converse(input, output);
}

const lastId = (file) => JSON.parse(readFileSync(file, "utf8").trim().split("\n").at(-1)).id;
const typesOf = (data) => Object.fromEntries(Object.entries(data ?? {}).map(([k, v]) => [k, v === null ? "null" : Array.isArray(v) ? "array" : typeof v]));

test("the engine pin holds for the installed package", () => {
  assert.deepEqual(checkEnginePin(REPO).version, PI_VERSION);
});

test("pinned Pi, sealed and without credentials, answers the controller's commands with the shapes the fake models", async () => {
  const entries = [thinkingEntry(), userEntry("a1b2c3d4", "f0e1d2c3", "hello"), assistantEntry("b2c3d4e5", "a1b2c3d4", "hi")];
  const realFile = session("real.jsonl", entries);
  const fakeFile = session("fake.jsonl", entries);
  const before = readFileSync(realFile, "utf8");
  const real = await realPi("real", realFile);
  const fake = await fakePi(fakeFile);
  exchanges.sealed = real;

  assert.equal(readFileSync(realFile, "utf8"), before, "a session that carries a thinking entry is not appended to at start");
  assert.deepEqual(real.agentFiles.filter((f) => f !== "auth.json" && f !== "models-store.json"), [], `agent dir: ${real.agentFiles}`);
  if (real.agentFiles.includes("auth.json")) assert.deepEqual(JSON.parse(readFileSync(join(scratch, "agent-real", "auth.json"), "utf8")), {}, "no credential was written");

  const st = real.responses.get_state;
  assert.equal(st.success, true);
  assert.equal(st.data.sessionFile, realFile);
  assert.equal(st.data.isStreaming, false);
  assert.equal(st.data.sessionId, fake.responses.get_state.data.sessionId);
  // K8 reads get_tree's leafId; for this session it is the file's last entry.
  assert.equal(real.responses.get_tree.data.leafId, lastId(realFile));
  assert.equal(real.responses.get_tree.data.leafId, fake.responses.get_tree.data.leafId);
  // Sealed, Pi still registers one bundled inline extension command, /llama
  // (llama.cpp router). No file extension, template or skill loads. Any slash
  // text is refused at admission (S1, S2), so it can't be invoked; this pins
  // the set so a change shows here.
  const offered = (real.responses.get_commands.data?.commands ?? []).map((c) => `${c.name}:${c.source}:${c.sourceInfo?.source}`);
  assert.deepEqual(offered, ["llama:extension:inline"]);

  for (const type of ["get_state", "clear_queue", "abort", "get_tree"]) {
    const r = real.responses[type], f = fake.responses[type];
    assert.equal(r.command, f.command, type);
    assert.equal(r.success, f.success, type);
    const rt = typesOf(r.data), ft = typesOf(f.data);
    for (const [k, t] of Object.entries(ft)) assert.equal(rt[k], t, `${type}.${k}: the fake models a field pinned Pi doesn't send that way`);
  }
  // clear_queue: one queue_update with both queues empty precedes the response, in both.
  for (const side of [real, fake]) {
    const i = side.lines.findIndex((v) => v.type === "response" && v.id === "smoke-clear_queue");
    const updates = side.lines.slice(0, i).filter((v) => v.type === "queue_update");
    assert.deepEqual(updates.at(-1), { type: "queue_update", steering: [], followUp: [] });
  }
});

test("sealed, pinned Pi ignores a trusted project's .pi resources; --approve past the seal would load them, and checkSeal refuses it (Filbert F2 on #1522)", async () => {
  const skill = join(scratch, "trusted", ".pi", "skills", "probe");
  mkdirSync(skill, { recursive: true });
  writeFileSync(join(skill, "SKILL.md"), "---\nname: probe\ndescription: A project skill that only a trusted load offers.\n---\nprobe\n");
  const entries = [thinkingEntry(), userEntry("a1b2c3d4", "f0e1d2c3", "hello"), assistantEntry("b2c3d4e5", "a1b2c3d4", "hi")];
  const offered = (side) => (side.responses.get_commands.data?.commands ?? []).map((c) => c.name);
  const sealed = await realPi("trust-sealed", session("trust-sealed.jsonl", entries, "trusted"), { project: "trusted", trusted: "trusted" });
  assert.deepEqual(offered(sealed), ["llama"], "the project skill is not loaded under the seal");
  // The control: the same trusted project, with --approve after the seal (Pi
  // keeps the last of --approve and --no-approve), loads the skill, so the
  // assertion above can see a load.
  const approved = await realPi("trust-approved", session("trust-approved.jsonl", entries, "trusted"), { project: "trusted", trusted: "trusted", after: ["--approve"] });
  assert.ok(offered(approved).includes("skill:probe"), `with --approve: ${offered(approved)}`);
  for (const flag of ["--approve", "-a"]) {
    assert.throws(() => checkSeal([...buildPiArgs({ sessionFile: "/s.jsonl" }), flag]), /not one of/, flag);
  }
});

test("pinned Pi appends thinking_level_change at start when the branch lacks one, so the leaf moves (K8 then fails closed)", async () => {
  const entries = [userEntry("a1b2c3d4", null, "hello"), assistantEntry("b2c3d4e5", "a1b2c3d4", "hi")];
  const realFile = session("bare-real.jsonl", entries);
  const fakeFile = session("bare-fake.jsonl", entries);
  const real = await realPi("bare", realFile);
  const fake = await fakePi(fakeFile);
  exchanges.startupAppend = real;
  for (const [file, side] of [[realFile, real], [fakeFile, fake]]) {
    const added = readFileSync(file, "utf8").trim().split("\n").map((l) => JSON.parse(l)).slice(1 + entries.length);
    assert.deepEqual(added.map((e) => e.type), ["thinking_level_change"], file);
    assert.equal(added[0].parentId, "b2c3d4e5");
    assert.notEqual(side.responses.get_tree.data.leafId, "b2c3d4e5", "the leaf moved off the loaded leaf");
    assert.equal(side.responses.get_tree.data.leafId, added[0].id);
  }
  if (process.env.CHAT03_SMOKE_OUT) writeFileSync(process.env.CHAT03_SMOKE_OUT, JSON.stringify({ pi: PI_VERSION, commands: COMMANDS, exchanges }, null, 2) + "\n");
});