From a5beb6d97d7e64c8bb842f1ca7f968c8483855ac Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Sat, 26 Sep 2026 16:36:20 -0500 Subject: [PATCH] feat(conversation): CHAT-02 read-only Pi history reader and two board routes (#1507) packages/conversation is a library with no server: safe-fs, the Pi session parser, CHAT-01 pages, pinned snapshots, cursors and follow. The control board adds GET /api/conversations and /api/conversation behind the Host and Origin guard. Both are read-only, their queries are validated, and each refusal code maps to a status. Dewey authored it (packet 0cf177b1, revision 2). Filbert reviewed the code: R1 revise (branch ids moving on append, the assumed-link bridge merging branches, one unreadable seat directory turning the catalogue into a 500), then R2 approve (3b14d66c). Darkwing reviewed the routes: R1 approve (07b10ad1), R2 approve (b9d92003). The package lands with the routes, because serve.mjs imports the reader at load. On an index export: the eight suites 24/90/43/17/14/15/63/18, conversation and control-board 153/153, webui 9/9. Co-Authored-By: Claude Opus 5.5 --- .../work/chat-02-routes-review-2026-09-26.md | 54 ++ .../chat-02-routes-review-r2-2026-09-26.md | 69 ++ .../dewey/work/chat-02/BACKEND-r1-286c3ad5.md | 218 +++++ agents/dewey/work/chat-02/BACKEND.md | 340 ++++++++ .../chat-02/evidence/backend/http-out.txt | 3 + .../chat-02/evidence/backend/http-real.mjs | 20 + .../evidence/backend/mutate-backend.txt | 39 + .../chat-02/evidence/backend/mutate-board.py | 30 + .../evidence/backend/mutate-routes.txt | 11 + .../work/chat-02/evidence/backend/mutate.py | 59 ++ .../chat-02/evidence/backend/smoke-stats.txt | 2 + .../work/chat-02/evidence/backend/smoke.mjs | 37 + .../work/chat-02-backend-review-2026-09-26.md | 206 +++++ packages/control-board/src/serve.mjs | 87 +- packages/control-board/tests/serve.test.mjs | 149 +++- packages/conversation/README.md | 207 +++++ packages/conversation/package.json | 11 + packages/conversation/src/parts.mjs | 124 +++ packages/conversation/src/pi.mjs | 287 +++++++ packages/conversation/src/reader.mjs | 401 +++++++++ packages/conversation/src/safe-fs.mjs | 123 +++ packages/conversation/tests/reader.test.mjs | 773 ++++++++++++++++++ 22 files changed, 3247 insertions(+), 3 deletions(-) create mode 100644 agents/darkwing/work/chat-02-routes-review-2026-09-26.md create mode 100644 agents/darkwing/work/chat-02-routes-review-r2-2026-09-26.md create mode 100644 agents/dewey/work/chat-02/BACKEND-r1-286c3ad5.md create mode 100644 agents/dewey/work/chat-02/BACKEND.md create mode 100644 agents/dewey/work/chat-02/evidence/backend/http-out.txt create mode 100644 agents/dewey/work/chat-02/evidence/backend/http-real.mjs create mode 100644 agents/dewey/work/chat-02/evidence/backend/mutate-backend.txt create mode 100644 agents/dewey/work/chat-02/evidence/backend/mutate-board.py create mode 100644 agents/dewey/work/chat-02/evidence/backend/mutate-routes.txt create mode 100644 agents/dewey/work/chat-02/evidence/backend/mutate.py create mode 100644 agents/dewey/work/chat-02/evidence/backend/smoke-stats.txt create mode 100644 agents/dewey/work/chat-02/evidence/backend/smoke.mjs create mode 100644 agents/filbert/work/chat-02-backend-review-2026-09-26.md create mode 100644 packages/conversation/README.md create mode 100644 packages/conversation/package.json create mode 100644 packages/conversation/src/parts.mjs create mode 100644 packages/conversation/src/pi.mjs create mode 100644 packages/conversation/src/reader.mjs create mode 100644 packages/conversation/src/safe-fs.mjs create mode 100644 packages/conversation/tests/reader.test.mjs diff --git a/agents/darkwing/work/chat-02-routes-review-2026-09-26.md b/agents/darkwing/work/chat-02-routes-review-2026-09-26.md new file mode 100644 index 00000000..56990d37 --- /dev/null +++ b/agents/darkwing/work/chat-02-routes-review-2026-09-26.md @@ -0,0 +1,54 @@ +# CHAT-02 board routes: Darkwing's review (#1507) + +Reviewer: Darkwing, 2026-09-26, per Sage's D3. Requested by Dewey. Scope: the +two read-only routes only. Filbert reviews `packages/conversation` in full. + +Candidate, base 34777c56, uncommitted. I verified both hashes: +- `packages/control-board/src/serve.mjs` afc95bdb…c540d +- `packages/control-board/tests/serve.test.mjs` e60aa14b…ecbc + +**Verdict: approve**, with one commit condition and two nonblocking notes. + +## What I checked + +- Order. `foreignRequest` runs first on every request, then the POST routes, + then the GET/HEAD check (405 otherwise), then these routes. A POST to + either path is 405, and a foreign Host or Origin is 403 before any read. +- Query validation. `/api/conversations` refuses any parameter. + `/api/conversation` accepts only `id`, `branch` and `cursor`, one value + each, each matching `QUERY_VALUE`. That is the same pattern as + `parts.mjs` `ID`, so every id the reader issues (`safeId`, `root`, `c-` + cursors, `pi-` conversations) passes. No path comes from the request. +- Responses. JSON with `no-store` and `nosniff`, no CORS headers. A thrown + error gives a fixed 500 body and logs to stderr only. +- Refusal bodies. Every `Refusal` message in `packages/conversation/src` is + a fixed string. The one interpolated message (`denied`, safe-fs.mjs:34) + interpolates only "session root" or "session file". No path or content + reaches the client through `error`. +- Status map. It covers every code the route can reach. `unknown-actor` and + `unsupported-purpose` are absent, and the route can't produce them because + it always passes the default actor and purpose. +- Tests: `serve.test.mjs` plus `packages/conversation/tests/`, 61/61 on the + pinned files. + +## Commit condition + +`serve.mjs` imports `../../conversation/src/reader.mjs` at module load, and +`packages/conversation/` is untracked. Committing the routes without that +package breaks the board's start, not only these routes. The package must +land in the same commit or an earlier one, after Filbert's review. + +## Notes (nonblocking) + +1. **A cursor needs its branch.** The header comment says `branch` and + `cursor` are optional. But `next()` compares `branch !== record.branch`, + and every cursor record carries a string branch (`safeId` or `root`). So + `?id=X&cursor=C` without `branch` is always 409 `cursor-foreign`, with + `reconcile: true`. That is safe, but a client that follows `nextCursor` + alone gets a refusal that reads like a stale view. The test passes the + page's branch, so it doesn't show this. Either say in the comment that a + cursor call must repeat `page.branch`, or answer 400 "cursor requires + branch". I'd take the comment now and let CHAT-03's client decide. +2. **New codes fall to 422.** A code the reader adds later maps to 422 + without a test failing. A test that runs the reader's refusal codes + through `REFUSAL_STATUS` would catch that. That's optional. diff --git a/agents/darkwing/work/chat-02-routes-review-r2-2026-09-26.md b/agents/darkwing/work/chat-02-routes-review-r2-2026-09-26.md new file mode 100644 index 00000000..e6694603 --- /dev/null +++ b/agents/darkwing/work/chat-02-routes-review-r2-2026-09-26.md @@ -0,0 +1,69 @@ +# CHAT-02 board routes, revision 2: Darkwing's review (#1507) + +Reviewer: Darkwing, 2026-09-26, at Sage's request. Scope: the route delta +since my R1 approval (`chat-02-routes-review-2026-09-26.md`, 07b10ad1). Filbert +reviewed the backend (packet `agents/dewey/work/chat-02/BACKEND.md`, 0cf177b1). + +Candidate, base 34777c56, uncommitted. I verified both hashes: +- `packages/control-board/src/serve.mjs` d62720dc…a2f3 +- `packages/control-board/tests/serve.test.mjs` d38aa2b2…3f4a + +**Verdict: approve.** The R1 commit condition still holds, and I have two +new nonblocking notes. + +## What I checked + +I kept no copy of the R1 files, so I read the whole route change against the +base (`git diff 34777c56 -- packages/control-board`) instead of only the +delta. It covers every item the packet's §0 lists and nothing else in the +route path. + +- **Cursor needs its branch.** This was my R1 note 1. `conversationQuery` now + answers 400 "a cursor call repeats the page's branch" when `cursor` comes + without `branch`. The check runs after the per-key validation, so a + malformed value still gets its own 400 first. The header comment says the + same. A test covers it, and removing the line fails it. +- **Status map.** My R1 note 2. `REFUSAL_STATUS` is exported and now has 16 + entries. I listed every `new Refusal(""` in + `packages/conversation/src` myself and got 15 codes plus + `unsupported-harness`, which reader.mjs:352 raises by value. That matches the + map exactly. `parts.mjs` raises none. `unavailable`, which safe-fs.mjs:58 + raises when a session root doesn't exist, is 404. That fits the rest of the + map, where not-found is 404. `unknown-actor` 403 and `unsupported-purpose` 422 + are explicit now. +- **Order and guards** are unchanged from R1. The foreign Host or Origin check + comes first, then the POST routes, then 405, then these routes. No path comes + from the request, and responses carry `no-store` and `nosniff` with no CORS + headers. +- **Tests.** `serve.test.mjs` plus `packages/conversation/tests/` pass + 67/67 on the pinned files. +- **Mutations** on a scratch clone of HEAD with the conversation package and + the two pinned files: + + | Mutation | Result | + |---|---| + | cursor-without-branch check removed | 1 fails | + | `unknown-actor` entry dropped | 1 fails (the scan test) | + | `nosniff` removed | 1 fails | + | repeated-parameter check removed | 1 fails | + | catalogue parameter check removed | 1 fails | + | `unavailable` changed from 404 to 422 | nothing fails | + + The last row is note 1 below. + +## Commit condition (unchanged) + +`serve.mjs` imports `../../conversation/src/reader.mjs` at module load, and +`packages/conversation/` is still untracked. The package must land in the +same commit as the routes or an earlier one. Otherwise the board fails to +start. + +## Notes (nonblocking) + +1. **The scan test checks keys, not values.** It proves every code has an + entry. No test proves `unavailable` is 404. If someone edits that value, or + any status no route test exercises, nothing fails. A table test that + asserts the whole `REFUSAL_STATUS` object would pin them. That's optional. +2. **The scan reads a fixed list of three files.** If a refusal is added to + `parts.mjs` or a new file, the scan won't see it, and that code falls to 422. + Reading every `.mjs` in `packages/conversation/src` would close the gap. diff --git a/agents/dewey/work/chat-02/BACKEND-r1-286c3ad5.md b/agents/dewey/work/chat-02/BACKEND-r1-286c3ad5.md new file mode 100644 index 00000000..45bfc265 --- /dev/null +++ b/agents/dewey/work/chat-02/BACKEND-r1-286c3ad5.md @@ -0,0 +1,218 @@ +# CHAT-02 backend: review packet (#1507, row 5) + +Author: Dewey, 2026-09-26. Brief: `BRIEF.md` R4 (`636b0fac…`), §2.1 and D3. +Base: `34777c56` on `refactor`. Nothing here is committed; the candidate is +the working tree, pinned by the hashes below. + +Reviewers (Sage's order): +- Filbert: the code, all nine files. +- Darkwing: the two board routes, `serve.mjs` and `serve.test.mjs`. + +The Console (`packages/webui`) is the next step and is not in this packet. + +## 1. Candidate hashes + +``` +f9008c01c608ea9aacdd15459f03a4ca8f8b4fe4d5225f3f8337bc5f9282f955 packages/conversation/package.json +6f2cab5a31a26f71cf1cc00d6aed2a683e011b6cc2b52ebd4bb50b404af96d5c packages/conversation/README.md +da336ed2a2a149ff56ac12af2440a8353d4573c33e9069381be60f1a0ca70151 packages/conversation/src/parts.mjs +28e22501ab133af7dee1a38f0dc209d005ead4192b64dae6ef999ec4191db8e0 packages/conversation/src/pi.mjs +1e1db046c65292dbcfa2d59b6f0d5009dc48062cbd6a7ec123a18b6707ec8142 packages/conversation/src/reader.mjs +98835b171c1473de2ba242347b191773dfac1e50f383e43ec43b2ef4a5122a3a packages/conversation/src/safe-fs.mjs +3f89f5e7a3cdd5bb31623c799105c538fd5b5eacae368af0170e43f1130267e6 packages/conversation/tests/reader.test.mjs +afc95bdb850e4b533516984d2dd7f3ff852c90b46b384f42bc550adf9f2c540d packages/control-board/src/serve.mjs +e60aa14b77350a2807e9e9b887acb9daec1c0468d94baaf9f77548dc54fcecbc packages/control-board/tests/serve.test.mjs +``` + +The first seven files are new. `serve.mjs` and `serve.test.mjs` are diffs +against `34777c56`: `git diff 34777c56 -- packages/control-board`. +`scan.mjs` is unchanged. D3 allowed edits there, but none were needed. + +## 2. What it does + +`packages/conversation/README.md` is the reference: API, sources, safe open, +parser rules, page limits, snapshot, epoch and cursor rules, follow, the +refusal table and costs. In short: + +- `rootsFromSpecs` turns the board's repository specs into roots, using + registrations only as hints. `createReader` serves `catalogue`, `open` and + `next`. +- Pages and cursors are CHAT-01 `page` and `cursor` records. Everything the + Console needs beyond CHAT-01 goes in `view`. +- The board serves `GET /api/conversations` (no parameters) and + `GET /api/conversation?id=&branch=&cursor=`. Both run after the existing + Host/Origin guard (`d1629d61`) and its GET/HEAD check. +- Responses are `application/json` with `no-store` and `nosniff`, and no CORS + headers. +- Status map: + - 404 for an unknown conversation or branch; + - 409 for cursor refusals, `source-replaced` and `incomplete-header`; + - 403 for `unsafe-path`, `foreign-project` and `unreadable`; + - 422 for the rest; + - 400 for any query parameter other than `id`, `branch` or `cursor`, a + repeated one, or a value that fails the id pattern; + - 500 with a fixed message for an exception, with details only on stderr. + +## 3. Choices and deviations to review + +1. **Catalogue rows are a summary shape, not CHAT-01 `catalogueItem`.** The + row is: conversation, seat, project, harness, history, title, readOnly, + `controlMode: "unavailable"`, the three separate time fields, + availability, `unsupportedReason` and refusal. `catalogueItem` carries + binding and control fields that belong to CHAT-03. I did not want to fill + them with placeholders that look authoritative. +2. **Registrations match on `sessionsDir`, not `sessionFile`.** Registrations + have no `sessionFile` field. The brief §2.1 rule is applied to the + directory instead: seat, layout `repo`, project and `samePath(sessionsDir)` + must all match. A registration never adds a root or names a file (F9). +3. **Placeholder rows for non-Pi seats.** A seat on another harness has no Pi + directory, so the board has no spec for it. A registration adds one + `unsupported-harness` row when its `sessionsDir` is the standard + directory under a project root that is already approved. Its directory is + never read. Rocko (`claude-code`) is the live case (F15). +4. **An epoch id is comparable only within one cursor chain.** + `sourceEpoch = "e-" + sha256(dev:ino:digest)`, taken at open and carried + forward while the prefix verifies. Two opens of an unchanged file give the + same value. After growth, a new open gets a new value for the same epoch, + while the old chain keeps its own. Only a new inode, a shorter file or a + changed prefix is a new epoch, and that is refused. +5. **Follow semantics.** On the last page, `follow` replaces `cursor`. + - `next` with it re-verifies the pinned prefix and takes a fresh snapshot. + - If the view's leaf is still on the new default branch, the ids of the + parts already served must be unchanged. It then returns only the new + parts. + - If the conversation went on down another branch, it returns an empty + page on the old branch and reports the new default in `view`. + - Silently switching branches would break "nothing switches silently". +6. **Missing parent next to malformed lines is bridged.** The history + continues from the previous valid entry, with an "assumed link" notice. + Pi itself would drop the history before that point. Without adjacent + malformed lines, the history stops with a notice. A follow refuses if a + line that was malformed now reads differently, which changes meaning + (F1 follow test). +7. **`unreadable` refusal.** `EACCES` or `EPERM` on a root listing or file + open becomes a per-row 403, so one unreadable file cannot fail the whole + catalogue. +8. **256 MiB file cap** (`too-large`, 422). The largest real file today is + 18.6 MB. +9. **Every page re-hashes the whole pinned prefix.** This makes detection + simple and total, at a cost measured in §4.3. +10. **Refusal code names** not already in CHAT-01 are CHAT-02 values, like + `unsupported-harness` (D2): `unsafe-path`, `foreign-project`, + `unreadable`, `not-a-pi-session`, `incomplete-header`, `too-large`, + `unknown-branch`, `unknown-actor` and `unsupported-purpose`. + +## 4. Evidence + +All runs are on the candidate hashes above. + +### 4.1 Suites and contract checks + +- `node --test --test-concurrency=1 packages/conversation/tests/ + packages/control-board/tests/ packages/webui/tests/ packages/seat/tests/`: + 175 tests, 175 pass. +- `node docs/plans/chat-00/check.mjs`, `chat-01/check.mjs` and + `chat-01c/check.mjs` all exit 0. + +### 4.2 Fixture map (brief §2.1) + +| # | Test | +|---|---| +| F1 | three tests in `reader.test.mjs`: malformed notice in place (with a hostile id), bridge and missing parent, and the follow that refuses when history changes meaning | +| F2–F15 | one named test each in `reader.test.mjs` | +| F16 | `serve.test.mjs` §12: foreign Host and cross-origin Origin on both routes give 403; a spy reader records zero calls, and no `access-control-*` header is sent | +| F17 | every reader call in `reader.test.mjs` runs inside a fingerprint of the whole fixture tree: size, SHA-256, mtime (ns), (dev, ino), mode and every directory listing. The route test in `serve.test.mjs` does the same. Files no read may touch are mode 000. | + +Additional coverage: mapping of every Pi entry type and role, pagination +at 100 parts and at the byte cap, surrogate-safe fragments, unknown and +empty and non-Pi files, `unreadable`, and CHAT-01 schema validation of +every page and cursor the tests produce (Python `jsonschema`). + +### 4.3 Real data (read-only, this checkout) + +- Catalogue: 19 rows. There are 18 available Pi conversations across darkwing, + dewey, filbert, researcher and sage, plus rocko as `unsupported-harness`. +- Reading every page of every conversation gave 102 pages and 8693 entries, + with no refusals. The slowest conversation took 662 ms; the largest file is + 18.6 MB. +- 186 pages and cursors sampled from that run: 0 schema-invalid. +- Over HTTP, through a temporary board on port 0 with a temporary `boardDir` + (the live board was not touched): 19 rows and 102 pages in 1517 ms. Rocko + returns 422 `unsupported-harness`. +- Scripts and output: `evidence/backend/smoke.mjs`, `smoke-stats.txt`, + `http-real.mjs`, `http-out.txt`. + +### 4.4 Mutation testing + +Mutations were run in scratch copies under `/tmp/dewey-chat02/`, never in +the served tree. Scripts and full output: `evidence/backend/mutate.py`, +`mutate-board.py`, `mutate-backend.txt` and `mutate-routes.txt`. + +Reader: 27 of 28 caught. + +| Mutant | Caught by | +|---|---| +| no O_NOFOLLOW and no lstat symlink check | F7, F8 | +| directory components unchecked | F7 | +| listing follows symlinks | F7, F8 | +| no prefix digest check | F4 | +| no dev/ino check | F3 | +| no shorter check | **not caught, equivalent** (below) | +| cwd always in project | F10 | +| no character cap | F14, schema | +| no page byte cap | F14 | +| no block cap | F14, schema | +| no foreign check | F6 | +| no branch binding | F6 | +| no expiry | F6 | +| no parentSession notice | F11 | +| branch parameter ignored | F12 | +| registration harness ignored | F15 | +| no placeholder roots | F15 | +| malformed notices dropped | F1 | +| no bridge | F1 | +| `incomplete` never set | F2, F5 | +| `next` re-reads fresh instead of pinned | F2, F5, F12, unknown/empty | +| follow skips history check | F1 follow | +| follow ignores branching | F12 | +| catalogue writes a file | F17 in six fixtures | +| wrong thinking visibility | mapping | +| raw tool ids | mapping, schema | +| id namespaces dropped | F1 | +| surrogate pair split | fragments | + +The equivalent mutant: removing the "file is shorter" check changes nothing +observable. A shorter file cannot supply the pinned length, so the prefix +digest differs and `source-replaced` still follows. The check stays because +it gives the refusal without hashing a truncated read. + +Board routes: 9 of 9 caught. The mutants were: routes before the guard, no +`nosniff`, every refusal 422, no query validation, unknown parameters +allowed, `reconcile` dropped, cursor ignored, registrations ignored, and a +CORS header sent. + +## 5. Known limits + +- **No `openat` in Node.** A directory component swapped between the checks + and the open is caught by the re-check after the open, not prevented. + Nothing is read from a descriptor whose (dev, ino) does not match. +- **The `fstat` mismatch branch of F8** is covered by design, not by a + test. The F8 test swaps in a symlink, which `O_NOFOLLOW` refuses first. A + plain-file swap between `lstat` and `open` needs a race the suite cannot + schedule. +- **An in-place rewrite with the same inode** is detected by the digest of + the pinned prefix, as the brief requires. Growth after the pin is not a + change. +- **Cost.** Each page reads and hashes the whole pinned prefix; §4.3 has the + cost on real files. +- **One actor.** Cursors live in memory and are lost when the board restarts; + the refusal is `cursor-unknown` with reconcile. `local-operator` is the + only actor, which is not multi-actor safety (CHAT-04R). +- **Claude history** waits for B1 and CHAT-03 (D2). + +## 6. After review + +- Pushing goes through Sage with Jason's word. The live board needs a restart + after the commit to serve the routes. +- The WebUI proxy allowlist for the two routes is Console work and is not in + this packet. diff --git a/agents/dewey/work/chat-02/BACKEND.md b/agents/dewey/work/chat-02/BACKEND.md new file mode 100644 index 00000000..a1ff3bd3 --- /dev/null +++ b/agents/dewey/work/chat-02/BACKEND.md @@ -0,0 +1,340 @@ +# CHAT-02 backend: review packet, revision 2 (#1507, row 5) + +Author: Dewey, 2026-09-26. Brief: `BRIEF.md` R4 (`636b0fac…`), §2.1 and D3. +Base: `1c5f6bc3` on `refactor`. Nothing here is committed; the candidate is +the working tree, pinned by the hashes below. + +Revision 1 was `286c3ad5`, kept as `BACKEND-r1-286c3ad5.md`. Filbert +reviewed it and asked for a revision +(`agents/filbert/work/chat-02-backend-review-2026-09-26.md`, `27d64e14…`). +Darkwing approved the routes in it +(`agents/darkwing/work/chat-02-routes-review-2026-09-26.md`, `07b10ad1…`) +with two nonblocking notes. §0 answers both reviews. + +Reviewers (Sage's order): +- Filbert: the code, all nine files. He asked to review the delta. +- Darkwing: the two board routes. `serve.mjs` and `serve.test.mjs` changed + for his notes, so the route delta needs his look. + +The Console (`packages/webui`) is the next step and is not in this packet. +The four WebUI browser failures Filbert saw in the shared tree come from my +unpinned Console edits to `app.js` and `index.html`. They are Console work. + +## 0. Revision 2 changes + +### Filbert, required + +1. **Branch ids are stable.** + - Roots are entries whose parent is null or absent, in file order. The + first root's line is `main`. `main` exists even in a file with no + entries, as an empty branch with a null leaf. + - At a fork the earliest child in file order continues its parent's + branch. Each later child starts a branch named `b.`. Each later + root (Pi's `resetLeaf`) starts `b.` too. + - Growth never renames a branch. `page.branch` always equals the cursor's + branch, and `open({branch})` with a name taken before growth still + works. + - The default branch is the one ending at Pi's default leaf, the last + entry in file order. `view.defaultBranch` names it; `view.branches` + flags it with `isDefault`. + - Follow rebuilds the cursor's own branch from a fresh snapshot. It + refuses `source-replaced` with reconcile if the branch is gone, is + shorter than the parts served, or the ids of those parts differ. + - Tests: F12 "two leaves" (branches `main` and `b.o4`, one branch value + across follows, the pre-growth name still opens, the `main` view picks + up its growth), F12 "a second root", F12 "a follow refuses when an + appended duplicate id changes the branch's earlier parts", F1 "a follow + stays on its branch". + - Your `branch.mjs`: open and follow both give `main`, every entry is + labelled `main`, and reopening with the first branch id works. +2. **The bridge is gone.** + - A missing parent stops the history with a `missing-parent` notice. When + malformed lines sit just before the entry, the notice names them: "Line + N could not be read and may have held it." Those lines are not repeated + as separate notices. + - The history before the gap is its own leaf branch and reads normally. + - Tests: F1 "a missing parent stops the history with a notice that names + the unreadable lines", F1 "an unreadable fork is never merged into + another branch's history" (your case). + - Your `bridge.mjs`: `b.y` shows only the notice and `leaf`. +3. **`EACCES` and `EPERM` on a directory component become `unreadable`.** + - `lstatOrNull` maps them through `denied()`. The root is refused on its + own row; the catalogue and opens under other roots are unaffected. + - Test: "a seat directory without search permission refuses that root, + not the catalogue", with the bad root first and then last. + - Your `perm.mjs`: the catalogue returns the good root's row and lists + `bad` in `refusedRoots` as `unreadable`. The open of an unknown id walks + both roots and returns an `unknown-conversation` refusal, not a throw. + +### Filbert, small + +- Redacted thinking (`redacted: true`) is `unavailable` with empty text, + whatever the `thinking` field says. Test: the mapping test. +- A relative header `cwd` is refused as `foreign-project`. Test: F10 + `relative.jsonl`. +- Malformed notices appear when a file has no valid entry. Each malformed + line keeps its own notice at its file position, on every branch, including + after the leaf. Test: F1 "a file whose entries are all unreadable". + +### Darkwing, nonblocking + +1. A cursor call without `branch` is now 400 ("a cursor call repeats the + page's branch"), and the header comment says so. +2. `REFUSAL_STATUS` is exported. A new test scans the reader's three source + files for every `new Refusal(""` plus `UNSUPPORTED_HARNESS` and + fails on a missing or stale entry. Its first run found `unavailable` + falling to 422; it is now 404. `unknown-actor` (403) and + `unsupported-purpose` (422) are now explicit. + +Route delta against revision 1: `git diff 34777c56 -- packages/control-board` +shows the whole change; against Darkwing's pins it is the header comment, +`export` on `REFUSAL_STATUS` with three new entries, one line in +`conversationQuery`, the new test and branch names in expectations +(`main`, `b.e5`, `b.e1`). + +## 1. Candidate hashes + +``` +f9008c01c608ea9aacdd15459f03a4ca8f8b4fe4d5225f3f8337bc5f9282f955 packages/conversation/package.json +1ea8c8c093cb726773bfa55c354126cf4b1720affa6d0e9cd0845570cbfb735e packages/conversation/README.md +da336ed2a2a149ff56ac12af2440a8353d4573c33e9069381be60f1a0ca70151 packages/conversation/src/parts.mjs +20e781240846298fa65f9a8226b5e1ee1e81338b81229334666d845c17c99ac1 packages/conversation/src/pi.mjs +72c3255bca7a894f6484b3224aabf054d40a0db78cbe46a6c492ed385333b8ad packages/conversation/src/reader.mjs +10a1ff9e91c1f1e4684fc38c5bca83c50d79bb1f459d269517396545a269203a packages/conversation/src/safe-fs.mjs +78b7719b0c183cc1d9fedf009fb8a5e4dae34a30d1b39f931824016cc07fa86b packages/conversation/tests/reader.test.mjs +d62720dcf1bd43ce9412356a04b5248a85aec59dd78194887e3c38010c0aa2f3 packages/control-board/src/serve.mjs +d38aa2b209e0fb22ab3c52c3cb26a8f05e9e269e567d40b7e6ad88430c593f4a packages/control-board/tests/serve.test.mjs +``` + +Unchanged from revision 1: `package.json` and `parts.mjs`. The first seven +files are new. `serve.mjs` and `serve.test.mjs` are diffs against the base. +`scan.mjs` is unchanged. D3 allowed edits there, but none were needed. + +## 2. What it does + +`packages/conversation/README.md` is the reference: API, sources, safe open, +parser rules, branch names, page limits, snapshot, epoch and cursor rules, +follow, the refusal table and costs. In short: + +- `rootsFromSpecs` turns the board's repository specs into roots, using + registrations only as hints. `createReader` serves `catalogue`, `open` and + `next`. +- Pages and cursors are CHAT-01 `page` and `cursor` records. Everything the + Console needs beyond CHAT-01 goes in `view`. +- The board serves `GET /api/conversations` (no parameters) and + `GET /api/conversation?id=&branch=&cursor=`. Both run after the existing + Host/Origin guard (`d1629d61`) and its GET/HEAD check. +- Responses are `application/json` with `no-store` and `nosniff`, and no CORS + headers. +- Status map: + - 404 for an unknown conversation or branch, and `unavailable`; + - 409 for cursor refusals, `source-replaced` and `incomplete-header`; + - 403 for `unsafe-path`, `foreign-project`, `unreadable` and + `unknown-actor`; + - 422 for the rest, each listed in `REFUSAL_STATUS`; + - 400 for any query parameter other than `id`, `branch` or `cursor`, a + repeated one, a value that fails the id pattern, or a cursor without a + branch; + - 500 with a fixed message for an exception, with details only on stderr. + +## 3. Choices and deviations to review + +1. **Catalogue rows are a summary shape, not CHAT-01 `catalogueItem`.** The + row is: conversation, seat, project, harness, history, title, readOnly, + `controlMode: "unavailable"`, the three separate time fields, + availability, `unsupportedReason` and refusal. `catalogueItem` carries + binding and control fields that belong to CHAT-03. I did not want to fill + them with placeholders that look authoritative. +2. **Registrations match on `sessionsDir`, not `sessionFile`.** Registrations + have no `sessionFile` field. The brief §2.1 rule is applied to the + directory instead: seat, layout `repo`, project and `samePath(sessionsDir)` + must all match. A registration never adds a root or names a file (F9). +3. **Placeholder rows for non-Pi seats.** A seat on another harness has no Pi + directory, so the board has no spec for it. A registration adds one + `unsupported-harness` row when its `sessionsDir` is the standard + directory under a project root that is already approved. Its directory is + never read. Rocko (`claude-code`) is the live case (F15). +4. **An epoch id is comparable only within one cursor chain.** + `sourceEpoch = "e-" + sha256(dev:ino:digest)`, taken at open and carried + forward while the prefix verifies. Two opens of an unchanged file give the + same value. After growth, a new open gets a new value for the same epoch, + while the old chain keeps its own. Only a new inode, a shorter file or a + changed prefix is a new epoch, and that is refused. +5. **Follow stays on its branch.** On the last page, `follow` replaces + `cursor`. + - `next` with it re-verifies the pinned prefix, takes a fresh snapshot and + rebuilds the cursor's branch by name. + - It returns the parts after the ones already served. The branch must + still exist and the ids of the served parts must be unchanged; + otherwise `source-replaced` with reconcile. + - If Pi's default moved to another branch, the page stays on the old + branch and `view.defaultBranch` names the new one. The Console decides + what to show; nothing switches silently. +6. **A missing parent stops the history.** There is no bridge. The notice + names any malformed lines just before the entry, since one of them may + have held the parent. The history before the gap is its own branch. A + malformed line keeps its notice at its file position on every branch, so + a branch's served prefix does not change as the file grows. +7. **`unreadable` refusal.** `EACCES` or `EPERM` on a root listing, a + directory component or a file open becomes a per-row 403, so one + unreadable path cannot fail the whole catalogue. +8. **256 MiB file cap** (`too-large`, 422). The largest real file today is + 18.6 MB. +9. **Every page re-hashes the whole pinned prefix.** This makes detection + simple and total, at a cost measured in §4.3. +10. **Refusal code names** not already in CHAT-01 are CHAT-02 values, like + `unsupported-harness` (D2): `unsafe-path`, `foreign-project`, + `unreadable`, `not-a-pi-session`, `incomplete-header`, `too-large`, + `unknown-branch`, `unavailable`, `unknown-actor` and + `unsupported-purpose`. + +## 4. Evidence + +All runs are on the candidate hashes above. + +### 4.1 Suites and contract checks + +- In `/tmp/dewey-chat02/overlay`, a `git archive` of `1c5f6bc3` with only + the nine files overlaid: `node --test --test-concurrency=1` over the + conversation, control-board, webui and seat suites gives 181 tests, 181 + pass (29, 124, 9 and 19). +- `node docs/plans/chat-00/check.mjs`, `chat-01/check.mjs` and + `chat-01c/check.mjs` all exit 0. + +### 4.2 Fixture map (brief §2.1) + +| # | Test | +|---|---| +| F1 | five tests in `reader.test.mjs`: malformed notice in place (with a hostile id); missing parent names the unreadable lines; an unreadable fork is never merged; a follow stays on its branch; an all-malformed file shows a notice per line | +| F12 | three tests: two leaves with stable names across growth; a follow that refuses when an appended duplicate id changes served parts; a second root | +| F2–F11, F13–F15 | one named test each in `reader.test.mjs` | +| F16 | `serve.test.mjs` §12: foreign Host and cross-origin Origin on both routes give 403; a spy reader records zero calls, and no `access-control-*` header is sent | +| F17 | every reader call in `reader.test.mjs` runs inside a fingerprint of the whole fixture tree: size, SHA-256, mtime (ns), (dev, ino), mode and every directory listing. The route test in `serve.test.mjs` does the same. Files no read may touch are mode 000. | + +Additional coverage: mapping of every Pi entry type and role (including +redacted thinking), pagination at 100 parts and at the byte cap, +surrogate-safe fragments, unknown and empty and non-Pi files, an unreadable +file or root, a seat directory at mode 000, every refusal code having an +HTTP status, and CHAT-01 schema validation of every page and cursor the +tests produce (Python `jsonschema`). + +### 4.3 Real data (read-only, this checkout) + +- Catalogue: 19 rows in 75 ms, no refused roots. There are 18 available Pi + conversations across darkwing, dewey, filbert, researcher and sage, plus + rocko as `unsupported-harness`. +- Reading every page of every conversation gave 102 pages and 8693 entries, + with no refusals. Every page is on branch `main`. The slowest conversation + took 798 ms (662 ms in revision 1, on files that have grown since; I did + not isolate the difference). The largest file is 18.6 MB. +- 186 pages and cursors sampled from that run: 0 schema-invalid. +- Over HTTP, through a temporary board on port 0 with a temporary `boardDir` + (the live board was not touched): 19 rows and 102 pages in 1487 ms. Rocko + returns 422 `unsupported-harness`. +- Scripts and output: `evidence/backend/smoke.mjs`, `smoke-stats.txt`, + `http-real.mjs`, `http-out.txt`. + +### 4.4 Mutation testing + +Mutations were run in scratch copies under `/tmp/dewey-chat02/`, never in +the served tree. The route scratch is a full copy of the checkout (without +`.git`, `agents`, `v1` and `skills/aws-*`), so each mutant's failures are +real test failures, not import errors. Scripts and full output: +`evidence/backend/mutate.py`, `mutate-board.py`, `mutate-backend.txt` and +`mutate-routes.txt`. + +Reader: 38 of 39 caught. + +| Mutant | Caught by | +|---|---| +| no O_NOFOLLOW and no lstat symlink check | F7, F8 | +| directory components unchecked | F7 | +| listing follows symlinks | F7, F8 | +| no prefix digest check | F4 | +| no dev/ino check | F3 | +| no shorter check | **not caught, equivalent** (below) | +| cwd always in project | F10 | +| relative cwd accepted | F10 | +| no character cap | F14, schema | +| no page byte cap | F14 | +| no block cap | F14, schema | +| no foreign check | F6 | +| no branch binding | F6 | +| no expiry | F6 | +| no parentSession notice | F11 | +| branch parameter ignored | F12 two leaves, F12 second root, F1 missing parent | +| unknown branch served | F12 two leaves | +| registration harness ignored | F15 | +| no placeholder roots | F15 | +| malformed notices dropped | four F1 tests | +| bridge restored | F1 missing parent, F1 unreadable fork, F1 follow | +| missing-parent lines not named | F1 missing parent, F1 unreadable fork, F1 follow | +| trailing malformed notices only on the default branch | F1 missing parent, F1 follow | +| branch named by its leaf | all three F12 tests, two F1 tests | +| later child continues the branch | F12 two leaves | +| first root not `main` | all three F12 tests, two F1 tests | +| no empty `main` | F1 all malformed, unknown/empty | +| redacted thinking shown | mapping | +| `EACCES` on a component rethrown | seat directory at mode 000 | +| `incomplete` never set | F2, F5 | +| `next` re-reads fresh instead of pinned | F2, F5, F12, F1 follow | +| follow skips the history check | F12 duplicate id | +| follow skips the id digest | F12 duplicate id | +| follow reads the default branch | F12 two leaves, F1 follow | +| catalogue writes a file | F17 in six fixtures | +| wrong thinking visibility | mapping | +| raw tool ids | mapping, schema | +| id namespaces dropped | F1 | +| surrogate pair split | fragments | + +Removed since revision 1: "no bridge" and "follow ignores branching", whose +behaviour no longer exists. + +The equivalent mutant: removing the "file is shorter" check changes nothing +observable. A shorter file cannot supply the pinned length, so the prefix +digest differs and `source-replaced` still follows. The check stays because +it gives the refusal without hashing a truncated read. + +Board routes: 11 of 11 caught. The mutants were: routes before the guard, no +`nosniff`, every refusal 422, no query validation, unknown parameters +allowed, `reconcile` dropped, cursor ignored, registrations ignored, a +cursor without branch served, `unavailable` falling to 422, and a CORS +header sent. + +## 5. Known limits + +- **No `openat` in Node.** A directory component swapped between the checks + and the open is caught by the re-check after the open, not prevented. + Nothing is read from a descriptor whose (dev, ino) does not match. +- **The `fstat` mismatch branch of F8** is covered by design, not by a + test. The F8 test swaps in a symlink, which `O_NOFOLLOW` refuses first. A + plain-file swap between `lstat` and `open` needs a race the suite cannot + schedule. +- **An in-place rewrite with the same inode** is detected by the digest of + the pinned prefix, as the brief requires. Growth after the pin is not a + change. +- **Follow compares entry ids, not content, for the parts already served.** + The pinned byte prefix is still re-hashed, so a rewrite of served bytes is + caught. An appended entry that reuses a served id is caught when it moves + that entry off the branch or puts another id in its place (F12 duplicate + id, both cases). If it keeps the same id in the same position with new + content, follow does not notice: the parts already served keep the old + text, while a fresh open shows the new one. Pi never writes duplicate + ids. +- **Cost.** Each page reads and hashes the whole pinned prefix; §4.3 has the + cost on real files. +- **One actor.** Cursors live in memory and are lost when the board restarts; + the refusal is `cursor-unknown` with reconcile. `local-operator` is the + only actor, which is not multi-actor safety (CHAT-04R). +- **Claude history** waits for B1 and CHAT-03 (D2). + +## 6. After review + +- Commit condition (Darkwing): `serve.mjs` imports + `../../conversation/src/reader.mjs` at load, so `packages/conversation/` + lands in the same commit as the routes or an earlier one, after Filbert's + approval. +- Pushing goes through Sage with Jason's word. The live board needs a restart + after the commit to serve the routes. +- The WebUI proxy allowlist for the two routes is Console work and is not in + this packet. diff --git a/agents/dewey/work/chat-02/evidence/backend/http-out.txt b/agents/dewey/work/chat-02/evidence/backend/http-out.txt new file mode 100644 index 00000000..e9222736 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/http-out.txt @@ -0,0 +1,3 @@ +rows 19 researcher:available dewey:available sage:available darkwing:available filbert:available darkwing:available filbert:available filbert:available filbert:available filbert:available filbert:available darkwing:available filbert:available dewey:available darkwing:available darkwing:available darkwing:available darkwing:available rocko:unsupported +pages 102 ms 1487 +unsupported 422 {"error":"this harness has no history reader yet","refusal":{"code":"unsupported-harness","reconcile":false}} diff --git a/agents/dewey/work/chat-02/evidence/backend/http-real.mjs b/agents/dewey/work/chat-02/evidence/backend/http-real.mjs new file mode 100644 index 00000000..42af4a3a --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/http-real.mjs @@ -0,0 +1,20 @@ +import { startServer } from "/mnt/storage/src/mosaic-stack/packages/control-board/src/serve.mjs"; +import { discoverRepoAgents } from "/mnt/storage/src/mosaic-stack/packages/control-board/src/scan.mjs"; +import { homedir } from "node:os"; +import { mkdtempSync } from "node:fs"; +const specs = discoverRepoAgents("/mnt/storage/src/mosaic-stack"); +const server = await startServer({ host: "127.0.0.1", port: 0, specs, boardDir: mkdtempSync("/tmp/dewey-chat02/board-"), seatsDir: homedir() + "/.mosaic-dev/seats", isAlive: () => true, page: "x" }); +const base = `http://127.0.0.1:${server.address().port}`; +const cat = await (await fetch(base + "/api/conversations")).json(); +console.log("rows", cat.conversations.length, cat.conversations.map(c => `${c.seat}:${c.availability}`).join(" ")); +const big = cat.conversations.filter(c => c.availability === "available"); +let pages = 0, t = Date.now(); +for (const c of big) { + let r = await (await fetch(`${base}/api/conversation?id=${c.conversation}`)).json(); + pages++; + while (r.page?.hasMore) { r = await (await fetch(`${base}/api/conversation?id=${c.conversation}&branch=${r.page.branch}&cursor=${r.page.nextCursor}`)).json(); pages++; if (!r.ok) { console.log("refused", r); break; } } +} +console.log("pages", pages, "ms", Date.now() - t); +const u = cat.conversations.find(c => c.availability === "unsupported"); +const ur = await fetch(`${base}/api/conversation?id=${u.conversation}`); console.log("unsupported", ur.status, JSON.stringify(await ur.json())); +server.close(); diff --git a/agents/dewey/work/chat-02/evidence/backend/mutate-backend.txt b/agents/dewey/work/chat-02/evidence/backend/mutate-backend.txt new file mode 100644 index 00000000..99fe7713 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/mutate-backend.txt @@ -0,0 +1,39 @@ +CAUGHT no O_NOFOLLOW + no lstat symlink -> F7: a symlinked file and a symlinked directory component are refused, ; F8: a file swapped for a symlink after the catalogue is refused +CAUGHT dir components unchecked -> F7: a symlinked file and a symlinked directory component are refused, +CAUGHT listing follows symlinks -> F7: a symlinked file and a symlinked directory component are refused, ; F8: a file swapped for a symlink after the catalogue is refused +CAUGHT no prefix digest check -> F4: a same-inode rewrite of the prefix refuses old cursors with reconc +CAUGHT no dev/ino check -> F3: a replaced file +MISSED no shorter check -> +CAUGHT cwd always in project -> F10: a header cwd naming another project is refused +CAUGHT relative cwd accepted -> F10: a header cwd naming another project is refused +CAUGHT no char cap -> F14: long strings split into fragments and parts, reassemble exactly, ; every page and cursor is a valid CHAT-01 record +CAUGHT no page byte cap -> F14: long strings split into fragments and parts, reassemble exactly, +CAUGHT no block cap -> F14: long strings split into fragments and parts, reassemble exactly, ; every page and cursor is a valid CHAT-01 record +CAUGHT no foreign check -> F6: unknown, foreign and expired cursors refuse and leave the cursor u +CAUGHT no branch binding -> F6: unknown, foreign and expired cursors refuse and leave the cursor u +CAUGHT no expiry -> F6: unknown, foreign and expired cursors refuse and leave the cursor u +CAUGHT no parentSession notice -> F11: parentSession renders with a marker and the parent is never opene +CAUGHT branch param ignored -> F12: a second root; F12: two leaves: the default leaf is shown and the other branch reads ; F1: a missing parent stops the history with a notice that names the un +CAUGHT unknown branch served -> F12: two leaves: the default leaf is shown and the other branch reads +CAUGHT registration harness ignored -> F15: a Claude seat is an unsupported-harness placeholder whose directo +CAUGHT no placeholder roots -> F15: a Claude seat is an unsupported-harness placeholder whose directo +CAUGHT malformed notices dropped -> F1: a file whose entries are all unreadable shows a notice per line; F1: a follow stays on its branch when the next entry's parent is unrea; F1: a malformed line is an unavailable part at its position, and readi; F1: a missing parent stops the history with a notice that names the un +CAUGHT bridge restored -> F1: a follow stays on its branch when the next entry's parent is unrea; F1: a missing parent stops the history with a notice that names the un; F1: an unreadable fork is never merged into another branch's history +CAUGHT missing-parent lines not named -> F1: a follow stays on its branch when the next entry's parent is unrea; F1: a missing parent stops the history with a notice that names the un; F1: an unreadable fork is never merged into another branch's history +CAUGHT trailing malformed only on default -> F1: a follow stays on its branch when the next entry's parent is unrea; F1: a missing parent stops the history with a notice that names the un +CAUGHT branch named by leaf -> F12: a follow refuses when an appended duplicate id changes the branch; F12: a second root; F12: two leaves: the default leaf is shown and the other branch reads ; F1: a follow stays on its branch when the next entry's parent is unrea; F1: a missing parent stops the history with a notice that names +CAUGHT later child continues branch -> F12: two leaves: the default leaf is shown and the other branch reads +CAUGHT first root not main -> F12: a follow refuses when an appended duplicate id changes the branch; F12: a second root; F12: two leaves: the default leaf is shown and the other branch reads ; F1: a follow stays on its branch when the next entry's parent is unrea; F1: a missing parent stops the history with a notice that names +CAUGHT no empty main -> F1: a file whose entries are all unreadable shows a notice per line; unknown conversations, empty files and non-Pi files refuse +CAUGHT redacted thinking shown -> native entries map to blocks: tools, thinking, bash, notices, ids that +CAUGHT EACCES on a component rethrown -> a seat directory without search permission refuses that root, not the +CAUGHT incomplete never set -> F2: a truncated trailing line marks the view incomplete, not an error; F5: growth between pages keeps the epoch and the page stops at the pin +CAUGHT next re-reads fresh -> F12: a second root; F12: two leaves: the default leaf is shown and the other branch reads ; F1: a follow stays on its branch when the next entry's parent is unrea; F2: a truncated trailing line marks the view incomplete, not an error; F5: growth between pages keeps the epoch and the page stops at th +CAUGHT follow skips history check -> F12: a follow refuses when an appended duplicate id changes the branch +CAUGHT follow skips the id digest -> F12: a follow refuses when an appended duplicate id changes the branch +CAUGHT follow reads the default branch -> F12: two leaves: the default leaf is shown and the other branch reads ; F1: a follow stays on its branch when the next entry's parent is unrea +CAUGHT catalogue writes a file -> F10: a header cwd naming another project is refused; F15: a Claude seat is an unsupported-harness placeholder whose directo; F1: a malformed line is an unavailable part at its position, and readi; F7: a symlinked file and a symlinked directory component are refused, ; F8: a file swapped for a symlin +CAUGHT thinking visibility wrong -> native entries map to blocks: tools, thinking, bash, notices, ids that +CAUGHT raw tool ids -> every page and cursor is a valid CHAT-01 record; native entries map to blocks: tools, thinking, bash, notices, ids that +CAUGHT id namespaces dropped -> F1: a malformed line is an unavailable part at its position, and readi +CAUGHT surrogate split -> fragments never cut a surrogate pair and keep an empty string diff --git a/agents/dewey/work/chat-02/evidence/backend/mutate-board.py b/agents/dewey/work/chat-02/evidence/backend/mutate-board.py new file mode 100644 index 00000000..90a21ffc --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/mutate-board.py @@ -0,0 +1,30 @@ +import subprocess, os +S="/tmp/dewey-chat02/scratch-board" +f=os.path.join(S,"packages/control-board/src/serve.mjs") +orig=open(f).read() +M=[ + ("conversation routes before the guard",[(""" const refused = foreignRequest(req);""",""" { const u0 = new URL(req.url, "http://localhost"); if (u0.pathname.startsWith("/api/conversation")) { let o; try { o = conversationResponse(reader, u0); } catch { o = { status: 500, body: {} }; } return sendConversationJson(res, o.status, o.body); } } + const refused = foreignRequest(req);""")]), + ("no nosniff",[(', "x-content-type-options": "nosniff"','')]), + ("all refusals 422",[("status: REFUSAL_STATUS[out.refusal.code] ?? 422","status: 422")]), + ("no query validation",[(" if (values.length !== 1 || !QUERY_VALUE.test(values[0])) return { error: `invalid ${key}` };\n","")]), + ("unknown params allowed",[(""" if (!["id", "branch", "cursor"].includes(key)) return { error: `unknown parameter: ${key}` };\n""","")]), + ("reconcile dropped",[("refusal: { code: out.refusal.code, reconcile: out.refusal.reconcile }","refusal: { code: out.refusal.code }")]), + ("cursor ignored",[(" const out = query.cursor\n"," const out = false\n")]), + ("registrations ignored",[("rootsFromSpecs(specs, loadRegistrations(seatsDir).registrations)","rootsFromSpecs(specs, [])")]), + ("cursor without branch served",[(" if (out.cursor && !out.branch) return"," if (false) return")]), + ("unavailable falls to 422",[(" unavailable: 404,\n","")]), + ("CORS header sent",[('"x-content-type-options": "nosniff" });','"x-content-type-options": "nosniff", "access-control-allow-origin": "*" });')]), +] +for name,reps in M: + src=orig + ok=True + for a,b in reps: + if a not in src: print("NOT APPLIED",name); ok=False; break + src=src.replace(a,b,1) + if not ok: continue + open(f,"w").write(src) + r=subprocess.run(["node","--test","--test-concurrency=1","tests/serve.test.mjs"],cwd=os.path.join(S,"packages/control-board"),capture_output=True,text=True,timeout=600) + fails=sorted(set(l.strip()[2:].split(" (")[0][:60] for l in r.stdout.splitlines() if l.lstrip().startswith("✖") and "failing tests" not in l)) + print(("CAUGHT " if r.returncode else "MISSED ")+name+" -> "+"; ".join(fails)) +open(f,"w").write(orig) diff --git a/agents/dewey/work/chat-02/evidence/backend/mutate-routes.txt b/agents/dewey/work/chat-02/evidence/backend/mutate-routes.txt new file mode 100644 index 00000000..8dd0c2c8 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/mutate-routes.txt @@ -0,0 +1,11 @@ +CAUGHT conversation routes before the guard -> conversation routes +CAUGHT no nosniff -> conversation routes: catalogue, first page, next page and fo +CAUGHT all refusals 422 -> conversation routes: catalogue, first page, next page and fo +CAUGHT no query validation -> conversation routes: catalogue, first page, next page and fo +CAUGHT unknown params allowed -> conversation routes: catalogue, first page, next page and fo +CAUGHT reconcile dropped -> conversation routes: catalogue, first page, next page and fo +CAUGHT cursor ignored -> conversation routes: catalogue, first page, next page and fo +CAUGHT registrations ignored -> conversation routes: catalogue, first page, next page and fo +CAUGHT cursor without branch served -> conversation routes: catalogue, first page, next page and fo +CAUGHT unavailable falls to 422 -> every refusal code the reader can raise has an HTTP status +CAUGHT CORS header sent -> conversation routes: catalogue, first page, next page and fo diff --git a/agents/dewey/work/chat-02/evidence/backend/mutate.py b/agents/dewey/work/chat-02/evidence/backend/mutate.py new file mode 100644 index 00000000..4990c163 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/mutate.py @@ -0,0 +1,59 @@ +import subprocess, sys, shutil, os +S="/tmp/dewey-chat02/scratch/packages/conversation" +M=[ + ("no O_NOFOLLOW + no lstat symlink","src/safe-fs.mjs",[("constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK","constants.O_RDONLY"),(' if (before.isSymbolicLink()) throw new Refusal("unsafe-path", "session file is a symlink");\n',''),(' if (!before.isFile()) throw',' if (false) throw'),("if (!st.isFile() || st.dev !== before.dev","if (st.dev !== before.dev")]), + ("dir components unchecked","src/safe-fs.mjs",[(' if (st.isSymbolicLink()) throw new Refusal("unsafe-path", "session root contains a symlink");\n',''),(' if (!st.isDirectory()) throw',' if (false) throw')]), + ("listing follows symlinks","src/safe-fs.mjs",[(' else if (dirent.isSymbolicLink()) refused.push({ name: dirent.name, code: "unsafe-path" });\n','')]), + ("no prefix digest check","src/reader.mjs",[('if (pinned.digest !== state.digest) throw','if (false) throw')]), + ("no dev/ino check","src/reader.mjs",[('if (pinned.file.dev !== state.dev || pinned.file.ino !== state.ino) throw','if (false) throw'),('if (fresh.file.dev !== state.dev || fresh.file.ino !== state.ino || fresh.length < state.length) throw','if (fresh.length < state.length) throw')]), + ("no shorter check","src/reader.mjs",[('if (pinned.shorter) throw','if (false) throw')]), + ("cwd always in project","src/reader.mjs",[(' if (typeof cwd !== "string" || !isAbsolute(cwd)) return false;',' return true;')]), + ("relative cwd accepted","src/reader.mjs",[(' if (typeof cwd !== "string" || !isAbsolute(cwd)) return false;',' if (typeof cwd !== "string" || !cwd) return false;')]), + ("no char cap","src/parts.mjs",[("chars + 1 > LIMITS.chars ||","false ||")]), + ("no page byte cap","src/parts.mjs",[("if (bytes + add > LIMITS.pageBytes) break;","")]), + ("no block cap","src/parts.mjs",[("current.length === LIMITS.blocks ||","false ||")]), + ("no foreign check","src/reader.mjs",[("if (actor !== record.actor || purpose !== record.purpose || conversation !== record.conversation || branch !== record.branch) {","if (false) {")]), + ("no branch binding","src/reader.mjs",[("|| branch !== record.branch) {",") {")]), + ("no expiry","src/reader.mjs",[("if (Date.parse(record.expiresAt) <= now()) {","if (false) {")]), + ("no parentSession notice","src/pi.mjs",[(" if (parsed.header.parentSession !== undefined && parsed.header.parentSession !== null) {\n units.push"," if (false) {\n units.push")]), + ("branch param ignored","src/reader.mjs",[("const built = build(snap, found.root, branch ?? null, conversation);","const built = build(snap, found.root, null, conversation);")]), + ("unknown branch served","src/reader.mjs",[("if (!parsed.branches.has(branch)) return null;","if (false) return null;")]), + ("registration harness ignored","src/reader.mjs",[("unsupportedReason: harness !== null && harness !== HISTORY ? UNSUPPORTED_HARNESS : null,","unsupportedReason: null,")]), + ("no placeholder roots","src/reader.mjs",[(" if (!reg || reg.layout !== \"repo\""," if (true || reg.layout !== \"repo\"")]), + ("malformed notices dropped","src/pi.mjs",[('out.push({ notice: "malformed", lines: [pending[mi++].line] });','mi++;')]), + ("bridge restored","src/pi.mjs",[(' path.push({ notice: "missing-parent", lines: lost });\n break;',' const prev = r.index > 0 ? parsed.byId.get(parsed.entries[r.index - 1].entry.id) : null;\n if (lost.length && prev) { path.push({ notice: "missing-parent", lines: lost }); r = prev; continue; }\n path.push({ notice: "missing-parent", lines: lost });\n break;')]), + ("missing-parent lines not named","src/pi.mjs",[("const lost = parsed.malformed.filter((m) => m.after === r.index - 1).map((m) => m.line);","const lost = [];")]), + ("trailing malformed only on default","src/pi.mjs",[(" flushBefore(Infinity);\n return out;"," if (leaf === parsed.defaultLeaf) flushBefore(Infinity);\n return out;")]), + ("branch named by leaf","src/pi.mjs",[(" const branchOf = (leaf) => {"," const branchOf = (leaf) => branchName(leaf);\n const unused = (leaf) => {")]), + ("later child continues branch","src/pi.mjs",[("if (children.get(p.entry.id)[0] !== r) return branchName(r);","if (children.get(p.entry.id).at(-1) !== r) return branchName(r);")]), + ("first root not main","src/pi.mjs",[("return r === firstRoot ? MAIN : branchName(r);","return branchName(r);")]), + ("no empty main","src/pi.mjs",[(" if (!branches.size) branches.set(MAIN, null);\n","")]), + ("redacted thinking shown","src/pi.mjs",[('const t = b.redacted === true ? "" : typeof b.thinking','const t = typeof b.thinking')]), + ("EACCES on a component rethrown","src/safe-fs.mjs",[('throw denied(err, "a session path component");','throw err;')]), + ("incomplete never set","src/reader.mjs",[("incomplete: buf.length > length };","incomplete: false };")]), + ("next re-reads fresh","src/reader.mjs",[("const pinned = snapshot(state.root, state.name, state.length);","const pinned = { ...snapshot(state.root, state.name), length: state.length };")]), + ("follow skips history check","src/reader.mjs",[("if (!built || built.entries.length < state.offset || idsDigest(built.entries, state.offset) !== state.prefix) {","if (!built) {")]), + ("follow skips the id digest","src/reader.mjs",[("|| idsDigest(built.entries, state.offset) !== state.prefix) {",") {")]), + ("follow reads the default branch","src/reader.mjs",[("const built = build(fresh, state.root, state.branch, conversation);","const built = build(fresh, state.root, null, conversation);")]), + ("catalogue writes a file","src/reader.mjs",[(" function catalogue() {"," function catalogue() {\n for (const r of listRoots()) { try { require_(r); } catch {} }")]), + ("thinking visibility wrong","src/pi.mjs",[('visibility: t ? "permitted-visible" : "unavailable"','visibility: "permitted-visible"')]), + ("raw tool ids","src/pi.mjs",[("call: safeId(b.id), name: safeId(b.name)","call: String(b.id), name: String(b.name)")]), + ("id namespaces dropped","src/pi.mjs",[("id: `n.${e.id}`","id: e.id")]), + ("surrogate split","src/parts.mjs",[("const width = cp > 0xffff ? 2 : 1;","const width = 1;")]), +] +orig={} +for f in ["src/safe-fs.mjs","src/reader.mjs","src/parts.mjs","src/pi.mjs"]: + orig[f]=open(os.path.join(S,f)).read() +for name,f,reps in M: + src=orig[f] + for a,b in reps: + if a not in src: print("NOT APPLIED",name,repr(a[:60])); break + src=src.replace(a,b) + else: + if name=="catalogue writes a file": + src=src.replace("require_(r);",'(await_import => 0)(); (require_fs => 0)(); import_fs.writeFileSync(r.dir + "/x.tmp", "")').replace('import { createHash, randomBytes } from "node:crypto";','import { createHash, randomBytes } from "node:crypto";\nimport * as import_fs from "node:fs";') + open(os.path.join(S,f),"w").write(src) + r=subprocess.run(["node","--test","tests/"],cwd=S,capture_output=True,text=True,timeout=600) + fails=sorted(set(l.strip()[2:].split(" (")[0] for l in r.stdout.splitlines() if l.lstrip().startswith("✖") and "failing tests" not in l)) + print(("CAUGHT " if r.returncode else "MISSED ")+name+" -> "+"; ".join(x[:70] for x in fails)[:300]) + open(os.path.join(S,f),"w").write(orig[f]) diff --git a/agents/dewey/work/chat-02/evidence/backend/smoke-stats.txt b/agents/dewey/work/chat-02/evidence/backend/smoke-stats.txt new file mode 100644 index 00000000..8d50aec0 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/smoke-stats.txt @@ -0,0 +1,2 @@ +{ convs: 18, pages: 102, entries: 8693, refusals: {}, maxMs: 798 } +catalogue 19 rows in 75 ms, refusedRoots []; 186 sampled pages and cursors: 0 schema-invalid; every page branch: main (102) diff --git a/agents/dewey/work/chat-02/evidence/backend/smoke.mjs b/agents/dewey/work/chat-02/evidence/backend/smoke.mjs new file mode 100644 index 00000000..67f6fa01 --- /dev/null +++ b/agents/dewey/work/chat-02/evidence/backend/smoke.mjs @@ -0,0 +1,37 @@ +import { discoverRepoAgents } from "/mnt/storage/src/mosaic-stack/packages/control-board/src/scan.mjs"; +import { rootsFromSpecs, createReader } from "/mnt/storage/src/mosaic-stack/packages/conversation/src/reader.mjs"; +import { readRegistration, LAYOUTS } from "/mnt/storage/src/mosaic-stack/packages/seat/src/seat.mjs"; +import { readdirSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +const seats = homedir() + "/.mosaic-dev/seats"; +const regs = []; +import { loadRegistrations } from "/mnt/storage/src/mosaic-stack/packages/control-board/src/scan.mjs"; regs.push(...loadRegistrations(seats).registrations); +const roots = rootsFromSpecs(discoverRepoAgents("/mnt/storage/src/mosaic-stack"), regs); +console.log(roots.map(r => `${r.seat} ${r.harness} ${r.unsupportedReason} ${r.engineStartedAt}`).join("\n")); +const reader = createReader({ roots }); +let t = Date.now(); +const cat = reader.catalogue(); +console.log("catalogue", cat.conversations.length, "ms", Date.now() - t, "refusedRoots", JSON.stringify(cat.refusedRoots)); +const byAvail = {}; for (const c of cat.conversations) byAvail[c.availability + ":" + c.refusal] = (byAvail[c.availability + ":" + c.refusal] ?? 0) + 1; +console.log(byAvail); +console.log(cat.conversations.slice(0, 3)); +const pages = []; +let stats = { convs: 0, pages: 0, entries: 0, refusals: {} , maxMs: 0}; +for (const c of cat.conversations.filter(c => c.availability === "available")) { + t = Date.now(); + let r = reader.open({ conversation: c.conversation }); + if (!r.ok) { stats.refusals[r.refusal.code] = (stats.refusals[r.refusal.code] ?? 0) + 1; continue; } + stats.convs++; + let n = 0; + while (true) { + stats.pages++; stats.entries += r.page.entries.length; n++; + if (pages.length < 400) pages.push(r.page); + if (r.cursor) pages.length < 400 && pages.push(r.cursor); + if (!r.page.hasMore) break; + r = reader.next({ cursor: r.page.nextCursor, conversation: c.conversation, branch: r.page.branch }); + if (!r.ok) { console.log("next refused", r.refusal); break; } + } + stats.maxMs = Math.max(stats.maxMs, Date.now() - t); +} +console.log(stats); +writeFileSync("/tmp/dewey-chat02/pages.json", JSON.stringify(pages)); diff --git a/agents/filbert/work/chat-02-backend-review-2026-09-26.md b/agents/filbert/work/chat-02-backend-review-2026-09-26.md new file mode 100644 index 00000000..88e07397 --- /dev/null +++ b/agents/filbert/work/chat-02-backend-review-2026-09-26.md @@ -0,0 +1,206 @@ +# CHAT-02 backend: Filbert's code review + +Reviewer: Filbert, 2026-09-26. Requested by Dewey, assigned by Sage (D4). + +Candidate: `agents/dewey/work/chat-02/BACKEND.md`, sha256 +`286c3ad5f147471f16ef88537c7680c5bd6130e6365ce1f0c4cb98cd74e75c2e`, and the +nine files its §1 pins. All nine verify. Measured against `BRIEF.md` R4 +(`636b0fac…`, verified) and Sage's D1–D4 (`2026-09-26_lead-decisions.md` +item 8). Jason's go on CHAT-02 is recorded in the same file (20:31Z). + +**Verdict: revise.** The safe-open path, snapshots, cursor refusals, byte +limits and the no-write discipline are sound, and the tests are strong. Three +defects need fixing, and I reproduced each with a scratch script (§2). Three +small items can go in the same revision (§3). No finding needs a lead ruling. + +## 1. What I checked and reproduced + +- **Suites.** The shared working tree now carries unpinned edits to + `packages/webui/src/public/app.js`, `index.html` and + `packages/ledger/src/ledger.mjs`, and four webui browser tests fail there. + To test the candidate alone, I ran a detached worktree at HEAD `1c5f6bc3` + with only the nine pinned files overlaid (pins re-verified in place). The + result: conversation, control-board, webui and seat, 175/175. `chat-00`, + `chat-01` and `chat-01c` `check.mjs` all exit 0. Dewey: the shared-tree + failures come from the unpinned webui files, not from this packet. +- **Real data, read-only.** Catalogue: 18 Pi rows in 71 ms. Following the + three most recently active views took 1, 33 and 177 ms per poll, with the + cache thrashing across three views. That is acceptable for §3 item 9. +- **Pi semantics** against the pinned 0.85.1 `session-manager.js`: + - the default leaf is the last entry in file order (`_buildIndex`); + - `branch()` moves the leaf in memory only; + - `resetLeaf()` starts a new root entry, so a file can have several roots; + - `retainedTail` is documented in `docs/session-format.md` 245. + + The parser matches all four. +- **Safe open.** It opens with `O_NOFOLLOW`, compares (dev, ino) after the + `fstat`, re-checks the components after the open, and reads only from the + descriptor. Registrations never add a root or name a file, and a spec must + have the exact `.pi/state//sessions` shape. The listing refuses + symlinks and odd names without opening them. +- **Accepted choices, packet §3:** + - item 1, the summary row; + - item 2, `sessionsDir` matching, which is stricter than the brief's + `sessionFile`; + - item 3, the placeholder row; + - item 4, the epoch comparable within one chain, which README 132–135 states; + - item 7, apart from finding 3 below; + - items 8, 9 and 10. + + Item 5 stands, apart from finding 1. Item 6 does not stand: see finding 2. + +## 2. Required + +1. **A branch's id changes every time the conversation grows.** + - The branch id is the leaf entry's id (`reader.mjs` 270). Every append + makes a new leaf, so one thread is named differently from one page to + the next. + - Scratch file: open a linear `a→b`, then append `c→d`. + - The open page has `branch: "b"`. + - The follow, sent with `branch: "b"` on a cursor bound to `b`, returns + a page with `branch: "d"`, and its entries carry `d`. + - `open({branch: "b"})` then refuses `unknown-branch` (404, reconcile). + - The F12 test asserts this behaviour (`g.page.branch === "o5"` after + following `o4`). + - Effects: + - One view's entries carry different `branch` values. + - A cursor bound to one branch serves a page labelled with another, + although CHAT-01 line 66 binds a cursor to its branch. + - Any branch id the Console keeps goes stale after one append. After a + board restart (`cursor-unknown`), reopening the stored branch fails. + - Fix: give each branch an id that growth cannot change. Recommended rule: + - walk from each root in file order; + - at a fork, the child earliest in file order continues the current + branch, and each later child starts a branch named after its own + entry id; + - the first root names the first branch; + - each later root (`resetLeaf`) starts its own branch. + - Appending never changes which child came first, so ids are stable. + - `view.branches` lists each branch's current leaf and last activity. + `page.branch` always equals the cursor's branch. + - Tests: + - a follow keeps the branch id; + - a pre-growth id still opens; + - entries across follows share one `branch` value; + - F12 with the new ids; + - a two-root file. +2. **The "assumed link" bridge merges branches.** + - `branchPath` (`pi.mjs` 94–104) joins a missing parent to the previous + valid entry in the file whenever a malformed line sits between them. + - Scratch file: `a` (root), `b→a`, then a malformed line that held + `X→a` (a fork from `a`), then `y→X`. + - `view.branches` lists `b` and `y` as separate branches. + - The default page for `y` still shows `root`, then `ON THE OTHER + BRANCH` (b's text), then the notice, then `leaf`. That is b's history + rendered as y's. + - F12 says "no merge", and the brief's F1 asks only that reading continues. + A notice saying the link is assumed doesn't make the merged history + correct. + - Nothing is lost without the bridge. The entry before the gap has no + children, so it is already a leaf, and the history before the gap reads + as its own branch. + - Fix: drop the bridge. Stop with the missing-parent notice and name the + unreadable line(s) in it. + - Tests: update the two F1 bridge tests, and keep a follow case in which + the view stays put and reports the new default. +3. **A directory component without search permission fails the whole + catalogue.** + - `lstatOrNull` (`safe-fs.mjs` 38–45) rethrows `EACCES`, so `checkRoot` + throws a plain error instead of a refusal. `catalogue()` rethrows it, and + the route returns 500. + - Scratch file: two seats, with `.pi/state/bad` at mode 000. + - `catalogue()` throws `EACCES` on lstat of `bad/sessions`. + - `open()` of an unknown id throws too, and so does any id resolved + after the bad root. + - Packet §3 item 7 and the test at line 639 cover the sessions directory + at mode 0300 (search allowed), not a component above it. + - Fix: map `EACCES`/`EPERM` in `lstatOrNull` through `denied()`. + - Test: a seat directory at mode 000 beside a readable one. The catalogue + lists the readable root and one `unreadable` entry in `refusedRoots`. + Opens in the readable root still work in either root order. + +## 3. Small, same revision + +1. **Redacted thinking is marked visible.** Pi stores Anthropic + `redacted_thinking` as `{type: "thinking", thinking: "[Reasoning + redacted]", redacted: true}` (`pi-ai` `anthropic-messages.js` 445–451). The + parser gives it `permitted-visible` with that placeholder. CHAT-01 says + unavailable reasoning has empty text. Map `redacted === true` to + `unavailable` with `""`, and never read `thinkingSignature`. There are no + live cases today. +2. **A relative header `cwd` resolves against the board's working + directory.** `cwdInProject` calls `resolve(cwd)`. With `cwd: "."` and the + board started from the checkout, the file passes. Pi writes absolute + paths. Refuse a non-absolute `cwd` as `foreign-project`, and add a test. +3. **A file whose entries are all malformed shows no notices.** With no leaf, + `build` uses an empty path (`reader.mjs` 272), so `view.unreadableLines` + is N and the page has no parts. Emit the malformed notices when there is no + leaf. + +## 4. Not findings + +- The F8 `fstat` mismatch is covered by design only. That is acceptable, as + packet §5 says. +- Cursors are reusable until expiry or eviction. A local process can evict + another view's cursors by opening 1000 pages, which ends in a reconcile. + That fits the one-actor loopback scope. +- Darkwing reviews the route code. I read it only as far as the reader calls + it: the query validation, status map and 500 path look right. + +## To reach approve + +Fix §2 items 1–3, with their tests. Take §3 items 1–3 or say why not. +Re-pin, re-run the suites and the mutation scripts, and send the new packet +hash. I'll review the delta. + +Reproductions: `agents/filbert/work/chat-02-backend-review-evidence/` +`branch.mjs` (§2 item 1), `bridge.mjs` (item 2) and `perm.mjs` (item 3). +Each builds its own temp project and imports the working-tree reader. Run +them with `node `. + +## Revision 2 delta review + +Candidate: `BACKEND.md` sha256 +`0cf177b14cfdc52b7026d67333dc16eef345d80a7d908d50d69bb4301050ca85`, and the +nine files its §1 pins. All nine verify. Revision 1 is frozen as +`BACKEND-r1-286c3ad5.md` (hash verified). I diffed `safe-fs.mjs`, `pi.mjs`, +`reader.mjs` and `serve.mjs` against my revision-1 snapshot. + +Checks: +- **Suites.** Detached worktree at `1c5f6bc3` with only the nine files + overlaid: 181/181. `chat-00`, `chat-01` and `chat-01c` exit 0. +- **My reproductions, rerun on the candidate:** + - `branch.mjs`: open and follow both give `main`, and the pre-growth + reopen works. + - `bridge.mjs`: `b.y` shows the missing-parent notice (naming line 4) and + `leaf` only. `main` stays separate. + - `perm.mjs`: the catalogue returns the readable row, with `bad` in + `refusedRoots` as `unreadable`. An unknown id refuses instead of throwing. +- **A new probe (`fork.mjs`):** a linear `main`, then a fork from its middle, + then growth back on `main`, then a second root. + - Every branch name held across all of it: `main`, `b.x`, `b.r`. + - Follows stayed on their branch, and returned an empty page when the + growth was elsewhere. + - `view.defaultBranch` tracked Pi's last entry. + - `open` by name still worked afterwards. +- **Naming rule.** At most one leaf can end each earliest-child walk. For + that reason, two leaves cannot share a name. For a file that only grows, a + name cannot change: children are sorted by file index, and the first root + is fixed. +- **Small items.** Redacted thinking gives `unavailable` with `""`; a relative + `cwd` is `foreign-project`; an all-malformed file gives one notice per + line. Each has a test. +- **Malformed notices on every branch,** after the leaf too. This keeps a + branch's served parts a prefix of its later parts, because new lines only + ever append at the end. + +**Verdict: approve** `0cf177b1`. + +One nonblocking note: the new §5 limit, where a duplicate id with new +content goes unnoticed, could be closed cheaply. `idsDigest` could hash the +serialized parts, whose sizes are already computed, instead of their ids. +Pi's `generateId` never reuses an id, so this can wait. + +Darkwing owns the route delta. I read only the `serve.mjs` diff: the cursor +without `branch` gives a 400, and there are three new `REFUSAL_STATUS` +entries. It looks right. diff --git a/packages/control-board/src/serve.mjs b/packages/control-board/src/serve.mjs index aacf4f08..655ac620 100644 --- a/packages/control-board/src/serve.mjs +++ b/packages/control-board/src/serve.mjs @@ -9,6 +9,14 @@ // seat's tmux pane through tools/tmux/agent-send.sh (#1505); // answers {delivered, exitCode, stdout, stderr, ...} // GET /healthz {"ok":true} +// GET /api/conversations +// read-only Pi conversation catalogue (#1507, CHAT-02) +// GET /api/conversation?id=[&branch=][&cursor=] +// one CHAT-01 history page: the first page of a branch (the +// default one without branch), or the next page (or a +// follow) for a cursor. A cursor call repeats the page's +// branch; without it the answer is 400. Refusals carry +// {error, refusal: {code, reconcile}}. // // POST requires Content-Type: application/json. A plain form post from another // site in the browser cannot set that header without a CORS preflight, and this @@ -24,6 +32,13 @@ // // Every /api/board request rescans, so the page is never staler than its // refresh timer. The scan rewrites the derived board files as a side effect. +// +// The conversation routes read only: packages/conversation lists the repository +// specs' Pi session roots (never fleet or connector ones), opens files +// O_NOFOLLOW and writes nothing. Registrations are hints there too. The +// conversation id is opaque and no path comes from the request. Cursors live +// in this server's memory, bound to the one actor this unauthenticated +// loopback route has, local-operator. import { createServer as createHttpServer } from "node:http"; import { readFileSync } from "node:fs"; @@ -31,7 +46,8 @@ import { join, resolve } from "node:path"; import { isIP } from "node:net"; import { hostname } from "node:os"; import { spawnSync } from "node:child_process"; -import { scan, markSeen, seenKey, ConfigError } from "./scan.mjs"; +import { scan, markSeen, seenKey, ConfigError, loadRegistrations } from "./scan.mjs"; +import { createReader, rootsFromSpecs } from "../../conversation/src/reader.mjs"; const MAX_BODY = 4096; @@ -149,13 +165,70 @@ export function foreignRequest(req) { return null; } +// HTTP status for each reader refusal. The body always carries the code. The +// tests hold every code the reader can raise to an entry here. +export const REFUSAL_STATUS = { + "unknown-conversation": 404, + "unknown-branch": 404, + unavailable: 404, + "cursor-unknown": 409, + "cursor-expired": 409, + "cursor-foreign": 409, + "source-replaced": 409, + "incomplete-header": 409, + "unsafe-path": 403, + "foreign-project": 403, + unreadable: 403, + "unsupported-harness": 422, + "not-a-pi-session": 422, + "too-large": 422, + "unknown-actor": 403, + "unsupported-purpose": 422, +}; +const QUERY_VALUE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +function sendConversationJson(res, status, body) { + res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store", "x-content-type-options": "nosniff" }); + res.end(JSON.stringify(body) + "\n"); +} + +// GET /api/conversation: id is required; branch and cursor are optional; any +// other, repeated or malformed parameter is a 400. +function conversationQuery(url) { + const out = {}; + for (const key of new Set(url.searchParams.keys())) { + const values = url.searchParams.getAll(key); + if (!["id", "branch", "cursor"].includes(key)) return { error: `unknown parameter: ${key}` }; + if (values.length !== 1 || !QUERY_VALUE.test(values[0])) return { error: `invalid ${key}` }; + out[key] = values[0]; + } + if (!out.id) return { error: "id is required" }; + if (out.cursor && !out.branch) return { error: "a cursor call repeats the page's branch" }; + return out; +} + +export function conversationResponse(reader, url) { + if (url.pathname === "/api/conversations") { + if ([...url.searchParams.keys()].length) return { status: 400, body: { error: "no parameters are accepted" } }; + return { status: 200, body: reader.catalogue() }; + } + const query = conversationQuery(url); + if (query.error) return { status: 400, body: { error: query.error } }; + const out = query.cursor + ? reader.next({ cursor: query.cursor, conversation: query.id, branch: query.branch }) + : reader.open({ conversation: query.id, branch: query.branch ?? null }); + if (out.ok) return { status: 200, body: out }; + return { status: REFUSAL_STATUS[out.refusal.code] ?? 422, body: { error: out.refusal.message, refusal: { code: out.refusal.code, reconcile: out.refusal.reconcile } } }; +} + export function loadPage(path = join(import.meta.dirname, "page.html")) { return readFileSync(path, "utf8"); } // specs: agent specs to scan on each request. boardDir: where scan writes. -export function createServer({ specs, boardDir, isAlive, now, seatsDir = null, discordDataRoot = null, page = loadPage(), isPidAlive, agentSend = DEFAULT_AGENT_SEND, exec = spawnSync }) { +export function createServer({ specs, boardDir, isAlive, now, seatsDir = null, discordDataRoot = null, page = loadPage(), isPidAlive, agentSend = DEFAULT_AGENT_SEND, exec = spawnSync, conversationReader = null }) { const rescan = () => scan(specs, { boardDir, isAlive, now, seatsDir, isPidAlive, discordDataRoot }); + const reader = conversationReader ?? createReader({ roots: () => rootsFromSpecs(specs, loadRegistrations(seatsDir).registrations) }); return createHttpServer((req, res) => { const refused = foreignRequest(req); if (refused) { @@ -212,6 +285,16 @@ export function createServer({ specs, boardDir, isAlive, now, seatsDir = null, d res.writeHead(200, { "content-type": "application/json", "cache-control": "no-store" }); return res.end(JSON.stringify(index) + "\n"); } + if (url.pathname === "/api/conversations" || url.pathname === "/api/conversation") { + let out; + try { + out = conversationResponse(reader, url); + } catch (err) { + process.stderr.write(`conversation read failed: ${err.message}\n`); + out = { status: 500, body: { error: "conversation read failed" } }; + } + return sendConversationJson(res, out.status, out.body); + } if (url.pathname === "/favicon.ico") { res.writeHead(204); return res.end(); diff --git a/packages/control-board/tests/serve.test.mjs b/packages/control-board/tests/serve.test.mjs index 1729d1e5..e1f5e18a 100644 --- a/packages/control-board/tests/serve.test.mjs +++ b/packages/control-board/tests/serve.test.mjs @@ -9,6 +9,9 @@ import { existsSync, chmodSync, statSync, + symlinkSync, + lstatSync, + readdirSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve, basename } from "node:path"; @@ -16,8 +19,9 @@ import { spawnSync, spawn } from "node:child_process"; import { createServer as createNetServer } from "node:net"; import { request as httpRequest } from "node:http"; import { ConfigError, markSeen } from "../src/scan.mjs"; -import { isLoopbackHost, startServer, DEFAULT_AGENT_SEND, REPLY_LIMIT, REPLY_TRAILER } from "../src/serve.mjs"; +import { isLoopbackHost, startServer, DEFAULT_AGENT_SEND, REPLY_LIMIT, REPLY_TRAILER, REFUSAL_STATUS } from "../src/serve.mjs"; import { writeRegistration, makeRegistration } from "../../seat/src/seat.mjs"; +import { UNSUPPORTED_HARNESS } from "../../conversation/src/reader.mjs"; const pkgRoot = resolve(import.meta.dirname, ".."); const cli = join(pkgRoot, "src", "cli.mjs"); @@ -1025,3 +1029,146 @@ test("Host/Origin guard: loopback names on this port are accepted, with or witho await closeServer(server); } }); + +// --------------------------------------------------------------------------- +// 12. Read-only conversation routes (#1507, CHAT-02 D3): GET /api/conversations +// and GET /api/conversation, served from packages/conversation. Same Host and +// Origin guard as every route (F16), application/json with nosniff and +// no-store, refusals mapped to 4xx with their code, and nothing written. +// --------------------------------------------------------------------------- + +function conversationFixture() { + const root = makeRoot(); + const proj = join(root, "proj"); + const sessionsDir = join(proj, ".pi", "state", "agent1", "sessions"); + const header = { type: "session", version: 3, id: "c0ffee00-0000-4000-8000-000000000001", timestamp: "2026-09-26T12:00:00.000Z", cwd: proj }; + const lines = [header]; + for (let i = 0; i < 120; i++) { + lines.push({ type: "message", id: `e${i}`, parentId: i ? `e${i - 1}` : null, timestamp: new Date(Date.parse("2026-09-26T12:00:01Z") + i * 1000).toISOString(), message: { role: i % 2 ? "assistant" : "user", content: [{ type: "text", text: `m${i}` }] } }); + } + writeSessionFile(sessionsDir, "s.jsonl", lines.map((l) => JSON.stringify(l))); + const outside = join(root, "outside.jsonl"); + writeFile(outside, JSON.stringify({ ...header, cwd: "/elsewhere" }) + "\n"); + chmodSync(outside, 0o000); + symlinkSync(outside, join(sessionsDir, "link.jsonl")); + const seatsDir = join(root, "seats"); + writeRegistration(seatsDir, makeRegistration({ + resolved: { seat: "rocko", project: "proj", sessionsDir: join(proj, ".pi", "state", "rocko", "sessions"), seatDir: join(root, "rocko"), launchScript: join(root, "rocko", "launch.sh"), layout: "repo", defaultWorkspace: null }, + harness: "claude-code", + })); + return { root, proj, sessionsDir, seatsDir, boardDir: join(root, "board"), specs: [{ agent: "agent1", project: "proj", sessionsDir, tmux: {} }] }; +} + +// Size, sha256, mtime, (dev, ino) and listing of every entry under dir. +function treePrint(dir) { + const out = {}; + for (const name of readdirSync(dir).sort()) { + const path = join(dir, name); + const st = lstatSync(path, { bigint: true }); + out[name] = [String(st.dev), String(st.ino), String(st.size), String(st.mtimeNs), st.isFile() ? createHashHex(readFileSync(path)) : null]; + } + return out; +} +function createHashHex(buf) { + return spawnSync("sha256sum", { input: buf, encoding: "utf8" }).stdout.split(" ")[0]; +} + +test("conversation routes (F16): a foreign Host, a wrong port and a cross-origin Origin get 403 before the reader runs, with no CORS headers", async () => { + const f = conversationFixture(); + const calls = []; + const spy = { catalogue: () => calls.push("catalogue"), open: () => calls.push("open"), next: () => calls.push("next") }; + const server = await startServer({ host: "127.0.0.1", port: 0, specs: f.specs, boardDir: f.boardDir, seatsDir: f.seatsDir, isAlive: () => true, page: "", conversationReader: spy }); + const port = server.address().port; + const own = `127.0.0.1:${port}`; + const cases = [ + ["foreign Host", { host: `rebind.example:${port}` }, "non-local Host refused"], + ["loopback Host, wrong port", { host: `127.0.0.1:${port + 1}` }, "non-local Host refused"], + ["cross-origin Origin", { host: own, origin: "http://rebind.example" }, "cross-origin request refused"], + ["opaque Origin", { host: own, origin: "null" }, "cross-origin request refused"], + ]; + try { + for (const path of ["/api/conversations", "/api/conversation?id=pi-00000000000000000000000000000000", "/api/conversation?id=x&cursor=c-1"]) { + for (const [label, headers, error] of cases) { + const r = await rawRequest(port, { path, headers }); + assert.equal(r.status, 403, `${path}, ${label}`); + assert.deepEqual(JSON.parse(r.text), { error }, `${path}, ${label}`); + for (const h of Object.keys(r.headers)) assert.ok(!h.startsWith("access-control-"), `${path}, ${label}: ${h}`); + } + } + assert.deepEqual(calls, [], "the reader never ran for a refused request"); + assert.equal((await rawRequest(port, { path: "/api/conversations", headers: { host: own, origin: `http://${own}` } })).status, 200, "same-origin passes"); + assert.deepEqual(calls, ["catalogue"]); + } finally { + await closeServer(server); + } +}); + +test("every refusal code the reader can raise has an HTTP status", () => { + const src = ["reader.mjs", "pi.mjs", "safe-fs.mjs"].map((f) => readFileSync(join(pkgRoot, "..", "conversation", "src", f), "utf8")).join("\n"); + const codes = new Set([...src.matchAll(/new Refusal\(\s*"([a-z-]+)"/g)].map((m) => m[1])); + codes.add(UNSUPPORTED_HARNESS); // raised by value, as a root's unsupportedReason + assert.ok(codes.size >= 15, [...codes].join(" ")); + assert.deepEqual([...codes].filter((c) => !(c in REFUSAL_STATUS)), []); + assert.deepEqual(Object.keys(REFUSAL_STATUS).filter((c) => !codes.has(c)), [], "no stale entries"); +}); + +test("conversation routes: catalogue, first page, next page and follow over HTTP; refusals map to 4xx with their code; nothing is written", async () => { + const f = conversationFixture(); + const before = treePrint(f.sessionsDir); + const server = await startServer({ host: "127.0.0.1", port: 0, specs: f.specs, boardDir: f.boardDir, seatsDir: f.seatsDir, isAlive: () => true, page: "" }); + const base = `http://127.0.0.1:${server.address().port}`; + const get = async (path) => { + const res = await fetch(base + path); + assert.equal(res.headers.get("content-type"), "application/json", path); + assert.equal(res.headers.get("x-content-type-options"), "nosniff", path); + assert.equal(res.headers.get("cache-control"), "no-store", path); + assert.equal(res.headers.get("access-control-allow-origin"), null, path); + return { status: res.status, body: await res.json() }; + }; + try { + const cat = await get("/api/conversations"); + assert.equal(cat.status, 200); + const rows = cat.body.conversations; + const pi = rows.find((c) => c.availability === "available"); + const link = rows.find((c) => c.availability === "denied"); + const claude = rows.find((c) => c.availability === "unsupported"); + assert.deepEqual([pi.seat, pi.title, pi.readOnly], ["agent1", "m0", true]); + assert.deepEqual([link.refusal, claude.seat, claude.unsupportedReason], ["unsafe-path", "rocko", "unsupported-harness"]); + + const first = await get(`/api/conversation?id=${pi.conversation}`); + assert.equal(first.status, 200); + assert.deepEqual([first.body.page.kind, first.body.page.entries.length, first.body.page.hasMore, first.body.page.branch], ["page", 100, true, "main"]); + const q = (cursor) => `/api/conversation?id=${pi.conversation}&branch=${first.body.page.branch}&cursor=${cursor}`; + const second = await get(q(first.body.page.nextCursor)); + assert.equal(second.status, 200); + assert.deepEqual([second.body.page.entries.length, second.body.page.hasMore], [20, false]); + assert.equal(second.body.page.entries.at(-1).content[0].text, "m119"); + const follow = await get(q(second.body.follow.id)); + assert.deepEqual([follow.status, follow.body.page.entries.length], [200, 0]); + const branch = await get(`/api/conversation?id=${pi.conversation}&branch=main`); + assert.equal(branch.status, 200); + + const refusals = [ + [`/api/conversation?id=pi-${"0".repeat(32)}`, 404, "unknown-conversation", true], + [`/api/conversation?id=${pi.conversation}&branch=b.e5`, 404, "unknown-branch", true], + [q("c-unknown"), 409, "cursor-unknown", true], + [`/api/conversation?id=${pi.conversation}&branch=b.e1&cursor=${first.body.page.nextCursor}`, 409, "cursor-foreign", true], + [`/api/conversation?id=${link.conversation}`, 403, "unsafe-path", false], + [`/api/conversation?id=${claude.conversation}`, 422, "unsupported-harness", false], + ]; + for (const [path, status, code, reconcile] of refusals) { + const r = await get(path); + assert.equal(r.status, status, path); + assert.deepEqual(r.body.refusal, { code, reconcile }, path); + assert.equal(typeof r.body.error, "string"); + } + for (const path of ["/api/conversation", "/api/conversation?id=a&id=b", "/api/conversation?id=a&path=/etc/passwd", "/api/conversation?id=a&path=x", `/api/conversation?id=${pi.conversation}&cursor=${second.body.follow.id}`, "/api/conversation?id=../x", `/api/conversation?id=${"a".repeat(129)}`, "/api/conversations?x=1"]) { + const r = await get(path); + assert.equal(r.status, 400, path); + } + assert.deepEqual(treePrint(f.sessionsDir), before, "the sessions directory is unchanged"); + assert.equal(existsSync(join(f.proj, ".pi", "state", "rocko")), false, "the Claude seat's directory was never created"); + } finally { + await closeServer(server); + } +}); diff --git a/packages/conversation/README.md b/packages/conversation/README.md new file mode 100644 index 00000000..48975c0e --- /dev/null +++ b/packages/conversation/README.md @@ -0,0 +1,207 @@ +# conversation + +Read-only Pi conversation histories for the Console: a catalogue of approved +session files, full branch history in CHAT-01 pages, and cursors. Opening a +conversation never resumes, forks, launches or controls anything, and nothing +here writes a file. + +Issue #1507 (CHAT-02, row 5). Brief: `agents/dewey/work/chat-02/BRIEF.md` R4. +Plain ESM, no dependencies, Node 24 or newer. A library with no server: the +control board serves it on two GET routes (D3). + +## API + +```js +import { rootsFromSpecs, createReader } from "@mosaic/conversation"; + +const reader = createReader({ roots: () => rootsFromSpecs(specs, registrations) }); +reader.catalogue(); // { ok, conversations, refusedRoots, generatedAt } +reader.open({ conversation, branch? }); // first page +reader.next({ cursor, conversation, branch }); // next page, or a follow +``` + +`open` and `next` return `{ ok: true, page, cursor, follow, view }` or +`{ ok: false, refusal: { code, reconcile, message } }`. + +- `page` is a CHAT-01 `page`, and `cursor` and `follow` are CHAT-01 `cursor` + records. `page.nextCursor` is `cursor.id` when more parts remain in the + snapshot. +- On the last page, `follow` replaces `cursor`. Calling `next` with it takes + a fresh snapshot and returns only what was appended since (see Follow). +- `view` holds what the Console needs beyond CHAT-01: `defaultBranch`, + `branches`, `incomplete` (a truncated trailing line), + `forkedFromEarlierSession` and `unreadableLines`. +- `actor` defaults to `local-operator` and `purpose` to `history`. Any other + value is refused. + +## Sources + +Roots come from the board's repository specs only: +`/.pi/state//sessions`, with the project named after the +root directory. Fleet and connector specs are not roots, and there is no +global scan. A conversation id is `pi-` plus a hash of project root, seat and +file name. It is resolved by listing the roots again, so no path comes from +the caller. + +A seat registration is seat-written, so it is a hint, not authority: + +- It counts for a root only when seat, layout `repo`, project and + `sessionsDir` all match (`samePath`). Anything else is ignored. +- It supplies `engineStartedAt` for conversations created at or after its + `startedAt`. +- A non-Pi `harness` makes the root unsupported. The root appears as one + catalogue row with `unsupportedReason: "unsupported-harness"` (D2), and its + directory is never read. +- A seat on another harness has no Pi sessions directory, so the board has no + spec for it. Its registration adds that placeholder row when it names the + standard directory under a project root that is already approved. This is + Rocko's case today: `claude-code`, no `sessions`. + +Deviation from brief §2.1: the brief speaks of a registration `sessionFile`. +Registrations have no such field, only `sessionsDir`, so the rule above +applies to the directory. A registration never adds a root or names a file. + +## Opening a file + +`src/safe-fs.mjs`: + +- Every component from the project root down to the sessions directory must + be a real directory (lstat). +- A session file must be a regular `*.jsonl` directly in the root. +- It is opened `O_RDONLY | O_NOFOLLOW | O_NONBLOCK`, and the descriptor's + (dev, ino) must equal the lstat taken before the open. +- The components are checked again after the open. Node has no `openat`, so + a directory swapped between the checks and the open is detected afterwards, + not prevented. + +The project root itself may be a symlink (the compatibility path to this +checkout). The header `cwd` must be the project or inside it. Real paths are +compared, and a `cwd` that no longer exists is compared as written. That +comparison calls `realpath` on the recorded `cwd`, which resolves the path but +opens nothing. + +`SessionManager.open` is never used. + +## Parser + +`src/pi.mjs`, pinned against `@earendil-works/pi-coding-agent` 0.85.1 +(`docs/session-format.md`, `dist/core/session-manager.js`): + +- Line 1 must be the session header. +- Entries form an `id`/`parentId` tree. Where an id repeats, the later entry + wins, as in Pi's index. +- The default leaf is the last valid entry in file order, as Pi loads it. + Each leaf ends one branch; other branches are read-only and opened by + `branch`. +- Branch names do not change while the file grows. The first root's line is + `main`, even before the file has entries. Each later root (Pi's + `resetLeaf`, or an entry whose parent is missing) and each later child at a + fork starts a branch named `b.`; the earliest child in file order + continues its parent's branch. An inner entry is not a branch. +- A malformed line becomes a notice at its file position on every branch, + after the leaf too, and reading continues. The same placement on every + branch keeps a branch's earlier parts unchanged while the file grows. +- A missing parent stops the history with a notice. When unreadable lines sit + just before the entry, the notice names them as the likely place of the + parent. The history is never joined across the gap: the entries before it + may belong to another branch, and they read as their own branch. A loop + stops with a notice. +- `parentSession` gives a "forked from an earlier session" notice and is + never opened. +- Redacted reasoning (`redacted: true`) is `unavailable` with empty text. + `thinkingSignature` is never read. +- A header `cwd` must be absolute and inside the project; a relative one is + refused as `foreign-project`. +- A compaction is a marker in place, and the full history stays on the path. + `retainedTail` entries are already on the path, so they are not rendered + twice. +- Model and thinking-level changes, labels, session names and extension state + are not shown. `custom_message` shows only with `display: true`. An unknown + entry type or role becomes a notice. +- Ids that do not fit the CHAT-01 id pattern are hashed (`h-` plus 40 hex). + OpenAI tool call ids such as `call_x|fc_y` are the real case. Entry ids are + namespaced (`n.` for native, `x.` for notices), so no native id can collide + with a notice. +- `get_entries` order is not used. + +## Pages + +`src/parts.mjs` applies the CHAT-01 limits: + +- at most 100 parts and 8 MiB of serialized UTF-8 per page; +- at most 64 blocks per part; +- at most 262144 characters per string. + +Strings split into fragments. A fragment is also cut at 1 MiB of JSON, and +blocks group into parts of at most 4 MiB, so one part always fits a page. +Content is never clipped, and a surrogate pair is never cut. + +## Snapshots, epochs, cursors + +- A snapshot is the file up to its last newline when the descriptor was read. + A trailing partial line is left out and sets `view.incomplete`. + `snapshotDigest` is the SHA-256 of those bytes. +- `sourceEpoch` is a hash of (dev, ino) and the digest at open. Follows carry + it forward while the prefix verifies, so growth keeps the epoch. A new open + after growth hashes the longer prefix and gets a different id for the same + epoch, so an epoch id is comparable only within one cursor chain. +- Every `next` re-reads the pinned prefix and refuses `source-replaced` (with + reconcile) when dev or ino changed, the file is shorter, or the prefix + digest differs (an in-place rewrite with the same inode). +- Cursors live in memory, with an LRU cap (1000) and a 10-minute TTL. They are + bound to actor, purpose, conversation, branch, snapshot, epoch and expiry. +- Refusals: `cursor-unknown` (never issued or evicted), `cursor-expired`, + `cursor-foreign` (any binding differs) and `source-replaced`. All carry + `reconcile: true`. A foreign attempt does not consume the cursor, so the old + view keeps working. +- `local-operator` is the only actor on this unauthenticated loopback route. + That is not multi-actor safety. Authenticated actors come with CHAT-04R. + +### Follow + +A follow cursor verifies the old prefix, then takes a fresh snapshot and +reads the same branch in it. `page.branch` is always the cursor's branch. + +- The parts before the cursor must be unchanged (compared as a digest of + entry ids). If they are, the page holds only the parts appended to this + branch, which is empty when the conversation continued on another branch. + `view.defaultBranch` shows where Pi's default leaf is now. +- If the branch is gone or its earlier parts changed (a duplicate id that + replaces an entry, or a missing parent that turns up later), it refuses + `source-replaced`. + +## Refusal codes + +| Code | Meaning | Reconcile | +|---|---|---| +| `unknown-conversation` | not in the approved roots now | yes | +| `unknown-branch` | not a branch of this conversation | yes | +| `unavailable` | the session root no longer exists | no | +| `unsupported-harness` | non-Pi seat (D2) | no | +| `unsafe-path` | symlink, bad name, not a regular file, swapped | no | +| `foreign-project` | header `cwd` outside the project | no | +| `unreadable` | permission denied on a file, a root or a directory above it inside the approved roots | no | +| `not-a-pi-session`, `incomplete-header` | first line is not a complete Pi header | incomplete: yes | +| `too-large` | over 256 MiB | no | +| `cursor-unknown`, `cursor-expired`, `cursor-foreign`, `source-replaced` | see above | yes | +| `unknown-actor`, `unsupported-purpose` | not `local-operator` / `history` | no | + +## Costs + +Each page reads and hashes the whole pinned prefix. A parse cache and a +branch-entries cache (two snapshots each) avoid re-parsing. On this checkout's +18 roots (largest file 18.6 MB), reading every page of every conversation took +at most 680 ms per conversation. + +## Tests + +`node --test packages/conversation/tests/` covers fixtures F1–F15 and F17 of +the brief (F16 is in the control-board suite, which serves the routes). + +- Every reader call runs inside a fingerprint of the fixture tree: size, + SHA-256, mtime, (dev, ino), mode and every directory listing, before and + after. +- Files that no read may touch are mode 000, so an attempted open would throw + `EACCES` rather than refuse. +- Every page and cursor is validated against + `docs/plans/chat-01/contracts.schema.json` with Python `jsonschema`. diff --git a/packages/conversation/package.json b/packages/conversation/package.json new file mode 100644 index 00000000..2e4bf4b9 --- /dev/null +++ b/packages/conversation/package.json @@ -0,0 +1,11 @@ +{ + "name": "@mosaic/conversation", + "version": "0.1.0", + "private": true, + "description": "Read-only Pi conversation histories for the Console: approved roots, safe opens, branch pages and cursors. No server; the control board serves it.", + "license": "UNLICENSED", + "type": "module", + "engines": { "node": ">=24" }, + "exports": { ".": "./src/reader.mjs" }, + "scripts": { "test": "node --test tests/" } +} diff --git a/packages/conversation/src/parts.mjs b/packages/conversation/src/parts.mjs new file mode 100644 index 00000000..f12846a6 --- /dev/null +++ b/packages/conversation/src/parts.mjs @@ -0,0 +1,124 @@ +// CHAT-01 history limits (#1507, CHAT-02): at most 100 parts per page, 8 MiB +// of serialized UTF-8 per page, 64 blocks per part and 262144 characters per +// string. Oversize content splits into fragments and continuation parts; it is +// never clipped. Byte limits are measured on the serialized JSON, not on +// characters. + +import { createHash } from "node:crypto"; + +export const LIMITS = Object.freeze({ parts: 100, pageBytes: 8 * 1024 * 1024, blocks: 64, chars: 262144 }); + +// Internal budgets that keep any single part well inside one page, so a page +// always holds at least one part. +export const FRAGMENT_BYTES = 1024 * 1024; +export const PART_BYTES = 4 * 1024 * 1024; + +export const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +// A native value used as a CHAT-01 id. Values that do not fit the id pattern +// (OpenAI tool calls such as "call_x|fc_y") map to a stable hash. +export function safeId(value) { + if (typeof value === "string" && ID.test(value)) return value; + return "h-" + createHash("sha256").update(String(value)).digest("hex").slice(0, 40); +} + +// Bytes one code point adds to a JSON string literal. +function jsonCost(cp) { + if (cp === 0x22 || cp === 0x5c) return 2; + if (cp < 0x20) return cp === 8 || cp === 9 || cp === 10 || cp === 12 || cp === 13 ? 2 : 6; + if (cp < 0x80) return 1; + if (cp < 0x800) return 2; + if (cp >= 0xd800 && cp <= 0xdfff) return 6; // lone surrogate, escaped by JSON.stringify + if (cp <= 0xffff) return 3; + return 4; +} + +// Splits a string into fragments of at most LIMITS.chars code points and +// FRAGMENT_BYTES of JSON. Never cuts a surrogate pair. "" is one fragment. +export function fragments(str) { + const out = []; + let start = 0, chars = 0, bytes = 0; + for (let i = 0; i < str.length;) { + const cp = str.codePointAt(i); + const width = cp > 0xffff ? 2 : 1; + const cost = jsonCost(cp); + if (chars > 0 && (chars + 1 > LIMITS.chars || bytes + cost > FRAGMENT_BYTES)) { + out.push(str.slice(start, i)); + start = i; + chars = 0; + bytes = 0; + } + chars += 1; + bytes += cost; + i += width; + } + out.push(str.slice(start)); + return out; +} + +const bytesOf = (value) => Buffer.byteLength(JSON.stringify(value), "utf8"); + +// One native unit (a message, a compaction, a notice) becomes one or more +// CHAT-01 entries. `unit.blocks` holds logical blocks: { fields, key, value }, +// where `value` is the string that may split and `key` names its field. A +// block with key null (an attachment) has no string and is one fragment. +export function unitParts(unit, base) { + const items = []; + unit.blocks.forEach((b, ordinal) => { + if (b.key === null) { + items.push({ ...b.fields, block: ordinal, fragment: 0, lastFragment: true }); + return; + } + const pieces = fragments(b.value); + pieces.forEach((piece, k) => { + items.push({ ...b.fields, [b.key]: piece, block: ordinal, fragment: k, lastFragment: k === pieces.length - 1 }); + }); + }); + const groups = []; + let current = [], currentBytes = 0; + for (const item of items) { + const size = bytesOf(item) + 1; + if (current.length && (current.length === LIMITS.blocks || currentBytes + size > PART_BYTES)) { + groups.push(current); + current = []; + currentBytes = 0; + } + current.push(item); + currentBytes += size; + } + groups.push(current); + return groups.map((content, part) => ({ + version: 2, + kind: "entry", + id: safeId(`${unit.id}:${part}`), + conversation: base.conversation, + branch: base.branch, + parent: unit.parent, + execution: base.execution, + nativeEntry: unit.nativeEntry, + role: unit.role, + request: null, + content, + part, + lastPart: part === groups.length - 1, + createdAt: unit.createdAt, + message: unit.message, + })); +} + +// Takes entries from `start` while the page stays within LIMITS. `shell` is +// the page record with an empty `entries` array. +export function takePage(entries, sizes, start, shell) { + let bytes = bytesOf(shell); + let end = start; + while (end < entries.length && end - start < LIMITS.parts) { + const add = sizes[end] + (end > start ? 1 : 0); + if (bytes + add > LIMITS.pageBytes) break; + bytes += add; + end += 1; + } + if (end === start && start < entries.length) throw new Error("internal: a single part exceeds the page byte limit"); + return end; +} + +export { bytesOf }; diff --git a/packages/conversation/src/pi.mjs b/packages/conversation/src/pi.mjs new file mode 100644 index 00000000..982a04a9 --- /dev/null +++ b/packages/conversation/src/pi.mjs @@ -0,0 +1,287 @@ +// Pi session parser for read-only histories (#1507, CHAT-02). Pinned against +// @earendil-works/pi-coding-agent 0.85.1 docs/session-format.md and +// dist/core/session-manager.js: +// +// - line 1 is the session header; every other line is an entry with id, +// parentId and timestamp, forming a tree; +// - on load Pi takes the leaf to be the last entry in file order and skips +// malformed lines (_buildIndex, parseSessionEntries). This parser uses the +// same default leaf and shows a malformed line as an unavailable notice; +// - resetLeaf starts a new root entry, so a file can hold several roots; +// - the full branch path is shown, including entries before a compaction; +// the compaction itself is a marker in place. retainedTail copies entries +// that are already on the path, so it is not rendered again. +// +// get_entries order is not a branch transcript and is not used, and +// parentSession is never followed or opened. + +import { Refusal } from "./safe-fs.mjs"; +import { safeId } from "./parts.mjs"; + +const toTime = (v) => { + const ms = typeof v === "number" ? v : typeof v === "string" ? Date.parse(v) : NaN; + if (!Number.isFinite(ms)) return null; + const iso = new Date(ms).toISOString(); + return /^\d{4}-/.test(iso) ? iso : null; +}; + +// Branch names. The first root's line of history is "main"; every later root +// (Pi's resetLeaf) and every later child at a fork starts a branch named after +// its first entry. Appending never changes which child came first, so a name +// holds for as long as the file only grows. +export const MAIN = "main"; +const branchName = (r) => safeId(`b.${r.entry.id}`); + +// `text` holds complete lines only (it ends with "\n" or is empty). +export function parseSnapshot(text) { + const lines = text.split("\n"); + lines.pop(); + let header; + try { + header = JSON.parse(lines[0] ?? ""); + } catch { + header = null; + } + if (!header || typeof header !== "object" || header.type !== "session") throw new Refusal("not-a-pi-session", "the first line is not a Pi session header"); + const entries = [], malformed = [], byId = new Map(), children = new Map(); + for (let i = 1; i < lines.length; i++) { + const line = lines[i]; + if (!line.trim()) continue; + let e; + try { + e = JSON.parse(line); + } catch { + e = null; + } + if (!e || typeof e !== "object" || Array.isArray(e) || typeof e.id !== "string" || typeof e.type !== "string" || e.type === "session") { + malformed.push({ line: i + 1, after: entries.length - 1 }); + continue; + } + const record = { entry: e, line: i + 1, index: entries.length }; + entries.push(record); + byId.set(e.id, record); // later wins, as in Pi's index + } + for (const r of byId.values()) { + const p = typeof r.entry.parentId === "string" ? r.entry.parentId : null; + if (!children.has(p)) children.set(p, []); + children.get(p).push(r); + } + for (const list of children.values()) list.sort((a, b) => a.index - b.index); + const parentOf = (r) => (typeof r.entry.parentId === "string" ? byId.get(r.entry.parentId) : undefined) ?? null; + const firstRoot = [...byId.values()].filter((r) => !parentOf(r)).sort((a, b) => a.index - b.index)[0] ?? null; + // Walk up while each entry is its parent's earliest child. The walk ends at + // a root or a later child, whose name the branch takes. A walk from a leaf + // always ends there; the loop guard only keeps a hostile file finite. + const branchOf = (leaf) => { + const seen = new Set(); + for (let r = leaf; !seen.has(r); ) { + seen.add(r); + const p = parentOf(r); + if (!p) return r === firstRoot ? MAIN : branchName(r); + if (children.get(p.entry.id)[0] !== r) return branchName(r); + r = p; + } + return branchName(leaf); + }; + const leaves = [...byId.values()].filter((r) => !children.has(r.entry.id)).sort((a, b) => a.index - b.index); + const defaultLeaf = entries.length ? byId.get(entries[entries.length - 1].entry.id) : null; + const defaultBranch = defaultLeaf ? branchOf(defaultLeaf) : MAIN; + // Branch name to leaf. The default branch ends at Pi's default leaf, and a + // file with no entries yet has an empty "main". + const branches = new Map(); + for (const leaf of leaves) { + const name = branchOf(leaf); + if (!branches.has(name)) branches.set(name, leaf); + } + if (defaultLeaf) branches.set(defaultBranch, defaultLeaf); + if (!branches.size) branches.set(MAIN, null); + return { header, entries, malformed, byId, branches, defaultLeaf, defaultBranch }; +} + +// Root-to-leaf items for one branch: { record } for native entries and +// { notice, lines? } for markers placed where they apply. A null leaf is a +// branch with no entries yet. +export function branchPath(parsed, leaf) { + const path = []; + const seen = new Set(); + const named = new Set(); + let r = leaf; + while (r) { + if (seen.has(r.entry.id)) { + path.push({ notice: "loop" }); + break; + } + seen.add(r.entry.id); + path.push({ record: r }); + const parentId = r.entry.parentId; + if (parentId === null || parentId === undefined) break; + const parent = typeof parentId === "string" ? parsed.byId.get(parentId) : null; + if (parent) { + r = parent; + continue; + } + // The parent is missing. Unreadable lines just before this entry may have + // held it; the notice names them. The history does not continue past the + // gap: the entries before it may belong to another branch. + const lost = parsed.malformed.filter((m) => m.after === r.index - 1).map((m) => m.line); + lost.forEach((line) => named.add(line)); + path.push({ notice: "missing-parent", lines: lost }); + break; + } + path.reverse(); + // Every other unreadable line is shown at its file position on every + // branch, after the leaf too: it may belong to any branch. Placing it the + // same way on every branch keeps a branch's earlier parts unchanged while + // the file grows. + const out = []; + const pending = parsed.malformed.filter((m) => !named.has(m.line)); + let mi = 0; + const flushBefore = (line) => { + while (mi < pending.length && pending[mi].line < line) out.push({ notice: "malformed", lines: [pending[mi++].line] }); + }; + for (const item of path) { + const line = item.record?.line ?? item.lines?.[0]; + if (line) flushBefore(line); + out.push(item); + } + flushBefore(Infinity); + return out; +} + +const NOTICE_TEXT = { + loop: "History before this point is unavailable: the entry chain loops.", + "missing-parent": "History before this point is unavailable: an earlier entry is missing from the file.", + "parent-session": "This session was forked from an earlier session. The earlier session is not opened here.", +}; + +function text(value) { + return { fields: { type: "text" }, key: "text", value: String(value) }; +} + +function contentText(content) { + if (typeof content === "string") return [text(content)]; + if (!Array.isArray(content)) return []; + return content.flatMap((b, i) => blockFor(b, i)); +} + +function blockFor(b, i, owner = "") { + if (!b || typeof b !== "object") return []; + if (b.type === "text") return [text(b.text ?? "")]; + if (b.type === "image") return [{ fields: { type: "attachment", attachment: safeId(`${owner}image.${i}`) }, key: null, value: null }]; + return [text(`[unsupported content block: ${safeId(String(b.type))}]`)]; +} + +// Unit ids are namespaced so no native id can collide with a notice: "n." +// for native entries, "x." for notices. Entry ids derive from them (parts.mjs). +// +// Converts one native entry to zero or more units. Entries that Pi keeps out +// of the transcript (model and thinking changes, labels, names, extension +// state) produce none. +function unitsFor(record, ctx) { + const e = record.entry; + const nativeEntry = safeId(e.id); + const createdAt = toTime(e.timestamp) ?? ctx.lastTime; + ctx.lastTime = createdAt; + const base = { id: `n.${e.id}`, nativeEntry, parent: typeof e.parentId === "string" ? safeId(e.parentId) : null, createdAt, message: nativeEntry }; + const notice = (value, suffix) => ({ ...base, id: `x.${suffix}.${e.id}`, message: safeId(`x.${suffix}.${e.id}`), role: "notice", blocks: [text(value)] }); + switch (e.type) { + case "message": + return messageUnits(e.message, base, notice); + case "compaction": + return [{ ...base, role: "compaction", blocks: [{ fields: { type: "compaction", nativeEntry }, key: "summary", value: String(e.summary ?? "") }] }]; + case "branch_summary": + return [{ ...base, role: "notice", blocks: [text(`Branch summary\n\n${e.summary ?? ""}`)] }]; + case "custom_message": + return e.display ? [{ ...base, role: "notice", blocks: contentText(e.content) }] : []; + case "model_change": + case "thinking_level_change": + case "label": + case "session_info": + case "custom": + return []; + default: + return [{ ...base, role: "notice", blocks: [text(`An entry of type ${safeId(e.type)} is not shown.`)] }]; + } +} + +function messageUnits(m, base, notice) { + if (!m || typeof m !== "object") return [{ ...base, role: "notice", blocks: [text("This entry has no readable message.")] }]; + const owner = `${base.nativeEntry}.`; + switch (m.role) { + case "user": + return [{ ...base, role: "user", blocks: typeof m.content === "string" ? [text(m.content)] : (Array.isArray(m.content) ? m.content : []).flatMap((b, i) => blockFor(b, i, owner)) }]; + case "assistant": { + const blocks = (Array.isArray(m.content) ? m.content : []).flatMap((b, i) => { + if (b?.type === "thinking") { + // Redacted reasoning is stored as a placeholder; it is unavailable, and + // thinkingSignature is never read. + const t = b.redacted === true ? "" : typeof b.thinking === "string" ? b.thinking : ""; + return [{ fields: { type: "thinking", visibility: t ? "permitted-visible" : "unavailable" }, key: "text", value: t }]; + } + if (b?.type === "toolCall") { + return [{ fields: { type: "tool-call", call: safeId(b.id), name: safeId(b.name) }, key: "argumentsText", value: JSON.stringify(b.arguments ?? {}) }]; + } + return blockFor(b, i, owner); + }); + const units = [{ ...base, role: "assistant", blocks }]; + if (m.stopReason === "error" || m.stopReason === "aborted") { + units.push(notice(m.errorMessage ? `The turn ended (${m.stopReason}): ${m.errorMessage}` : `The turn ended (${m.stopReason}).`, "end")); + } + return units; + } + case "toolResult": { + const content = Array.isArray(m.content) ? m.content : []; + const joined = content.filter((b) => b?.type === "text").map((b) => String(b.text ?? "")).join("\n"); + const blocks = [{ fields: { type: "tool-result", call: safeId(m.toolCallId), isError: m.isError === true }, key: "text", value: joined }]; + content.forEach((b, i) => { + if (b?.type === "image") blocks.push(...blockFor(b, i, owner)); + }); + return [{ ...base, role: "tool", blocks }]; + } + case "bashExecution": { + const call = safeId(`bash:${base.nativeEntry}`); + return [{ + ...base, + role: "tool", + blocks: [ + { fields: { type: "tool-call", call, name: "bash" }, key: "argumentsText", value: JSON.stringify({ command: String(m.command ?? "") }) }, + { fields: { type: "tool-result", call, isError: m.cancelled === true || (m.exitCode !== 0 && m.exitCode !== undefined) }, key: "text", value: String(m.output ?? "") }, + ], + }]; + } + case "custom": + return m.display ? [{ ...base, role: "notice", blocks: contentText(m.content) }] : []; + case "branchSummary": + return [{ ...base, role: "notice", blocks: [text(`Branch summary\n\n${m.summary ?? ""}`)] }]; + case "compactionSummary": + return [{ ...base, role: "compaction", blocks: [{ fields: { type: "compaction", nativeEntry: base.nativeEntry }, key: "summary", value: String(m.summary ?? "") }] }]; + default: + return [{ ...base, role: "notice", blocks: [text(`A message with role ${safeId(String(m.role))} is not shown.`)] }]; + } +} + +// All units for one branch, in order. +export function branchUnits(parsed, path) { + const ctx = { lastTime: toTime(parsed.header.timestamp) ?? new Date(0).toISOString() }; + const units = []; + if (parsed.header.parentSession !== undefined && parsed.header.parentSession !== null) { + units.push({ id: "x.parent-session", nativeEntry: "x.parent-session", parent: null, createdAt: ctx.lastTime, message: "x.parent-session", role: "notice", blocks: [text(NOTICE_TEXT["parent-session"])] }); + } + for (const item of path) { + if (item.record) { + units.push(...unitsFor(item.record, ctx)); + continue; + } + const lines = item.lines ?? []; + const id = item.notice === "malformed" ? `x.line-${lines[0]}` : `x.${item.notice}`; + const value = item.notice === "malformed" + ? `Line ${lines[0]} could not be read. It may belong to this branch or another one.` + : item.notice === "missing-parent" && lines.length + ? `${NOTICE_TEXT["missing-parent"]} ${lines.length === 1 ? `Line ${lines[0]} could not be read and may have held it.` : `Lines ${lines.join(", ")} could not be read and may have held it.`}` + : NOTICE_TEXT[item.notice]; + units.push({ id, nativeEntry: id, parent: null, createdAt: ctx.lastTime, message: id, role: "notice", blocks: [text(value)] }); + } + return units; +} + +export { toTime }; diff --git a/packages/conversation/src/reader.mjs b/packages/conversation/src/reader.mjs new file mode 100644 index 00000000..ed266d2d --- /dev/null +++ b/packages/conversation/src/reader.mjs @@ -0,0 +1,401 @@ +// Read-only Pi conversation histories for the Console (#1507, CHAT-02). +// +// A library with no server: the control board owns the two GET routes (D3). +// Only approved roots are read, /.pi/state//sessions from +// the board's repository specs. There is no global scan, and no path comes +// from the caller: a conversation is an opaque id that is resolved by listing +// the roots again. Opening a conversation never resumes, forks, launches or +// controls anything, and nothing here writes a file. +// +// A seat registration is seat-written, so it is a hint, not authority. It can +// narrow a root (a non-Pi harness makes the root unsupported, D2) and supply +// the engine launch time. It never adds a root or names a file. + +import { createHash, randomBytes } from "node:crypto"; +import { realpathSync } from "node:fs"; +import { join, resolve, basename, sep, isAbsolute } from "node:path"; +import { Refusal, listSessionFiles, openSessionFile, readRange, closeSync } from "./safe-fs.mjs"; +import { parseSnapshot, branchPath, branchUnits, toTime } from "./pi.mjs"; +import { unitParts, takePage, bytesOf, safeId } from "./parts.mjs"; +import { samePath } from "../../seat/src/seat.mjs"; + +export const ACTOR = "local-operator"; +export const HISTORY = "pi"; +export const UNSUPPORTED_HARNESS = "unsupported-harness"; +const NO_STREAM = "no-stream"; +const HEAD_BYTES = 1024 * 1024; +const TAIL_BYTES = 2 * 1024 * 1024; +const TITLE_CHARS = 200; +export const MAX_FILE_BYTES = 256 * 1024 * 1024; + +const sha256 = (data) => createHash("sha256").update(data).digest("hex"); + +// Approved roots from the board's specs. Only repository specs qualify: +// sessionsDir must be exactly /.pi/state//sessions and the +// project must be the root's directory name. Fleet and connector specs do not. +export function rootsFromSpecs(specs, registrations = []) { + const roots = []; + for (const spec of specs) { + if (!spec || spec.connector || typeof spec.sessionsDir !== "string" || typeof spec.agent !== "string") continue; + const dir = resolve(spec.sessionsDir); + const projectRoot = resolve(dir, "..", "..", "..", ".."); + if (join(projectRoot, ".pi", "state", spec.agent, "sessions") !== dir || basename(projectRoot) !== spec.project) continue; + const reg = registrations.find((r) => r && r.seat === spec.agent && r.layout === "repo" && samePath(r.sessionsDir, dir) && (r.project === null || r.project === spec.project)) ?? null; + const harness = reg?.harness ?? null; + roots.push({ + seat: spec.agent, + project: spec.project, + projectRoot, + dir, + harness: harness ?? HISTORY, + unsupportedReason: harness !== null && harness !== HISTORY ? UNSUPPORTED_HARNESS : null, + engineStartedAt: toTime(reg?.startedAt) ?? null, + }); + } + // A seat on another harness has no Pi sessions directory, so no spec. Its + // registration adds a placeholder root, never read, when it names the + // standard directory under a project root already approved above (D2). + const projects = new Map(roots.map((r) => [r.projectRoot, r.project])); + for (const reg of registrations) { + if (!reg || reg.layout !== "repo" || typeof reg.harness !== "string" || reg.harness === HISTORY) continue; + if (roots.some((r) => r.seat === reg.seat)) continue; + for (const [projectRoot, project] of projects) { + const dir = join(projectRoot, ".pi", "state", reg.seat, "sessions"); + if ((reg.project !== null && reg.project !== project) || !samePath(reg.sessionsDir, dir)) continue; + roots.push({ seat: reg.seat, project, projectRoot, dir, harness: reg.harness, unsupportedReason: UNSUPPORTED_HARNESS, engineStartedAt: toTime(reg.startedAt) ?? null }); + break; + } + } + return roots; +} + +export function conversationId(root, name) { + return "pi-" + sha256(`${root.projectRoot}\0${root.seat}\0${name}`).slice(0, 32); +} + +function unsupportedId(root) { + return "unsupported-" + sha256(`${root.projectRoot}\0${root.seat}`).slice(0, 32); +} + +// The header's cwd must be an absolute path to the project or inside it. Pi +// writes absolute paths; a relative one would resolve against the board's own +// directory, so it is refused. Real paths are compared first (the checkout is +// reachable through a compatibility symlink); a cwd that no longer exists is +// compared as written. +export function cwdInProject(cwd, projectRoot) { + if (typeof cwd !== "string" || !isAbsolute(cwd)) return false; + const inside = (a, b) => a === b || a.startsWith(b.endsWith(sep) ? b : b + sep); + if (inside(resolve(cwd), resolve(projectRoot))) return true; + try { + return inside(realpathSync(cwd), realpathSync(projectRoot)); + } catch { + return false; + } +} + +const firstChars = (s, n) => { + const cps = Array.from(String(s).replace(/\s+/g, " ").trim()); + return cps.length > n ? cps.slice(0, n).join("") + "…" : cps.join(""); +}; + +function userText(m) { + if (!m || m.role !== "user") return null; + if (typeof m.content === "string") return m.content; + if (Array.isArray(m.content)) { + const t = m.content.find((b) => b?.type === "text" && typeof b.text === "string" && b.text.trim()); + return t ? t.text : null; + } + return null; +} + +const parseLine = (line) => { + try { + const v = JSON.parse(line); + return v && typeof v === "object" && !Array.isArray(v) ? v : null; + } catch { + return null; + } +}; + +// A catalogue row. This is a CHAT-02 summary, not a CHAT-01 catalogueItem: +// that record needs an execution-bound currentView, which D1 moved to CHAT-03. +function baseRow(root, conversation) { + return { + conversation, + seat: root.seat, + project: root.project, + harness: root.harness, + history: root.unsupportedReason ? null : HISTORY, + title: null, + readOnly: true, + controlMode: "unavailable", + conversationCreatedAt: null, + engineStartedAt: null, + lastActivityAt: null, + availability: "available", + unsupportedReason: root.unsupportedReason, + refusal: null, + }; +} + +// One catalogue row from a head and a tail window; the whole file is not read. +function summarize(root, name) { + const row = baseRow(root, conversationId(root, name)); + let file; + try { + file = openSessionFile(root, name); + } catch (err) { + if (!(err instanceof Refusal)) throw err; + return { ...row, availability: "denied", refusal: err.code }; + } + try { + const head = readRange(file.fd, 0, Math.min(file.size, HEAD_BYTES)).toString("utf8"); + const headLines = head.split("\n"); + if (headLines.length < 2) return { ...row, availability: "unavailable", refusal: "incomplete-header" }; + const header = parseLine(headLines[0]); + if (!header || header.type !== "session") return { ...row, availability: "unavailable", refusal: "not-a-pi-session" }; + if (!cwdInProject(header.cwd, root.projectRoot)) return { ...row, availability: "denied", refusal: "foreign-project" }; + row.conversationCreatedAt = toTime(header.timestamp); + if (row.conversationCreatedAt && root.engineStartedAt && row.conversationCreatedAt >= root.engineStartedAt) row.engineStartedAt = root.engineStartedAt; + let name_ = null, firstUser = null; + for (const line of headLines.slice(1, -1)) { + const e = parseLine(line); + if (e?.type === "session_info" && typeof e.name === "string" && e.name.trim()) name_ = e.name; + if (firstUser === null && e?.type === "message") firstUser = userText(e.message); + } + const tailStart = Math.max(0, file.size - TAIL_BYTES); + const tail = readRange(file.fd, tailStart, file.size - tailStart).toString("utf8"); + const complete = tail.slice(0, tail.lastIndexOf("\n") + 1).split("\n").slice(tailStart > 0 ? 1 : 0, -1); + let lastName = null; + for (let i = complete.length - 1; i >= 0; i--) { + const e = parseLine(complete[i]); + if (!e || e.type === "session" || typeof e.id !== "string") continue; + if (row.lastActivityAt === null) row.lastActivityAt = toTime(e.timestamp); + if (lastName === null && e.type === "session_info" && typeof e.name === "string" && e.name.trim()) lastName = e.name; + if (row.lastActivityAt !== null && lastName !== null) break; + } + const title = lastName ?? name_ ?? firstUser; + row.title = title ? firstChars(title, TITLE_CHARS) : null; + return row; + } finally { + closeSync(file.fd); + } +} + +// Complete lines only: the snapshot ends at the last "\n" present when the +// descriptor was read, so growth during the read is cut there. +function readSnapshot(root, name, pinnedLength = null) { + const file = openSessionFile(root, name); + try { + if (file.size > MAX_FILE_BYTES) throw new Refusal("too-large", "session file is larger than the reader accepts"); + if (pinnedLength !== null) { + if (file.size < pinnedLength) return { file, shorter: true }; + const buf = readRange(file.fd, 0, pinnedLength); + return { file, buf, length: pinnedLength }; + } + const buf = readRange(file.fd, 0, file.size); + const length = buf.lastIndexOf(0x0a) + 1; + return { file, buf: buf.subarray(0, length), length, incomplete: buf.length > length }; + } finally { + closeSync(file.fd); + } +} + +export function createReader({ roots, now = () => Date.now(), ttlMs = 10 * 60 * 1000, maxCursors = 1000, cacheSize = 2 } = {}) { + const listRoots = typeof roots === "function" ? roots : () => roots ?? []; + const cursors = new Map(); + const parsedCache = new Map(); + const partsCache = new Map(); + + const remember = (cache, key, make) => { + if (cache.has(key)) { + const v = cache.get(key); + cache.delete(key); + cache.set(key, v); + return v; + } + const v = make(); + cache.set(key, v); + while (cache.size > cacheSize) cache.delete(cache.keys().next().value); + return v; + }; + + function resolveConversation(conversation) { + if (typeof conversation !== "string") return null; + for (const root of listRoots()) { + if (root.unsupportedReason) { + if (unsupportedId(root) === conversation) return { root, name: null }; + continue; + } + let listing; + try { + listing = listSessionFiles(root); + } catch (err) { + if (err instanceof Refusal) continue; + throw err; + } + for (const name of [...listing.files, ...listing.refused.map((r) => r.name)]) { + if (conversationId(root, name) === conversation) return { root, name }; + } + } + return null; + } + + function catalogue() { + const conversations = [], refusedRoots = []; + for (const root of listRoots()) { + if (root.unsupportedReason) { + conversations.push({ ...baseRow(root, unsupportedId(root)), engineStartedAt: root.engineStartedAt, availability: "unsupported" }); + continue; + } + let listing; + try { + listing = listSessionFiles(root); + } catch (err) { + if (!(err instanceof Refusal)) throw err; + refusedRoots.push({ seat: root.seat, project: root.project, refusal: err.code }); + continue; + } + for (const name of listing.files) conversations.push(summarize(root, name)); + for (const r of listing.refused) conversations.push({ ...baseRow(root, conversationId(root, r.name)), availability: "denied", refusal: r.code }); + } + conversations.sort((a, b) => (b.lastActivityAt ?? "").localeCompare(a.lastActivityAt ?? "") || a.conversation.localeCompare(b.conversation)); + return { ok: true, conversations, refusedRoots, generatedAt: new Date(now()).toISOString() }; + } + + // Parsed snapshot plus one branch's entries, cached by content. A null + // branch is the default one; an unknown branch gives null. + function build(snap, root, branchName, conversation) { + const key = `${snap.file.dev}:${snap.file.ino}:${snap.length}:${snap.digest}`; + const parsed = remember(parsedCache, key, () => parseSnapshot(snap.buf.toString("utf8"))); + if (!cwdInProject(parsed.header.cwd, root.projectRoot)) throw new Refusal("foreign-project", "the session belongs to another project"); + const branch = branchName ?? parsed.defaultBranch; + if (!parsed.branches.has(branch)) return null; + const leaf = parsed.branches.get(branch); + const { entries, sizes } = remember(partsCache, `${key}:${conversation}:${branch}`, () => { + const base = { conversation, branch, execution: safeId(parsed.header.id) }; + const entries = branchUnits(parsed, branchPath(parsed, leaf)).flatMap((u) => unitParts(u, base)); + return { entries, sizes: entries.map(bytesOf) }; + }); + return { parsed, leaf, branch, entries, sizes }; + } + + const idsDigest = (entries, end) => sha256(entries.slice(0, end).map((e) => e.id).join("\n")); + + function issue(state) { + while (cursors.size >= maxCursors) cursors.delete(cursors.keys().next().value); + const id = "c-" + randomBytes(16).toString("hex"); + const expiresAt = new Date(now() + ttlMs).toISOString(); + const record = { + version: 2, kind: "cursor", id, conversation: state.conversation, branch: state.branch, snapshotDigest: state.digest, + sourceEpoch: state.epoch, lastEntry: state.lastEntry, expiresAt, actor: state.actor, purpose: state.purpose, + }; + cursors.set(id, { record, state }); + return record; + } + + // One page from `offset`, with a next cursor when more parts remain in this + // snapshot and a follow cursor when the page reaches its end. + function pageFrom({ root, name, snap, built, conversation, offset, epoch, actor, purpose, incomplete }) { + const { entries, sizes } = built; + const shell = { + version: 2, kind: "page", conversation, branch: built.branch, snapshotDigest: snap.digest, sourceEpoch: epoch, + entries: [], nextCursor: "c-" + "0".repeat(32), hasMore: true, readOnly: true, streamEpoch: NO_STREAM, throughSequence: 0, + }; + const end = takePage(entries, sizes, offset, shell); + const hasMore = end < entries.length; + const state = { + root, name, dev: snap.file.dev, ino: snap.file.ino, length: snap.length, digest: snap.digest, epoch, + conversation, branch: built.branch, + offset: end, prefix: idsDigest(entries, end), lastEntry: end > 0 ? entries[end - 1].id : null, actor, purpose, incomplete, + }; + const cursor = hasMore ? issue({ ...state, follow: false }) : null; + const follow = hasMore ? null : issue({ ...state, follow: true }); + const page = { ...shell, entries: entries.slice(offset, end), nextCursor: cursor ? cursor.id : null, hasMore }; + const parsed = built.parsed; + const view = { + conversation, + branch: built.branch, + defaultBranch: parsed.defaultBranch, + branches: [...parsed.branches] + .sort(([, a], [, b]) => (a?.index ?? -1) - (b?.index ?? -1)) + .map(([branch, leaf]) => ({ branch, isDefault: branch === parsed.defaultBranch, lastActivityAt: leaf ? toTime(leaf.entry.timestamp) : null })), + incomplete, + forkedFromEarlierSession: parsed.header.parentSession !== undefined && parsed.header.parentSession !== null, + unreadableLines: parsed.malformed.length, + }; + return { ok: true, page, cursor, follow, view }; + } + + function refuse(err) { + if (err instanceof Refusal) return { ok: false, refusal: { code: err.code, reconcile: err.reconcile, message: err.message } }; + throw err; + } + + function checkCaller(actor, purpose) { + if (actor !== ACTOR) throw new Refusal("unknown-actor", "only the local operator reads histories on this route"); + if (purpose !== "history") throw new Refusal("unsupported-purpose", "only history pages are served"); + } + + function snapshot(root, name, pinnedLength = null) { + const snap = readSnapshot(root, name, pinnedLength); + if (snap.shorter) return snap; + if (snap.length === 0) throw new Refusal("incomplete-header", "the session header is not complete yet", { reconcile: true }); + return { ...snap, digest: sha256(snap.buf) }; + } + + function open({ conversation, branch = null, actor = ACTOR, purpose = "history" } = {}) { + try { + checkCaller(actor, purpose); + const found = resolveConversation(conversation); + if (!found) throw new Refusal("unknown-conversation", "no such conversation in the approved roots", { reconcile: true }); + if (found.root.unsupportedReason) throw new Refusal(found.root.unsupportedReason, "this harness has no history reader yet"); + const snap = snapshot(found.root, found.name); + const built = build(snap, found.root, branch ?? null, conversation); + if (!built) throw new Refusal("unknown-branch", "no such branch in this conversation", { reconcile: true }); + const epoch = "e-" + sha256(`${snap.file.dev}:${snap.file.ino}:${snap.digest}`).slice(0, 40); + return pageFrom({ root: found.root, name: found.name, snap, built, conversation, offset: 0, epoch, actor, purpose, incomplete: snap.incomplete }); + } catch (err) { + return refuse(err); + } + } + + function next({ cursor, conversation, branch, actor = ACTOR, purpose = "history" } = {}) { + try { + const held = typeof cursor === "string" ? cursors.get(cursor) : undefined; + if (!held) throw new Refusal("cursor-unknown", "the cursor is unknown", { reconcile: true }); + const { record, state } = held; + if (Date.parse(record.expiresAt) <= now()) { + cursors.delete(cursor); + throw new Refusal("cursor-expired", "the cursor has expired", { reconcile: true }); + } + if (actor !== record.actor || purpose !== record.purpose || conversation !== record.conversation || branch !== record.branch) { + throw new Refusal("cursor-foreign", "the cursor belongs to another view", { reconcile: true }); + } + // The source must still be the snapshot's file with the same prefix. + const pinned = snapshot(state.root, state.name, state.length); + if (pinned.shorter) throw new Refusal("source-replaced", "the session file is shorter than the snapshot", { reconcile: true }); + if (pinned.file.dev !== state.dev || pinned.file.ino !== state.ino) throw new Refusal("source-replaced", "the session file was replaced", { reconcile: true }); + if (pinned.digest !== state.digest) throw new Refusal("source-replaced", "the session file was rewritten", { reconcile: true }); + const args = { root: state.root, name: state.name, conversation, epoch: state.epoch, actor, purpose }; + if (!state.follow) { + const built = build(pinned, state.root, state.branch, conversation); + return pageFrom({ ...args, snap: pinned, built, offset: state.offset, incomplete: state.incomplete }); + } + // Follow: a fresh snapshot that extends the verified prefix. The view + // stays on its branch; view.defaultBranch shows where Pi's default is. + const fresh = snapshot(state.root, state.name); + if (fresh.file.dev !== state.dev || fresh.file.ino !== state.ino || fresh.length < state.length) throw new Refusal("source-replaced", "the session file was replaced", { reconcile: true }); + if (sha256(fresh.buf.subarray(0, state.length)) !== state.digest) throw new Refusal("source-replaced", "the session file was rewritten", { reconcile: true }); + const built = build(fresh, state.root, state.branch, conversation); + if (!built || built.entries.length < state.offset || idsDigest(built.entries, state.offset) !== state.prefix) { + throw new Refusal("source-replaced", "the history before this point changed", { reconcile: true }); + } + return pageFrom({ ...args, snap: fresh, built, offset: state.offset, incomplete: fresh.incomplete }); + } catch (err) { + return refuse(err); + } + } + + return { catalogue, open, next, cursorCount: () => cursors.size }; +} diff --git a/packages/conversation/src/safe-fs.mjs b/packages/conversation/src/safe-fs.mjs new file mode 100644 index 00000000..582e964f --- /dev/null +++ b/packages/conversation/src/safe-fs.mjs @@ -0,0 +1,123 @@ +// Read-only file access for approved Pi session roots (#1507, CHAT-02). +// +// A root is /.pi/state//sessions. The project root comes +// from the board's own configuration and is trusted as given (it may itself +// be a symlink, like the compatibility path to this checkout). Every +// component below it must be a real directory, never a symlink, and a +// session file must be a regular file directly inside the root. +// +// A file is opened O_RDONLY | O_NOFOLLOW | O_NONBLOCK, and the descriptor's +// (dev, ino) must match the lstat taken before the open. Node has no openat, +// so a directory component swapped between the checks and the open is caught +// by re-checking the components after the open, not prevented outright. The +// seat that owns a root can write there anyway; the checks keep anything +// outside the root from being read through it. +// +// Nothing here writes, renames, creates or migrates a file. + +import { lstatSync, openSync, fstatSync, readSync, closeSync, readdirSync, constants } from "node:fs"; +import { join, relative, isAbsolute, sep, basename } from "node:path"; + +export class Refusal extends Error { + constructor(code, message, { reconcile = false } = {}) { + super(message); + this.code = code; + this.reconcile = reconcile; + } +} + +const SESSION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]*\.jsonl$/; + +// Permission errors inside a root are one conversation's problem, not the +// catalogue's: they become a refusal instead of a thrown error. +function denied(err, what) { + if (err.code === "EACCES" || err.code === "EPERM") return new Refusal("unreadable", `${what} is not readable`); + return err; +} + +// A directory above the file without search permission is refused the same +// way, so one bad root does not fail the catalogue. +function lstatOrNull(path) { + try { + return lstatSync(path, { bigint: true }); + } catch (err) { + if (err.code === "ENOENT" || err.code === "ENOTDIR") return null; + throw denied(err, "a session path component"); + } +} + +// Every component from the project root down to the sessions directory must +// be a real directory. +export function checkRoot(root) { + const rel = relative(root.projectRoot, root.dir); + if (!rel || rel.startsWith("..") || isAbsolute(rel)) throw new Refusal("unsafe-path", "session root is outside its project"); + let path = root.projectRoot; + for (const part of rel.split(sep)) { + path = join(path, part); + const st = lstatOrNull(path); + if (!st) throw new Refusal("unavailable", "session root does not exist"); + if (st.isSymbolicLink()) throw new Refusal("unsafe-path", "session root contains a symlink"); + if (!st.isDirectory()) throw new Refusal("unsafe-path", "session root is not a directory"); + } +} + +// Session files directly inside a root, by name. Symlinks and anything that +// is not a regular *.jsonl file are reported, never followed. +export function listSessionFiles(root) { + checkRoot(root); + const files = [], refused = []; + let dirents; + try { + dirents = readdirSync(root.dir, { withFileTypes: true }); + } catch (err) { + throw denied(err, "session root"); + } + for (const dirent of dirents) { + if (!dirent.name.endsWith(".jsonl")) continue; + if (!SESSION_NAME.test(dirent.name)) refused.push({ name: dirent.name, code: "unsafe-path" }); + else if (dirent.isSymbolicLink()) refused.push({ name: dirent.name, code: "unsafe-path" }); + else if (dirent.isFile()) files.push(dirent.name); + } + return { files: files.sort(), refused }; +} + +// Opens one session file read-only. The caller must close the returned fd. +export function openSessionFile(root, name) { + if (typeof name !== "string" || name !== basename(name) || !SESSION_NAME.test(name)) throw new Refusal("unsafe-path", "not a session file name"); + checkRoot(root); + const path = join(root.dir, name); + const before = lstatOrNull(path); + if (!before) throw new Refusal("unknown-conversation", "session file no longer exists", { reconcile: true }); + if (before.isSymbolicLink()) throw new Refusal("unsafe-path", "session file is a symlink"); + if (!before.isFile()) throw new Refusal("unsafe-path", "session file is not a regular file"); + let fd; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + } catch (err) { + if (err.code === "ELOOP") throw new Refusal("unsafe-path", "session file became a symlink"); + if (err.code === "ENOENT") throw new Refusal("unknown-conversation", "session file no longer exists", { reconcile: true }); + throw denied(err, "session file"); + } + try { + const st = fstatSync(fd, { bigint: true }); + if (!st.isFile() || st.dev !== before.dev || st.ino !== before.ino) throw new Refusal("unsafe-path", "session file changed while it was opened"); + checkRoot(root); + return { fd, dev: st.dev.toString(), ino: st.ino.toString(), size: Number(st.size) }; + } catch (err) { + closeSync(fd); + throw err; + } +} + +export function readRange(fd, start, length) { + const buf = Buffer.alloc(length); + let done = 0; + while (done < length) { + const n = readSync(fd, buf, done, length - done, start + done); + if (n === 0) break; + done += n; + } + return done === length ? buf : buf.subarray(0, done); +} + +export { closeSync }; diff --git a/packages/conversation/tests/reader.test.mjs b/packages/conversation/tests/reader.test.mjs new file mode 100644 index 00000000..198ca7e7 --- /dev/null +++ b/packages/conversation/tests/reader.test.mjs @@ -0,0 +1,773 @@ +// packages/conversation (#1507, CHAT-02): fixtures F1–F15 and F17 from the +// brief (agents/dewey/work/chat-02/BRIEF.md §2.1). F16 is in the control-board +// suite, which serves these routes. Every reader call runs inside noWrites(), +// which fingerprints the fixture tree before and after (F17). Every page and +// cursor is checked against the CHAT-01 schema at the end. + +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, readFileSync, readdirSync, rmSync, chmodSync, symlinkSync, lstatSync, renameSync, openSync, writeSync, closeSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve, relative } from "node:path"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { rootsFromSpecs, createReader, conversationId, cwdInProject, UNSUPPORTED_HARNESS } from "../src/reader.mjs"; +import { fragments, safeId, LIMITS } from "../src/parts.mjs"; + +const repoRoot = resolve(import.meta.dirname, "..", "..", ".."); +const schemaPath = join(repoRoot, "docs", "plans", "chat-01", "contracts.schema.json"); +const tmp = mkdtempSync(join(tmpdir(), "conversation-")); +const records = []; +after(() => { + // chmod 000 fixtures would block removal + spawnSync("chmod", ["-R", "u+rwx", tmp]); + rmSync(tmp, { recursive: true, force: true }); +}); + +let fixtureCount = 0; +const time = (i) => new Date(Date.UTC(2026, 8, 26, 12, 0, 0) + i * 1000).toISOString(); + +// A project with one seat's sessions directory. +function project(seat = "pi-seat") { + const base = join(tmp, `f${++fixtureCount}`); + const proj = join(base, "proj"); + const dir = join(proj, ".pi", "state", seat, "sessions"); + mkdirSync(dir, { recursive: true }); + return { base, proj, dir, seat, spec: { agent: seat, project: "proj", sessionsDir: dir } }; +} + +const header = (proj, extra = {}) => ({ type: "session", version: 3, id: "0f5e1c2a-1111-4222-8333-944455556666", timestamp: time(0), cwd: proj, ...extra }); +const msg = (id, parentId, role, content, extra = {}, i = 1) => ({ type: "message", id, parentId, timestamp: time(i), message: { role, content, ...extra } }); +const user = (id, parentId, text, i) => msg(id, parentId, "user", [{ type: "text", text }], {}, i); +const assistant = (id, parentId, text, i) => msg(id, parentId, "assistant", [{ type: "text", text }], { stopReason: "stop" }, i); +const lines = (...values) => values.map((v) => (typeof v === "string" ? v : JSON.stringify(v)) + "\n").join(""); + +function writeSession(dir, name, head, entries, trailing = "") { + const path = join(dir, name); + writeFileSync(path, lines(head, ...entries) + trailing); + return path; +} + +// Fingerprint of every file and directory under `root`, symlinks not followed. +function fingerprint(root) { + const out = {}; + const walk = (path) => { + const st = lstatSync(path, { bigint: true }); + const rec = { dev: String(st.dev), ino: String(st.ino), mtime: String(st.mtimeNs), size: String(st.size), mode: String(st.mode) }; + if (st.isDirectory()) { + rec.list = st.mode & 0o400n ? readdirSync(path).sort() : null; + for (const n of rec.list ?? []) walk(join(path, n)); + } else if (st.isFile() && st.mode & 0o400n) { + rec.sha256 = createHash("sha256").update(readFileSync(path)).digest("hex"); + } + out[path] = rec; + }; + walk(root); + return out; +} + +// F17: a reader operation leaves every file and listing in `root` unchanged. +function noWrites(root, fn) { + const before = fingerprint(root); + const result = fn(); + assert.deepEqual(fingerprint(root), before, "a reader operation changed the fixture tree"); + for (const key of ["page", "cursor", "follow"]) if (result?.[key]) records.push(result[key]); + return result; +} + +function readerFor(fx, opts = {}) { + return createReader({ roots: rootsFromSpecs([fx.spec], opts.registrations ?? []), ...opts }); +} + +function openOne(fx, reader, name = "s.jsonl", args = {}) { + return noWrites(fx.base, () => reader.open({ conversation: conversationId(rootsFromSpecs([fx.spec])[0], name), ...args })); +} + +// Every page from an open, following next cursors. +function readAll(fx, reader, first) { + const pages = [first]; + let r = first; + while (r.ok && r.page.hasMore) { + const page = r.page; + r = noWrites(fx.base, () => reader.next({ cursor: page.nextCursor, conversation: page.conversation, branch: page.branch })); + assert.equal(r.ok, true, JSON.stringify(r.refusal)); + pages.push(r); + } + return pages; +} + +const texts = (page) => page.entries.map((e) => e.content.map((b) => b.text ?? b.summary ?? b.argumentsText ?? "").join("")); +const roles = (page) => page.entries.map((e) => e.role); + +test("a plain conversation: catalogue row, one page, CHAT-01 records", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("a1", null, "hello", 1), + { type: "session_info", id: "a2", parentId: "a1", timestamp: time(2), name: "Greeting" }, + assistant("a3", "a2", "hi there", 3), + { type: "model_change", id: "a4", parentId: "a3", timestamp: time(4), provider: "x", modelId: "y" }, + ]); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + assert.equal(cat.conversations.length, 1); + const row = cat.conversations[0]; + assert.equal(row.title, "Greeting"); + assert.equal(row.availability, "available"); + assert.equal(row.readOnly, true); + assert.equal(row.conversationCreatedAt, time(0)); + assert.equal(row.lastActivityAt, time(4)); + assert.equal(row.engineStartedAt, null); + const r = openOne(fx, reader); + assert.equal(r.ok, true); + assert.deepEqual(roles(r.page), ["user", "assistant"]); + assert.deepEqual(texts(r.page), ["hello", "hi there"]); + assert.equal(r.page.hasMore, false); + assert.equal(r.page.nextCursor, null); + assert.equal(r.cursor, null); + assert.ok(r.follow, "the last page carries a follow cursor"); + assert.equal(r.page.branch, "main"); + assert.equal(r.view.defaultBranch, "main"); + assert.equal(r.page.entries[1].parent, "a2"); + assert.equal(r.page.readOnly, true); +}); + +test("native entries map to blocks: tools, thinking, bash, notices, ids that do not fit", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("b1", null, "run it", 1), + msg("b2", "b1", "assistant", [ + { type: "thinking", thinking: "plan" }, + { type: "thinking", thinking: "" }, + { type: "toolCall", id: "call_x|fc_y", name: "bash", arguments: { command: "ls" } }, + { type: "thinking", thinking: "[Reasoning redacted]", redacted: true, thinkingSignature: "opaque" }, + ], { stopReason: "toolUse" }, 2), + msg("b3", "b2", "toolResult", [{ type: "text", text: "a" }, { type: "image", data: "AAAA", mimeType: "image/png" }, { type: "text", text: "b" }], { toolCallId: "call_x|fc_y", toolName: "bash", isError: false }, 3), + { type: "message", id: "b4", parentId: "b3", timestamp: time(4), message: { role: "bashExecution", command: "pwd", output: "/p", exitCode: 1, cancelled: false } }, + { type: "custom_message", id: "b5", parentId: "b4", timestamp: time(5), customType: "x", content: "shown", display: true }, + { type: "custom_message", id: "b6", parentId: "b5", timestamp: time(6), customType: "x", content: "hidden", display: false }, + msg("b7", "b6", "assistant", [{ type: "text", text: "partial" }], { stopReason: "error", errorMessage: "overloaded" }, 7), + { type: "future_kind", id: "b8", parentId: "b7", timestamp: time(8) }, + ]); + const r = openOne(fx, readerFor(fx)); + assert.equal(r.ok, true); + assert.deepEqual(roles(r.page), ["user", "assistant", "tool", "tool", "notice", "assistant", "notice", "notice"]); + const [, a, tool, bash, shown, , end, unknown] = r.page.entries; + assert.deepEqual(a.content.map((b) => [b.type, b.visibility ?? null]), [["thinking", "permitted-visible"], ["thinking", "unavailable"], ["tool-call", null], ["thinking", "unavailable"]]); + assert.equal(a.content[3].text, "", "redacted reasoning shows no placeholder"); + assert.equal(a.content[2].call, safeId("call_x|fc_y")); + assert.match(a.content[2].call, /^h-[0-9a-f]{40}$/); + assert.equal(a.content[2].argumentsText, '{"command":"ls"}'); + assert.deepEqual(tool.content.map((b) => b.type), ["tool-result", "attachment"]); + assert.equal(tool.content[0].text, "a\nb"); + assert.equal(tool.content[0].call, a.content[2].call); + assert.deepEqual(bash.content.map((b) => b.type), ["tool-call", "tool-result"]); + assert.equal(bash.content[1].isError, true); + assert.equal(shown.content[0].text, "shown"); + assert.match(end.content[0].text, /overloaded/); + assert.match(unknown.content[0].text, /future_kind/); + assert.equal(new Set(r.page.entries.map((e) => e.id)).size, r.page.entries.length, "entry ids are unique"); +}); + +test("F1: a malformed line is an unavailable part at its position, and reading continues", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("c1", null, "one", 1), + assistant("c2", "c1", "two", 2), + '{"type":"message","id":"c3", broken', + user("c4", "c2", "four", 4), + "[1,2,3]", + assistant("x.line-6", "c4", "five", 5), // a native id shaped like a notice id + ]); + const r = openOne(fx, readerFor(fx)); + assert.equal(r.ok, true); + assert.deepEqual(roles(r.page), ["user", "assistant", "notice", "user", "notice", "assistant"]); + assert.equal(new Set(r.page.entries.map((e) => e.id)).size, 6, "native ids never collide with notice ids"); + assert.match(texts(r.page)[2], /^Line 4 could not be read/); + assert.match(texts(r.page)[4], /^Line 6 could not be read/); + assert.equal(r.view.unreadableLines, 2); + const cat = noWrites(fx.base, () => readerFor(fx).catalogue()); + assert.equal(cat.conversations[0].availability, "available"); +}); + +test("F1: a missing parent stops the history with a notice that names the unreadable lines", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("d1", null, "one", 1), + "garbage", + assistant("d3", "d2", "three", 3), + ]); + const reader = readerFor(fx); + const r = openOne(fx, reader); + assert.deepEqual([roles(r.page), r.page.branch], [["notice", "assistant"], "b.d3"]); + assert.match(texts(r.page)[0], /missing from the file\. Line 3 could not be read and may have held it\.$/); + // The history before the gap is its own branch, and the line shows there too. + const main = openOne(fx, reader, "s.jsonl", { branch: "main" }); + assert.deepEqual([roles(main.page), texts(main.page)[0]], [["user", "notice"], "one"]); + assert.match(texts(main.page)[1], /^Line 3 could not be read\. It may belong to this branch or another one\.$/); + const fx2 = project(); + writeSession(fx2.dir, "s.jsonl", header(fx2.proj), [user("e1", null, "one", 1), assistant("e3", "e2", "three", 3)]); + const r2 = openOne(fx2, readerFor(fx2)); + assert.deepEqual(roles(r2.page), ["notice", "assistant"], "no unreadable line: history stops with a notice"); + assert.equal(texts(r2.page)[0], "History before this point is unavailable: an earlier entry is missing from the file."); +}); + +test("F1: an unreadable fork is never merged into another branch's history", () => { + // Line 4 held X, a fork from a; y is X's child. + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("a", null, "root", 1), + assistant("b", "a", "ON THE OTHER BRANCH", 2), + '{"type":"message","id":"X","parentId":"a",', + user("y", "X", "leaf", 4), + ]); + const reader = readerFor(fx); + const r = openOne(fx, reader); + assert.deepEqual(r.view.branches.map((b) => [b.branch, b.isDefault]), [["main", false], ["b.y", true]]); + assert.deepEqual(roles(r.page), ["notice", "user"]); + assert.match(texts(r.page)[0], /Line 4 could not be read and may have held it/); + assert.ok(!texts(r.page).some((t) => /ON THE OTHER BRANCH|^root$/.test(t)), "no text from the other branch"); +}); + +test("F1: a follow stays on its branch when the next entry's parent is unreadable", () => { + const fx = project(); + const path = writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("t1", null, "one", 1), "garbage"]); + const reader = readerFor(fx); + const r = openOne(fx, reader); + assert.deepEqual([roles(r.page), r.page.branch], [["user", "notice"], "main"]); + // t3's parent was on the unreadable line, so t3 starts its own branch. The + // view keeps its branch and parts, and reports the new default. + appendFileSync(path, lines(assistant("t3", "t2", "three", 3))); + const f = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: r.page.branch })); + assert.deepEqual([f.ok, f.page.entries.length, f.page.branch, f.view.defaultBranch], [true, 0, "main", "b.t3"]); + const again = openOne(fx, reader); + assert.deepEqual([roles(again.page), again.page.branch], [["notice", "assistant"], "b.t3"]); + assert.match(texts(again.page)[0], /Line 3 could not be read and may have held it/); +}); + +test("F1: a file whose entries are all unreadable shows a notice per line", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), ["garbage", "[1]", '{"type":"message"}']); + const r = openOne(fx, readerFor(fx)); + assert.deepEqual([r.ok, r.page.branch, r.view.unreadableLines], [true, "main", 3]); + assert.deepEqual(texts(r.page).map((t) => t.match(/^Line (\d+)/)[1]), ["2", "3", "4"]); +}); + +test("F2: a truncated trailing line marks the view incomplete, not an error", () => { + const fx = project(); + const path = writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("f1", null, "one", 1)], '{"type":"message","id":"f2","parentId":"f1"'); + const reader = readerFor(fx); + const r = openOne(fx, reader); + assert.equal(r.ok, true); + assert.equal(r.view.incomplete, true); + assert.deepEqual(texts(r.page), ["one"]); + assert.equal(r.view.unreadableLines, 0); + appendFileSync(path, ',"timestamp":"' + time(2) + '","message":{"role":"assistant","content":[{"type":"text","text":"two"}]}}\n'); + const f = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: r.page.branch })); + assert.equal(f.ok, true); + assert.deepEqual(texts(f.page), ["two"]); + assert.equal(f.view.incomplete, false); + assert.equal(f.page.sourceEpoch, r.page.sourceEpoch); +}); + +// 150 one-part entries: two pages. +function longSession(fx) { + const entries = []; + for (let i = 0; i < 150; i++) entries.push(i % 2 ? assistant(`g${i}`, i ? `g${i - 1}` : null, `m${i}`, i + 1) : user(`g${i}`, i ? `g${i - 1}` : null, `m${i}`, i + 1)); + return writeSession(fx.dir, "s.jsonl", header(fx.proj), entries); +} + +test("pagination: 100 parts, then the rest; parts concatenate to the whole branch", () => { + const fx = project(); + longSession(fx); + const reader = readerFor(fx); + const pages = readAll(fx, reader, openOne(fx, reader)); + assert.deepEqual(pages.map((p) => p.page.entries.length), [100, 50]); + assert.deepEqual(pages.flatMap((p) => texts(p.page)), Array.from({ length: 150 }, (_, i) => `m${i}`)); + assert.equal(pages[0].cursor.lastEntry, pages[0].page.entries[99].id); + assert.equal(pages[0].follow, null); + assert.ok(pages[1].follow); +}); + +test("F3: a replaced file (new inode) refuses old cursors with reconcile", () => { + const fx = project(); + const path = longSession(fx); + const reader = readerFor(fx); + const r = openOne(fx, reader); + const content = readFileSync(path); + writeFileSync(path + ".new", content); + renameSync(path + ".new", path); + for (const cursor of [r.cursor.id]) { + const x = noWrites(fx.base, () => reader.next({ cursor, conversation: r.page.conversation, branch: r.page.branch })); + assert.deepEqual([x.ok, x.refusal.code, x.refusal.reconcile], [false, "source-replaced", true]); + } + const short = project(); + writeSession(short.dir, "s.jsonl", header(short.proj), [user("h1", null, "one", 1)]); + const reader2 = readerFor(short); + const r2 = openOne(short, reader2); + const p2 = join(short.dir, "s.jsonl"); + writeFileSync(p2 + ".new", readFileSync(p2)); + renameSync(p2 + ".new", p2); + const f = noWrites(short.base, () => reader2.next({ cursor: r2.follow.id, conversation: r2.page.conversation, branch: r2.page.branch })); + assert.deepEqual([f.ok, f.refusal.code, f.refusal.reconcile], [false, "source-replaced", true], "follow cursor too"); + const reopened = openOne(short, reader2); + assert.notEqual(reopened.page.sourceEpoch, r2.page.sourceEpoch, "a reopen is a new epoch"); +}); + +test("F4: a same-inode rewrite of the prefix refuses old cursors with reconcile", () => { + const fx = project(); + const path = longSession(fx); + const reader = readerFor(fx); + const r = openOne(fx, reader); + const ino = lstatSync(path).ino; + const text = readFileSync(path, "utf8").replace('"m3"', '"M3"'); + const fd = openSync(path, "r+"); + writeSync(fd, text, 0); + closeSync(fd); + assert.equal(lstatSync(path).ino, ino, "same inode"); + const x = noWrites(fx.base, () => reader.next({ cursor: r.cursor.id, conversation: r.page.conversation, branch: r.page.branch })); + assert.deepEqual([x.ok, x.refusal.code, x.refusal.reconcile], [false, "source-replaced", true]); + // Shorter than the snapshot. + const r2 = openOne(fx, reader); + writeFileSync(path, lines(header(fx.proj))); + const y = noWrites(fx.base, () => reader.next({ cursor: r2.cursor.id, conversation: r2.page.conversation, branch: r2.page.branch })); + assert.deepEqual([y.ok, y.refusal.code, y.refusal.reconcile], [false, "source-replaced", true]); +}); + +test("F5: growth between pages keeps the epoch and the page stops at the pinned length", () => { + const fx = project(); + const path = longSession(fx); + const reader = readerFor(fx); + const r = openOne(fx, reader); + appendFileSync(path, lines(user("g150", "g149", "later", 200)) + '{"type":"mess'); + const r2 = noWrites(fx.base, () => reader.next({ cursor: r.cursor.id, conversation: r.page.conversation, branch: r.page.branch })); + assert.equal(r2.ok, true); + assert.equal(r2.page.entries.length, 50); + assert.equal(r2.page.snapshotDigest, r.page.snapshotDigest); + assert.equal(r2.page.sourceEpoch, r.page.sourceEpoch); + assert.equal(r2.view.incomplete, false, "the pinned snapshot ended on a complete line"); + assert.ok(!texts(r2.page).includes("later")); + // Following picks up the growth, up to the last complete line. + const f = noWrites(fx.base, () => reader.next({ cursor: r2.follow.id, conversation: r2.page.conversation, branch: r2.page.branch })); + assert.equal(f.ok, true); + assert.deepEqual(texts(f.page), ["later"]); + assert.equal(f.page.sourceEpoch, r.page.sourceEpoch); + assert.notEqual(f.page.snapshotDigest, r.page.snapshotDigest); + assert.equal(f.page.branch, "main"); + assert.equal(f.view.incomplete, true); + // A read of a file that grows by a partial line cuts at the last newline. + const again = openOne(fx, reader); + assert.equal(again.view.incomplete, true); + const all = readAll(fx, reader, again).flatMap((p) => texts(p.page)); + assert.equal(all.length, 151); +}); + +test("F6: unknown, foreign and expired cursors refuse and leave the cursor usable", () => { + const fx = project(); + longSession(fx); + let clock = Date.parse("2026-09-26T12:00:00Z"); + const reader = readerFor(fx, { now: () => clock, ttlMs: 60_000 }); + const r = openOne(fx, reader); + const good = { cursor: r.cursor.id, conversation: r.page.conversation, branch: r.page.branch }; + const cases = [ + [{ ...good, cursor: "c-unknown" }, "cursor-unknown"], + [{ ...good, actor: "someone-else" }, "cursor-foreign"], + [{ ...good, purpose: "drafts" }, "cursor-foreign"], + [{ ...good, conversation: "pi-" + "0".repeat(32) }, "cursor-foreign"], + [{ ...good, branch: "g1" }, "cursor-foreign"], + ]; + for (const [args, code] of cases) { + const x = noWrites(fx.base, () => reader.next(args)); + assert.deepEqual([x.ok, x.refusal.code, x.refusal.reconcile], [false, code, true], code); + } + const ok = noWrites(fx.base, () => reader.next(good)); + assert.equal(ok.ok, true, "the old view's cursor still works after the refusals"); + clock += 60_000; + const expired = noWrites(fx.base, () => reader.next(good)); + assert.deepEqual([expired.ok, expired.refusal.code, expired.refusal.reconcile], [false, "cursor-expired", true]); + const gone = noWrites(fx.base, () => reader.next(good)); + assert.equal(gone.refusal.code, "cursor-unknown"); + // Evicted cursors are unknown. + const small = readerFor(fx, { maxCursors: 2 }); + const first = openOne(fx, small); + openOne(fx, small); + openOne(fx, small); + const evicted = noWrites(fx.base, () => small.next({ cursor: first.cursor.id, conversation: first.page.conversation, branch: first.page.branch })); + assert.equal(evicted.refusal.code, "cursor-unknown"); + assert.ok(small.cursorCount() <= 2); + // open refuses other actors and purposes outright. + assert.equal(openOne(fx, reader, "s.jsonl", { actor: "x" }).refusal.code, "unknown-actor"); + assert.equal(openOne(fx, reader, "s.jsonl", { purpose: "drafts" }).refusal.code, "unsupported-purpose"); +}); + +// A file outside every root that no read may touch: unreadable, so an +// attempted open would throw EACCES rather than refuse. +function forbidden(base, name = "outside.jsonl") { + const dir = join(base, "outside"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, name); + writeFileSync(path, lines(header("/elsewhere"), user("z1", null, "secret", 1))); + chmodSync(path, 0o000); + return path; +} + +test("F7: a symlinked file and a symlinked directory component are refused, never opened", () => { + const fx = project(); + const outside = forbidden(fx.base); + symlinkSync(outside, join(fx.dir, "link.jsonl")); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("i1", null, "ok", 1)]); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + const link = cat.conversations.find((c) => c.availability === "denied"); + assert.deepEqual([link.refusal, link.title], ["unsafe-path", null]); + const r = openOne(fx, reader, "link.jsonl"); + assert.deepEqual([r.ok, r.refusal.code], [false, "unsafe-path"]); + assert.equal(cat.conversations.filter((c) => c.availability === "available").length, 1); + + // .pi/state/ is a symlink to a directory holding a sessions dir. + const fy = project("other"); + const realSeat = join(fy.base, "real-seat"); + mkdirSync(join(realSeat, "sessions"), { recursive: true }); + writeFileSync(join(realSeat, "sessions", "s.jsonl"), lines(header(fy.proj), user("j1", null, "hidden", 1))); + chmodSync(join(realSeat, "sessions", "s.jsonl"), 0o000); + mkdirSync(join(fy.proj, ".pi", "state"), { recursive: true }); + symlinkSync(realSeat, join(fy.proj, ".pi", "state", "linked")); + const spec = { agent: "linked", project: "proj", sessionsDir: join(fy.proj, ".pi", "state", "linked", "sessions") }; + const reader2 = createReader({ roots: rootsFromSpecs([spec]) }); + const cat2 = noWrites(fy.base, () => reader2.catalogue()); + assert.deepEqual(cat2.conversations, []); + assert.deepEqual(cat2.refusedRoots, [{ seat: "linked", project: "proj", refusal: "unsafe-path" }]); + const root = rootsFromSpecs([spec])[0]; + const r2 = noWrites(fy.base, () => reader2.open({ conversation: conversationId(root, "s.jsonl") })); + assert.deepEqual([r2.ok, r2.refusal.code], [false, "unknown-conversation"]); +}); + +test("F8: a file swapped for a symlink after the catalogue is refused", () => { + const fx = project(); + const outside = forbidden(fx.base); + const path = writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("k1", null, "ok", 1)]); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + const r = openOne(fx, reader); + assert.equal(r.ok, true); + rmSync(path); + symlinkSync(outside, path); + const x = noWrites(fx.base, () => reader.open({ conversation: cat.conversations[0].conversation })); + assert.deepEqual([x.ok, x.refusal.code], [false, "unsafe-path"]); + const f = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: r.page.branch })); + assert.deepEqual([f.ok, f.refusal.code], [false, "unsafe-path"]); +}); + +test("F9: registrations never add or redirect a root", () => { + const fx = project("seat-a"); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("l1", null, "ok", 1)]); + const outside = forbidden(fx.base); + const other = project("seat-a"); + writeSession(other.dir, "o.jsonl", header(other.proj), [user("l2", null, "other project", 1)]); + const linkDir = join(fx.base, "link-to-other"); + symlinkSync(other.dir, linkDir); + const reg = (extra) => ({ version: 1, seat: "seat-a", project: "proj", layout: "repo", harness: "claude-code", startedAt: time(0), ...extra }); + const registrations = [ + reg({ sessionsDir: join(fx.base, "outside") }), // outside every root + reg({ sessionsDir: linkDir }), // a symlink to another project's root + reg({ sessionsDir: other.dir }), // another project's directory + reg({ sessionsDir: fx.dir, project: "someone-else" }), // this directory, another project + reg({ seat: "seat-b", sessionsDir: join(fx.base, "outside") }), // a non-Pi placeholder outside the project + ]; + const roots = rootsFromSpecs([fx.spec], registrations); + assert.deepEqual(roots.map((r) => [r.seat, r.dir, r.harness, r.unsupportedReason]), [["seat-a", fx.dir, "pi", null]]); + const reader = createReader({ roots }); + const cat = noWrites(fx.base, () => reader.catalogue()); + assert.deepEqual(cat.conversations.map((c) => [c.title, c.availability]), [["ok", "available"]]); + assert.ok(outside); + // A registration that matches the root narrows it and supplies launch time. + const matched = rootsFromSpecs([fx.spec], [reg({ harness: "pi", sessionsDir: fx.dir, startedAt: time(-5) })]); + assert.equal(matched[0].engineStartedAt, time(-5)); + const cat2 = noWrites(fx.base, () => createReader({ roots: matched }).catalogue()); + assert.equal(cat2.conversations[0].engineStartedAt, time(-5)); + // Only repository specs are roots. + assert.deepEqual(rootsFromSpecs([ + { agent: "x", project: "fleet", sessionsDir: join(fx.base, "fleet", "x", ".pi", "agent", "sessions") }, + { agent: "seat-a", project: "wrong", sessionsDir: fx.dir }, + { agent: "seat-b", project: "proj", sessionsDir: fx.dir }, + { ...fx.spec, connector: "discord" }, + ]), []); +}); + +test("F10: a header cwd naming another project is refused", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(join(fx.base, "another-project")), [user("m1", null, "x", 1)]); + writeSession(fx.dir, "sub.jsonl", header(join(fx.proj, "packages", "x")), [user("m2", null, "sub", 1)]); + writeSession(fx.dir, "prefix.jsonl", header(fx.proj + "-evil"), [user("m3", null, "x", 1)]); + // Relative to the reader's own directory this names the project; it is still refused. + writeSession(fx.dir, "relative.jsonl", header(relative(process.cwd(), join(fx.proj, "packages")) || "."), [user("m5", null, "x", 1)]); + const linkRoot = join(fx.base, "compat"); + symlinkSync(fx.proj, linkRoot); + writeSession(fx.dir, "link.jsonl", header(linkRoot), [user("m4", null, "via compat link", 1)]); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + const by = Object.fromEntries(cat.conversations.map((c) => [c.conversation, c])); + const root = rootsFromSpecs([fx.spec])[0]; + for (const name of ["s.jsonl", "prefix.jsonl", "relative.jsonl"]) { + assert.deepEqual([by[conversationId(root, name)].availability, by[conversationId(root, name)].refusal], ["denied", "foreign-project"], name); + assert.deepEqual(openOne(fx, reader, name).refusal.code, "foreign-project", name); + } + assert.equal(openOne(fx, reader, "sub.jsonl").ok, true); + assert.equal(openOne(fx, reader, "link.jsonl").ok, true); + assert.equal(cwdInProject(undefined, fx.proj), false); +}); + +test("F11: parentSession renders with a marker and the parent is never opened", () => { + const fx = project(); + const parent = forbidden(fx.base, "parent.jsonl"); + writeSession(fx.dir, "s.jsonl", header(fx.proj, { parentSession: parent }), [user("n1", null, "forked", 1)]); + const r = openOne(fx, readerFor(fx)); + assert.equal(r.ok, true); + assert.deepEqual(roles(r.page), ["notice", "user"]); + assert.match(texts(r.page)[0], /forked from an earlier session/); + assert.equal(r.view.forkedFromEarlierSession, true); +}); + +test("F12: two leaves: the default leaf is shown and the other branch reads alone", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("o1", null, "root", 1), + assistant("o2", "o1", "first answer", 2), + user("o3", "o2", "on branch one", 3), + assistant("o4", "o1", "second answer", 4), + ]); + const reader = readerFor(fx); + const r = openOne(fx, reader); + // o1's earliest child continues "main"; the later child o4 starts "b.o4". + assert.deepEqual(texts(r.page), ["root", "second answer"]); + assert.equal(r.page.branch, "b.o4"); + assert.deepEqual(r.view.branches.map((b) => [b.branch, b.isDefault]), [["main", false], ["b.o4", true]]); + const other = openOne(fx, reader, "s.jsonl", { branch: "main" }); + assert.deepEqual(texts(other.page), ["root", "first answer", "on branch one"]); + assert.equal(other.page.branch, "main"); + assert.ok(other.page.entries.every((e) => e.branch === "main")); + for (const inner of ["o2", "b.o2", "o3", "b.o3", "o4"]) assert.equal(openOne(fx, reader, "s.jsonl", { branch: inner }).refusal.code, "unknown-branch", `${inner} is not a branch name`); + // Following a non-default branch after the default moves on adds nothing. + const path = join(fx.dir, "s.jsonl"); + appendFileSync(path, lines(user("o5", "o4", "more on default", 5))); + const f = noWrites(fx.base, () => reader.next({ cursor: other.follow.id, conversation: other.page.conversation, branch: "main" })); + assert.deepEqual([f.ok, f.page.entries.length, f.page.branch, f.view.defaultBranch], [true, 0, "main", "b.o4"]); + // Following the default branch continues on it under the same name. + const g = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: "b.o4" })); + assert.deepEqual([texts(g.page), g.page.branch, g.view.defaultBranch], [["more on default"], "b.o4", "b.o4"]); + assert.ok([...r.page.entries, ...g.page.entries].every((e) => e.branch === "b.o4"), "one branch value across follows"); + // The pre-growth name still opens, now with the growth. + assert.deepEqual(texts(openOne(fx, reader, "s.jsonl", { branch: "b.o4" }).page), ["root", "second answer", "more on default"]); + // A new default elsewhere: the old default view stays put. + appendFileSync(path, lines(assistant("o6", "o3", "back on branch one", 6))); + const h = noWrites(fx.base, () => reader.next({ cursor: g.follow.id, conversation: g.page.conversation, branch: "b.o4" })); + assert.deepEqual([h.page.entries.length, h.page.branch, h.view.defaultBranch], [0, "b.o4", "main"]); + // And the "main" view picks its growth up. + const k = noWrites(fx.base, () => reader.next({ cursor: f.follow.id, conversation: f.page.conversation, branch: "main" })); + assert.deepEqual([texts(k.page), k.page.branch], [["back on branch one"], "main"]); +}); + +test("F12: a follow refuses when an appended duplicate id changes the branch's earlier parts", () => { + // Where an id repeats the later entry wins, so an append can move an entry. + for (const [extra, why] of [ + [[{ ...assistant("r2", "x1", "moved", 4) }], "r2 moves to the other root: main is shorter"], + [[{ ...assistant("r2", "x1", "moved", 4) }, assistant("r5", "r1", "in its place", 5)], "same length, another entry in r2's place"], + ]) { + const fx = project(); + const path = writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("r1", null, "one", 1), user("x1", null, "other root", 2), assistant("r2", "r1", "two", 3)]); + const reader = readerFor(fx); + const r = openOne(fx, reader, "s.jsonl", { branch: "main" }); + assert.deepEqual(texts(r.page), ["one", "two"]); + appendFileSync(path, lines(...extra)); + const f = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: "main" })); + assert.deepEqual([f.ok, f.refusal?.code, f.refusal?.reconcile], [false, "source-replaced", true], why); + } +}); + +test("F12: a second root (Pi's resetLeaf) starts its own branch", () => { + const fx = project(); + const path = writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("r1", null, "first root", 1), + assistant("r2", "r1", "first reply", 2), + user("r3", null, "second root", 3), + ]); + const reader = readerFor(fx); + const r = openOne(fx, reader); + assert.deepEqual([texts(r.page), r.page.branch], [["second root"], "b.r3"]); + assert.deepEqual(r.view.branches.map((b) => [b.branch, b.isDefault]), [["main", false], ["b.r3", true]]); + assert.deepEqual(texts(openOne(fx, reader, "s.jsonl", { branch: "main" }).page), ["first root", "first reply"]); + appendFileSync(path, lines(assistant("r4", "r3", "second reply", 4))); + const f = noWrites(fx.base, () => reader.next({ cursor: r.follow.id, conversation: r.page.conversation, branch: "b.r3" })); + assert.deepEqual([texts(f.page), f.page.branch], [["second reply"], "b.r3"]); +}); + +test("F13: compaction is a marker in place, then the retained content", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [ + user("p1", null, "early", 1), + assistant("p2", "p1", "early answer", 2), + { type: "compaction", id: "p3", parentId: "p2", timestamp: time(3), summary: "they talked", firstKeptEntryId: "p2", tokensBefore: 1000 }, + user("p4", "p3", "after", 4), + ]); + const r = openOne(fx, readerFor(fx)); + assert.deepEqual(roles(r.page), ["user", "assistant", "compaction", "user"]); + const marker = r.page.entries[2].content[0]; + assert.deepEqual([marker.type, marker.summary, marker.nativeEntry], ["compaction", "they talked", "p3"]); +}); + +test("F14: long strings split into fragments and parts, reassemble exactly, and pages respect the byte cap", () => { + const fx = project(); + const long = "x".repeat(LIMITS.chars + 1); + const emoji = "😀".repeat(LIMITS.chars + 3); // 4 bytes each: splits on bytes before characters + const blocks = Array.from({ length: 70 }, (_, i) => ({ type: "text", text: `block ${i}` })); + const wide = "€".repeat(200_000); // 3 bytes each, ~600 KB per part + const entries = [ + user("q1", null, long, 1), + assistant("q2", "q1", emoji, 2), + msg("q3", "q2", "assistant", blocks, { stopReason: "stop" }, 3), + ]; + for (let i = 4; i < 44; i++) entries.push(user(`q${i}`, `q${i - 1}`, wide, i)); + writeSession(fx.dir, "s.jsonl", header(fx.proj), entries); + const reader = readerFor(fx); + const pages = readAll(fx, reader, openOne(fx, reader)); + for (const p of pages) { + assert.ok(p.page.entries.length <= LIMITS.parts); + assert.ok(Buffer.byteLength(JSON.stringify(p.page), "utf8") <= LIMITS.pageBytes, "page within 8 MiB"); + for (const e of p.page.entries) { + assert.ok(e.content.length <= LIMITS.blocks); + for (const b of e.content) assert.ok([...(b.text ?? "")].length <= LIMITS.chars); + } + } + assert.ok(pages.length >= 3, "the multibyte parts fill pages on bytes before 100 parts"); + assert.ok(pages[1].page.entries.length < LIMITS.parts); + // Reassemble per native entry: parts in order, fragments in order. + const all = pages.flatMap((p) => p.page.entries); + const rebuilt = new Map(); + for (const e of all) { + const blocksOf = rebuilt.get(e.nativeEntry) ?? []; + for (const b of e.content) blocksOf[b.block] = (blocksOf[b.block] ?? "") + b.text; + rebuilt.set(e.nativeEntry, blocksOf); + } + assert.equal(rebuilt.get("q1")[0], long); + assert.equal(rebuilt.get("q2")[0], emoji); + assert.deepEqual(rebuilt.get("q3"), blocks.map((b) => b.text)); + assert.equal(rebuilt.get("q20")[0], wide); + const q1 = all.filter((e) => e.nativeEntry === "q1"); + assert.deepEqual(q1.flatMap((e) => e.content.map((b) => [b.fragment, b.lastFragment])), [[0, false], [1, true]]); + const q3 = all.filter((e) => e.nativeEntry === "q3"); + assert.deepEqual(q3.map((e) => [e.part, e.lastPart, e.content.length]), [[0, false, 64], [1, true, 6]]); + assert.equal(new Set(all.map((e) => e.id)).size, all.length); +}); + +test("fragments never cut a surrogate pair and keep an empty string", () => { + assert.deepEqual(fragments(""), [""]); + const s = "a" + "😀".repeat(LIMITS.chars); + const parts = fragments(s); + assert.equal(parts.join(""), s); + for (const p of parts) { + assert.ok(!/^[\udc00-\udfff]/.test(p) && !/[\ud800-\udbff]$/.test(p)); + assert.ok(Buffer.byteLength(JSON.stringify(p)) <= 1024 * 1024 + 2); + } +}); + +test("F15: a Claude seat is an unsupported-harness placeholder whose directory is never read", () => { + const fx = project("pi-seat"); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("r1", null, "ok", 1)]); + // Rocko's case: a claude-code registration and no Pi sessions directory. + const claudeDir = join(fx.proj, ".pi", "state", "claude-seat", "sessions"); + const regs = [{ version: 1, seat: "claude-seat", project: "proj", layout: "repo", harness: "claude-code", sessionsDir: claudeDir, startedAt: time(-1) }]; + // And a spec'd seat with sessions whose registration says claude-code. + const both = join(fx.proj, ".pi", "state", "mixed", "sessions"); + mkdirSync(both, { recursive: true }); + writeFileSync(join(both, "s.jsonl"), lines(header(fx.proj), user("r2", null, "never read", 1))); + chmodSync(join(both, "s.jsonl"), 0o000); + regs.push({ version: 1, seat: "mixed", project: "proj", layout: "repo", harness: "claude-code", sessionsDir: both, startedAt: time(-1) }); + const roots = rootsFromSpecs([fx.spec, { agent: "mixed", project: "proj", sessionsDir: both }], regs); + const reader = createReader({ roots }); + const cat = noWrites(fx.base, () => reader.catalogue()); + const unsupported = cat.conversations.filter((c) => c.availability === "unsupported"); + assert.deepEqual(unsupported.map((c) => [c.seat, c.harness, c.unsupportedReason, c.history]).sort(), [["claude-seat", "claude-code", UNSUPPORTED_HARNESS, null], ["mixed", "claude-code", UNSUPPORTED_HARNESS, null]]); + for (const c of unsupported) { + const r = noWrites(fx.base, () => reader.open({ conversation: c.conversation })); + assert.deepEqual([r.ok, r.refusal.code, r.refusal.reconcile], [false, UNSUPPORTED_HARNESS, false]); + } +}); + +test("unknown conversations, empty files and non-Pi files refuse", () => { + const fx = project(); + writeFileSync(join(fx.dir, "empty.jsonl"), ""); + writeFileSync(join(fx.dir, "notpi.jsonl"), lines({ hello: 1 })); + writeFileSync(join(fx.dir, "header-only.jsonl"), lines(header(fx.proj))); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + assert.deepEqual(cat.conversations.map((c) => [c.availability, c.refusal]).sort(), [["available", null], ["unavailable", "incomplete-header"], ["unavailable", "not-a-pi-session"]]); + assert.equal(openOne(fx, reader, "empty.jsonl").refusal.code, "incomplete-header"); + assert.equal(openOne(fx, reader, "notpi.jsonl").refusal.code, "not-a-pi-session"); + const empty = openOne(fx, reader, "header-only.jsonl"); + assert.deepEqual([empty.ok, empty.page.entries.length, empty.page.branch, empty.view.branches], [true, 0, "main", [{ branch: "main", isDefault: true, lastActivityAt: null }]]); + appendFileSync(join(fx.dir, "header-only.jsonl"), lines(user("s1", null, "first", 1))); + const f = noWrites(fx.base, () => reader.next({ cursor: empty.follow.id, conversation: empty.page.conversation, branch: "main" })); + assert.deepEqual([texts(f.page), f.page.branch], [["first"], "main"]); + const u = noWrites(fx.base, () => reader.open({ conversation: "pi-" + "f".repeat(32) })); + assert.deepEqual([u.refusal.code, u.refusal.reconcile], ["unknown-conversation", true]); +}); + +test("an unreadable file or root inside the roots is refused per row, not a failed catalogue", () => { + const fx = project(); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("u1", null, "ok", 1)]); + writeSession(fx.dir, "locked.jsonl", header(fx.proj), [user("u2", null, "locked", 1)]); + chmodSync(join(fx.dir, "locked.jsonl"), 0o000); + const reader = readerFor(fx); + const cat = noWrites(fx.base, () => reader.catalogue()); + assert.deepEqual(cat.conversations.map((c) => [c.availability, c.refusal]).sort(), [["available", null], ["denied", "unreadable"]]); + assert.equal(openOne(fx, reader, "locked.jsonl").refusal.code, "unreadable"); + chmodSync(fx.dir, 0o300); + try { + const cat2 = noWrites(fx.base, () => reader.catalogue()); + assert.deepEqual([cat2.conversations, cat2.refusedRoots.map((r) => r.refusal)], [[], ["unreadable"]]); + } finally { + chmodSync(fx.dir, 0o700); + } +}); + +test("a seat directory without search permission refuses that root, not the catalogue", () => { + const fx = project("good"); + writeSession(fx.dir, "s.jsonl", header(fx.proj), [user("p1", null, "readable", 1)]); + const badDir = join(fx.proj, ".pi", "state", "bad", "sessions"); + mkdirSync(badDir, { recursive: true }); + writeSession(badDir, "s.jsonl", header(fx.proj), [user("q1", null, "hidden", 1)]); + const bad = { agent: "bad", project: "proj", sessionsDir: badDir }; + const goodId = conversationId(rootsFromSpecs([fx.spec])[0], "s.jsonl"); + chmodSync(join(fx.proj, ".pi", "state", "bad"), 0o000); + try { + for (const specs of [[bad, fx.spec], [fx.spec, bad]]) { + const reader = createReader({ roots: rootsFromSpecs(specs) }); + const cat = noWrites(fx.base, () => reader.catalogue()); + assert.deepEqual([cat.conversations.map((c) => [c.seat, c.availability]), cat.refusedRoots], [[["good", "available"]], [{ seat: "bad", project: "proj", refusal: "unreadable" }]]); + const r = noWrites(fx.base, () => reader.open({ conversation: goodId })); + assert.deepEqual(texts(r.page), ["readable"]); + const u = noWrites(fx.base, () => reader.open({ conversation: "pi-" + "e".repeat(32) })); + assert.equal(u.refusal.code, "unknown-conversation"); + } + } finally { + chmodSync(join(fx.proj, ".pi", "state", "bad"), 0o700); + } +}); + +test("every page and cursor is a valid CHAT-01 record", () => { + assert.ok(records.length > 50); + const script = ` +import json, sys +from jsonschema import Draft202012Validator, FormatChecker +s = json.load(open(sys.argv[1])) +bad = [] +for i, r in enumerate(json.load(sys.stdin)): + v = Draft202012Validator({"$defs": s["$defs"], "$ref": "#/$defs/" + r["kind"]}, format_checker=FormatChecker()) + for e in v.iter_errors(r): + bad.append(f"{i} {r['kind']}: {e.message[:200]}") + break +print(json.dumps(bad)) +`; + const run = spawnSync("python3", ["-c", script, schemaPath], { input: JSON.stringify(records), encoding: "utf8", maxBuffer: 1 << 30 }); + assert.equal(run.status, 0, run.stderr); + assert.deepEqual(JSON.parse(run.stdout), []); +});