From 4f3fbc25a0df293447a4d98828dab6b34e337859 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 22 Oct 2026 09:42:51 -0500 Subject: [PATCH 1/3] row 216 (C2-05a, #1717): library index cache and lookup queries find, get, search, manuscript, universeInfo; get pages links, manuscript takes no chapter, search marks a too-long line, PRD 1.14 levels (rulings da9555ca2); unrun --- agents/researcher/work/queue-216/BUILD.md | 184 ++++++++++ packages/library/src/index-build.mjs | 75 ++++ packages/library/src/index.mjs | 2 + packages/library/src/query.mjs | 240 +++++++++++++ packages/library/tests/index-build.test.mjs | 91 +++++ packages/library/tests/query-fixture.mjs | 31 ++ packages/library/tests/query.test.mjs | 369 ++++++++++++++++++++ 7 files changed, 992 insertions(+) create mode 100644 agents/researcher/work/queue-216/BUILD.md create mode 100644 packages/library/src/index-build.mjs create mode 100644 packages/library/src/query.mjs create mode 100644 packages/library/tests/index-build.test.mjs create mode 100644 packages/library/tests/query-fixture.mjs create mode 100644 packages/library/tests/query.test.mjs diff --git a/agents/researcher/work/queue-216/BUILD.md b/agents/researcher/work/queue-216/BUILD.md new file mode 100644 index 000000000..22579ca70 --- /dev/null +++ b/agents/researcher/work/queue-216/BUILD.md @@ -0,0 +1,184 @@ +# Row 216 build notes (C2-05a, lookup index and queries, #1717) + +Phase A only: code, tests and this file. No test, suite or mutant has been run. `node --check` passed on every changed `.mjs`. + +- Branch: `researcher/row216` +- Base: `d372102882d6858fa77af4aee453f866c886ad5c` (origin/refactor) +- Design: `agents/gosalyn/work/case2/DESIGN.md` on origin/gosalyn/case2-design, section 3 (table at lines 625-631, bullets 633-641), 1.2-1.4, B12, D2 +- Brief: `docs/plans/2026-10-20_case2-rows.md`, C2-05a at lines 320-357 + +## Files + +| Path | Status | +|---|---| +| `packages/library/src/index-build.mjs` | new | +| `packages/library/src/query.mjs` | new | +| `packages/library/src/index.mjs` | edit: 2 export lines added | +| `packages/library/tests/query-fixture.mjs` | new, test helper | +| `packages/library/tests/query.test.mjs` | new | +| `packages/library/tests/index-build.test.mjs` | new | +| `agents/researcher/work/queue-216/BUILD.md` | new (this file) | + +No existing test or source file changed beyond `index.mjs`. + +## Design rows and where each is tested + +| Design row | Code | Tests (query.test.mjs unless noted) | +|---|---|---| +| `library_find` | `find` | find is bound to its universe; every result carries its scope; long list pages; large items page under the cap | +| `library_get` | `get` | hybrid gets a source file; get lists visible links both ways; get pages its lines | +| `library_search` | `search` | search returns scoped line hits; item that cannot fit alone is refused | +| `library_manuscript` | `manuscript` | one chapter at most 400 lines; refused without chapter; hybrid reads a source chapter | +| `library_universe` | `universeInfo` | universe info reports a hybrid's sources, constraints, notes; unknown universe refused | +| Hybrid semantics (1.4) | `sees` | a hybrid sees its sources canon minus exclusions, no playground | +| Unmapped never appears | index build + `sees` | unmapped files never appear (query.test); index holds mapped files only (index-build.test) | +| Per-commit cache (3) | `indexFor` | same commit, path, mapping return one index; different path, mapping or commit get their own; eight-entry bound; failed read not kept; unchecked mapping refused (index-build.test) | +| Rebuild time (3, C2-05a gate) | `indexFor` | 3,000-file rebuild, time reported with `t.diagnostic`, no threshold asserted | +| REPLY_CAP parity | `QUERY_REPLY_CAP` | equals `REPLY_CAP` from harness tools.mjs | + +## Choices where the design table is silent or differs + +These are departures or gaps. Each is built as stated, so Sage can rule before Phase B. A ruling that changes one of them is a small edit. + +1. **Paging arguments (gap a).** The table has no offset. `find` and `search` take an optional `offset` (a position in the filtered list, default 0). A reply carries `next`: the position of the next item, or `null`. +2. **`library_get` has no universe (gap b).** Built with a required `universe`. Without it the reply could not be isolated. A missing file and a file the universe cannot see both return `null`, so a caller cannot tell them apart. `get` also takes an optional `from` line. +3. **`get` over the cap (gap c).** Lines page by `from` with `next`. `links_out` and `links_in` come back whole on every page. If the frontmatter and links alone exceed the cap, the call is refused with `query-item-too-large`. A hub note with many backlinks may be refused. This needs a ruling: link paging, or accept the refusal. +4. **`manuscript` without a chapter (gap d).** Refused with `query-chapter-required`. The design names no list form. A chapter name that matches two visible files is refused with `query-chapter-ambiguous`. +5. **`search` excerpt (gap e).** The excerpt is the whole line, not a truncation. A line that alone exceeds the cap is refused. Truncating it would cut an item. +6. **`exclude` form (gap f).** `checkMapping` accepts only a folder path for `exclude` (`folder()`), so an exclusion is a path prefix. There is no type or source form to build. Confirm. +7. **Pin constraints.** `pin` does not hide any file. It is reported by `universeInfo` only. Confirm. +8. **Playground through `get`.** `scope` applies to lists. `get` by an explicit path returns a playground file of the caller's own universe. Confirm. +9. **Links.** A link into a file the universe cannot see is dropped from `links_out` and `links_in`, in line with N1. Links from a visible file that point into another universe are kept in the index (the reader keeps them under R6), and the query layer drops them for universes that cannot see them. +10. **`find` matching.** Case-insensitive substring over the front matter `name` and the file name. An empty `query` matches every file in scope. +11. **Cache key.** `repo`, `commit` and a sha256 of the canonical mapping. Keying by commit alone would return the wrong scopes after a mapping change. The cache holds eight entries, least recently used out. Failed reads are not kept. +12. **`universeInfo`.** Reads only the checked mapping and the served commit. It builds no index. +13. **Reply commit.** Every reply carries the index commit, which is the served commit passed in. + +## Provisional refusal codes (pending Sage approval) + +The brief says new refusal codes need Sage's approval first. These are provisional and must not be treated as settled: + +- `query-argument-invalid` +- `query-universe-unknown` +- `query-chapter-required` +- `query-chapter-ambiguous` +- `query-item-too-large` + +Reused, existing codes: `library-mapping-invalid`, plus the reader's `library-git`, `library-commit` through `indexFor`. + +## Grep results (grep only, not a test run) + +- Test references to the changed paths: `packages/library/tests/*.mjs` import `../src/index.mjs` and `../src/mapping.mjs`. `packages/bus/tests/library.test.mjs` and `library-sync.test.mjs` import the library index as a namespace. Neither enumerates its exports, so the two new export lines break no assertion. +- `packages/cli/tests/stack-copy.test.mjs:52` checks that `packages/library/src/index.mjs` exists, and it still does. +- 64-hex pins: none in `packages/library/tests`, none on `index.mjs`, `query.mjs` or `index-build.mjs`. +- `scripts/test-*.sh`: no reference to the new modules. + +## Flag for Sage + +- `packages/library/tests/helpers.mjs` `repo()` builds its fixtures under `tmpdir()`, which is `/tmp`. The launcher rules forbid that. The new tests do not use it; they use `tests/query-fixture.mjs`, which builds under `/mnt/storage/scratch/tmp`. The existing helper should be fixed in a separate row, not here. + +## Phase B commands (not run) + +Run only after Sage sends "slot clear". Set the scratch TMPDIR first, per the launcher rule. + +``` +cd /mnt/storage/src/mosaic-stack/.claude/worktrees/agent-aa3cc98787bc5152c/packages/library +export TMPDIR=/mnt/storage/scratch/tmp +node --test tests/query.test.mjs tests/index-build.test.mjs +node --test tests/*.test.mjs +``` + +Also run `packages/bus/tests/library.test.mjs` and `library-sync.test.mjs`, since they import the index. + +## Status + +Not yet verified. Nothing has been tested and nothing is committed. The working tree holds the Phase A changes. AGENTS.md invariant 8 allows a commit only after the applicable suites are green, so the commit waits for Phase B. The dispatch asked for a local commit in Phase A; Sage to confirm that the commit waits for Phase B. + +## Round 2 (2026-10-22, sage-2708, sage-2716): rulings applied, rebased onto aca16404c + +Added after the text above; nothing above is edited. "Nothing is committed", "Provisional refusal codes", +`query-chapter-required` and the tmpdir flag above were true when written and are superseded here. + +- Branch `researcher/row216-r2`, one local commit on origin/refactor `aca16404c`. Row 268 (C2-01c) is in + that base. Row 253 has not landed; the work rebases onto 253's tip before any gate. Not pushed, not run. +- Rulings: `agents/sage/work/queue-216/RULINGS.md` at da9555ca2, rulings 1-13 and the tmpdir flag. +- Drift check, d37210288..aca16404c against my paths: the library files that changed are `mapping.mjs`, + `scope.mjs`, `README.md` and the mapping and scope tests. None is a file I add or edit. `query.mjs` reads + the checked mapping through them, so the whole library package runs in phase B. + +### What changed from phase A + +| Ruling | Change in `query.mjs` | Test | +|---|---|---| +| 3 `get` over the cap | `links_out` and `links_in` page together with `links_from`; every reply carries `links_next`, `next` and the front matter. A page with lines left carries at least one line. Links take at most half the cap while lines remain. `query-item-too-large` is left for front matter alone or one line alone (or one link alone) | hub note with 900 backlinks pages and adds up; front matter alone and a line alone refused; bad `links_from` | +| 4 `manuscript` no chapter | `chapter` optional: the first visible chapter in path order. Every reply carries `next_chapter`. `query-chapter-required` removed. Ambiguity still refuses | first chapter and `next_chapter`; ambiguous name refused | +| 5 `search` long line | hit comes back `{path, line, excerpt: null, excerpt_too_long: true, scope}`, other hits still return | marked hit and unmarked hits in one reply | +| 6 `exclude` | path-prefix reading only. The test names `EXCLUSION` and `EXCLUDED_PATHS` in `query-fixture.mjs`, so C2-15 (row 267) swaps two constants | exclusion hides from the hybrid and not from the source | +| 8 hybrid `get` | unchanged code; test added | hybrid gets a source's playground file as `null`; the source gets it with `scope: "playground"` | +| PRD 1.14 (row 268) | `universeInfo` reports `kind` from the mapping (`draft` included) | four tests below | + +PRD 1.14 tests, one each: a lookup naming a `book` leaves `book: null` files out (find and search); +`library_manuscript` never reads a `book: null` file; a draft universe has no canon, so its lookups return +files only with `scope: "playground"`; `library_universe` on a draft returns kind, name and prefixes. +The levels fixture is `WITH_LEVELS` in `query.test.mjs`: harbor gets `Notes/Harbor/` as a universe-level +playground prefix, and `idea` is a draft over `Drafts/New World/`. + +`query-fixture.mjs` builds under `tmpdir()`, as `helpers.mjs` does (ruling "tmpdir flag"). The earlier +flag about `helpers.mjs` building under `/tmp` is closed by that ruling: the gate sets `TMPDIR`. + +Refusal codes now: `query-argument-invalid`, `query-universe-unknown`, `query-chapter-ambiguous`, +`query-item-too-large`. All four approved in the rulings. + +### Grep results (grep only, not a test run) + +- `git grep` over `packages/*/tests/*` and `scripts/*` for `query.mjs` and `index-build`: no hit outside + `packages/library/tests/query*.test.mjs` and `index-build.test.mjs`. +- 64-hex pins naming a library path: none. Positive control: the same pathspec finds the sha256 lines in + `packages/cli/tests/case1-hostcheck.test.mjs` and the `packages/library/src/index.mjs` existence check at + `packages/cli/tests/stack-copy.test.mjs:52`. +- Tests that read `packages/library/src` as a directory: `packages/cli/tests/stack-release-fixture.mjs:309` + copies `packages/library/src` whole into a release fixture. It carries no file list, so two new files break + no assertion. `stack-release.test.mjs` is not in the gate unless Sage wants it. +- Tests that import the library index as a namespace: `packages/bus/tests/library.test.mjs` and + `library-sync.test.mjs`. Neither enumerates exports, so the two export lines add no break. Both run + whole in phase B. + +### Run limits (fixed before any run) + +Every node file runs as its own `setsid -w timeout -k 10 LIMIT node --test FILE`; nothing runs unbounded. + +- Node test files: LIMIT 900 s. The library test files have no recorded time. 900 s is above the longest + recorded whole-package run I know of (cli, 591 s, row 281 BUILD.md "Runs" step 3), so it holds the 3x rule + for any single file. The first run records each file's time, and a file past 300 s gets a limit re-derived + from it before the next run. `query.test.mjs` builds a 900-file repo and a 3,000-file index rebuild is in + `index-build.test.mjs`; both are the files most likely to need it. +- No shell script is in this row's gate, so no script limit applies. + +Any timeout goes to Sage, then Mos, unplaced (mos-1275). + +### Phase B (not run; Sage's CLEAR first, then a rebase onto 253's tip) + +``` +export TMPDIR=/mnt/storage/scratch/tmp +cd /packages/library +for f in tests/*.test.mjs; do setsid -w timeout -k 10 900 node --test "$f"; done +cd ../bus +for f in tests/library.test.mjs tests/library-sync.test.mjs; do setsid -w timeout -k 10 900 node --test "$f"; done +cd ../cli +setsid -w timeout -k 10 900 node --test tests/stack-copy.test.mjs +``` + +Mutants I propose (my list, not read off the brief; each is meant to be killed by the named test, and none +has been thrown; the kill claims are unverified until CLEAR): + +- M-a: `sees()` returns true for a file of another universe: killed by "find is bound to its universe". +- M-b: the playground filter dropped: killed by "playground appears only when asked for". +- M-c: the hybrid reads a source's playground: killed by "a hybrid gets a source playground file as null". +- M-d: `links_next` always null: killed by the hub paging test. +- M-e: the book filter lets `book: null` through: killed by the PRD 1.14 book test. + +### Status + +Built, not run. `node --check` has not been run on this round's files (no node before the CLEAR); the +phase A pass above covered the earlier text only. The commit is local and waits for no gate under Sage's +sage-2716 instruction, which allows git and docs work only; push and claim wait for Sage's go. diff --git a/packages/library/src/index-build.mjs b/packages/library/src/index-build.mjs new file mode 100644 index 000000000..201de119d --- /dev/null +++ b/packages/library/src/index-build.mjs @@ -0,0 +1,75 @@ +// The lookup index of design 3 (C2-05a): types, names, links and backlinks for one commit of her repo. +// It is built from the reader's output, so it holds only mapped, readable files: R1, R2 and R7 leave a +// file out, and an unresolved link is not in it. It is cached in memory by commit id. The key also +// carries the repo path and a digest of the mapping, because one commit reads differently under another +// mapping. The index is not universe-aware; query.mjs decides what a universe may see. +import { createHash } from 'node:crypto'; +import { readLibrary } from './reader.mjs'; +import { raise } from './problems.mjs'; + +const CACHE_LIMIT = 8; +const cache = new Map(); + +const push = (map, key, value) => { + if (!map.has(key)) map.set(key, []); + map.get(key).push(value); +}; +// A file's name for lookups: its front matter `name` when it is text, else its file name. +const displayName = file => { + const name = file.frontmatter?.name; + return typeof name === 'string' && name.trim() ? name.trim() : file.name; +}; +// A link as the index keeps it: the file at the other end, its line, and the field when front matter holds it. +const linkAt = (path, link) => ({ path, line: link.line, ...(link.field === undefined ? {} : { field: link.field }) }); + +// Builds the index from readLibrary's output. Pure: no git, no clock. +export function buildIndex(library, mapping) { + const files = new Map(), types = new Map(), names = new Map(), linksOut = new Map(), linksIn = new Map(); + for (const file of library.files) { + const name = displayName(file); + files.set(file.path, { + path: file.path, scope: file.scope, universe: file.universe, book: file.book, type: file.type, + folder: file.folder, stem: file.name, name, frontmatter: file.frontmatter, lines: file.lines, + }); + if (file.type !== null) push(types, file.type, file.path); + push(names, name.toLowerCase(), file.path); + linksOut.set(file.path, file.links.map(link => linkAt(link.path, link))); + for (const link of file.links) push(linksIn, link.path, linkAt(file.path, link)); + } + return Object.freeze({ + commit: library.commit, mapping, files, paths: [...files.keys()], types, names, linksOut, linksIn, + problems: library.problems, + }); +} + +const digest = mapping => createHash('sha256').update(JSON.stringify(mapping)).digest('hex'); + +// The index for one commit under one mapping, read once and kept in memory. A repeat call returns the same +// object. Past CACHE_LIMIT entries the oldest goes. A failed read is not kept. +export async function indexFor({ repo, commit, mapping }) { + if (!mapping || mapping.kind !== 'library') raise('library-mapping-invalid', 'expected a checked library mapping'); + const key = JSON.stringify([repo, commit, digest(mapping)]); + if (cache.has(key)) { + const hit = cache.get(key); + cache.delete(key); + cache.set(key, hit); + return hit; + } + const pending = readLibrary({ repo, commit, mapping }).then(library => buildIndex(library, mapping)); + cache.set(key, pending); + while (cache.size > CACHE_LIMIT) cache.delete(cache.keys().next().value); + try { + return await pending; + } catch (error) { + if (cache.get(key) === pending) cache.delete(key); + throw error; + } +} + +export function clearIndexCache() { + cache.clear(); +} + +export function indexCacheSize() { + return cache.size; +} diff --git a/packages/library/src/index.mjs b/packages/library/src/index.mjs index e038309d6..627e7738e 100644 --- a/packages/library/src/index.mjs +++ b/packages/library/src/index.mjs @@ -8,3 +8,5 @@ export { renderItems, KINDS, NOTICES, UNREADABLE } from './render.mjs'; export { survey, formatSurvey, surveyMain } from './survey.mjs'; export { SYNC_STATE_FILE, mirrorPath, ensureMirror, withLibraryLock, servedCommit, readSyncState, libraryStatus, readLibraryFile } from './mirror.mjs'; export { syncCommand, sync, ASKPASS, SLOT, SYNC_INTERVAL_MS } from './sync.mjs'; +export { buildIndex, indexFor, clearIndexCache, indexCacheSize } from './index-build.mjs'; +export { sees, find, get, search, manuscript, universeInfo, QUERY_REPLY_CAP, LOOKUP_LIMIT, MANUSCRIPT_LINES } from './query.mjs'; diff --git a/packages/library/src/query.mjs b/packages/library/src/query.mjs new file mode 100644 index 000000000..cdc9e5cb5 --- /dev/null +++ b/packages/library/src/query.mjs @@ -0,0 +1,240 @@ +// The lookup queries of design 3 (C2-05a): find, get, search, manuscript and universeInfo. Each takes a +// source {repo, commit, mapping} and an argument object, and returns a reply that fits QUERY_REPLY_CAP. +// Universe isolation is here, in sees(): a file reaches a reply only when sees() allows it. Scope comes +// from the path (D2). Unmapped files are not in the index, so they never appear. Lists page by position +// and never cut an item. A search line too long to fit alone comes back as a marked hit with no excerpt; +// a find item, a file's front matter or one of its lines that cannot fit alone is refused. +import { raise } from './problems.mjs'; +import { TYPES } from './mapping.mjs'; +import { indexFor } from './index-build.mjs'; + +// The same cap as packages/harness/src/tools.mjs REPLY_CAP. The query tests assert the two stay equal. +export const QUERY_REPLY_CAP = 40_000; +export const LOOKUP_LIMIT = 50; +export const MANUSCRIPT_LINES = 400; +const SCOPES = Object.freeze(['canon', 'playground']); + +const bytes = value => JSON.stringify(value).length; +const invalid = message => raise('query-argument-invalid', message); +const inside = (folder, path) => path.startsWith(folder.endsWith('/') ? folder : `${folder}/`); + +function textArg(value, name) { + if (typeof value !== 'string') invalid(`${name} must be text`); + return value; +} +function scopeArg(value) { + if (value === undefined) return 'canon'; + if (!SCOPES.includes(value)) invalid('scope must be "canon" or "playground"'); + return value; +} +function positionArg(value, name, fallback) { + if (value === undefined) return fallback; + if (!Number.isSafeInteger(value) || value < 1) invalid(`${name} must be a line number of 1 or more`); + return value; +} +function offsetArg(value) { + if (value === undefined) return 0; + if (!Number.isSafeInteger(value) || value < 0) invalid('offset must be a whole number of 0 or more'); + return value; +} +function universeArg(mapping, value) { + if (typeof value !== 'string' || !value) invalid('universe is required'); + if (!Object.hasOwn(mapping.universes, value)) raise('query-universe-unknown', `this library has no universe ${JSON.stringify(value)}`); + return value; +} + +// Whether universe U may see a file. Its own files, in either scope. A hybrid also sees the canon of +// its declared sources, minus the folders its exclude constraints name. A hybrid never sees a source's +// playground. Pin constraints do not hide anything; they are reported by universeInfo only. +export function sees(mapping, universe, file) { + if (file.universe === universe) return true; + const entry = mapping.universes[universe]; + if (entry.kind !== 'hybrid' || !entry.sources.includes(file.universe) || file.scope !== 'canon') return false; + return !entry.constraints.some(c => c.exclude !== undefined && inside(c.exclude, file.path)); +} + +// The files one universe may see in one scope, with optional book and type filters, in path order. +async function visible(source, universe, { scope, book, type }) { + const index = await indexFor(source); + const files = index.paths + .map(path => index.files.get(path)) + .filter(file => file.scope === scope && sees(index.mapping, universe, file) + && (book === undefined || file.book === book) && (type === undefined || file.type === type)); + return { index, files }; +} + +// A reply whose size is over the cap is refused, never cut. +function checked(reply) { + if (bytes(reply) > QUERY_REPLY_CAP) raise('query-item-too-large', `the reply is over the ${QUERY_REPLY_CAP}-character cap`); + return reply; +} + +// Picks items from `offset`, at most `limit`, while the reply still fits. build(picked, next) returns +// the reply; `next` is the position of the first item not picked, or null when none is left. An item +// that cannot fit even alone is refused: skipping it would hide it, and cutting it would break it. +function fitPage(items, { offset = 0, limit = LOOKUP_LIMIT, build }) { + const picked = []; + let at = offset; + while (at < items.length && picked.length < limit) { + // The size check uses items.length for `next`, the widest number a cursor can take, so a page never runs over. + if (bytes(build([...picked, items[at]], items.length)) > QUERY_REPLY_CAP) { + if (!picked.length) raise('query-item-too-large', `the item at position ${at} alone is over the ${QUERY_REPLY_CAP}-character cap`); + break; + } + picked.push(items[at]); + at += 1; + } + return checked(build(picked, at < items.length ? at : null)); +} + +// find: the files whose name or file name holds `query`, case-insensitive. Up to 50 per page. +export async function find(source, { universe: name, query, type, book, scope, offset } = {}) { + const universe = universeArg(source.mapping, name); + const needle = textArg(query, 'query').toLowerCase(); + if (type !== undefined && !TYPES.includes(type)) invalid(`type must be one of ${TYPES.join(', ')}`); + const { index, files } = await visible(source, universe, { + scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'), type, + }); + const hits = files.filter(file => file.name.toLowerCase().includes(needle) || file.stem.toLowerCase().includes(needle)); + return fitPage(hits, { + offset: offsetArg(offset), + build: (picked, next) => ({ + commit: index.commit, + results: picked.map(file => ({ + path: file.path, type: file.type, name: file.name, appears_in: file.frontmatter?.appears_in ?? null, + scope: file.scope, universe: file.universe, book: file.book, + })), + next, + }), + }); +} + +// get: one visible file, its front matter, its lines from `from`, and its visible links both ways. +// Returns null when the file is missing or not visible, so a caller cannot tell the two apart. Lines page +// by `from` (reply `next`), links page together by `links_from` (reply `links_next`); `links_out` come +// before `links_in`. Every page carries the front matter. Links take at most half the cap, so a hub note +// with many backlinks stays readable and still leaves room for lines. A page with lines left carries at +// least one line, so links shrink when a line needs the room; read on with `from: next` and +// `links_from: links_next`. A `from` past the last line returns links alone. Only front matter, or a +// line that cannot fit with no links beside it, is refused. +export async function get(source, { universe: name, path, from, links_from: linksFrom } = {}) { + const universe = universeArg(source.mapping, name); + const startLine = positionArg(from, 'from', 1); + const linkStart = offsetArg(linksFrom); + if (typeof path !== 'string') invalid('path must be text'); + const index = await indexFor(source); + const file = index.files.get(path); + if (!file || !sees(index.mapping, universe, file)) return null; + const reachable = links => links.filter(link => { + const other = index.files.get(link.path); + return other !== undefined && sees(index.mapping, universe, other); + }); + const outs = reachable(index.linksOut.get(file.path) ?? []); + const ins = reachable(index.linksIn.get(file.path) ?? []); + const links = [...outs.map(link => ({ out: true, link })), ...ins.map(link => ({ out: false, link }))]; + const lines = file.lines.slice(startLine - 1); + const linkRest = Math.max(0, links.length - linkStart); + const page = (nLines, nLinks) => { + const taken = links.slice(linkStart, linkStart + nLinks); + return { + path: file.path, commit: index.commit, scope: file.scope, frontmatter: file.frontmatter, + lines: lines.slice(0, nLines), + links_out: taken.filter(item => item.out).map(item => item.link), + links_in: taken.filter(item => !item.out).map(item => item.link), + next: nLines < lines.length ? startLine + nLines : null, + links_next: nLinks < linkRest ? linkStart + nLinks : null, + }; + }; + // How many leading items fit in `room`, by each item's own size. + const count = (items, room) => { + let used = 0, n = 0; + while (n < items.length && used + bytes(items[n]) + 1 <= room) used += bytes(items[n++]) + 1; + return n; + }; + const base = bytes(page(0, 0)); + if (base > QUERY_REPLY_CAP) raise('query-item-too-large', `the front matter of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); + if (lines.length && bytes(page(1, 0)) > QUERY_REPLY_CAP) { + raise('query-item-too-large', `line ${lines[0][0]} of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); + } + const rest = links.slice(linkStart); + let nLinks = count(rest, (lines.length ? Math.floor(QUERY_REPLY_CAP / 2) : QUERY_REPLY_CAP) - base); + if (!nLinks && rest.length) nLinks = count(rest, QUERY_REPLY_CAP - base); + if (!nLinks && rest.length) raise('query-item-too-large', `a link of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); + while (lines.length && nLinks > 0 && bytes(page(1, nLinks)) > QUERY_REPLY_CAP) nLinks = Math.floor(nLinks / 2); + let nLines = Math.max(lines.length ? 1 : 0, count(lines, QUERY_REPLY_CAP - bytes(page(0, nLinks)))); + while (nLines > 1 && bytes(page(nLines, nLinks)) > QUERY_REPLY_CAP) nLines -= 1; + return checked(page(nLines, nLinks)); +} + +// search: lines whose text holds `text`, case-insensitive. The excerpt is the whole line. Up to 50 per page. +export async function search(source, { universe: name, text, book, scope, offset } = {}) { + const universe = universeArg(source.mapping, name); + const needle = textArg(text, 'text').toLowerCase(); + if (!needle) invalid('text must not be empty'); + const { index, files } = await visible(source, universe, { + scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'), + }); + const hits = []; + for (const file of files) { + for (const [line, body] of file.lines) { + if (!body.toLowerCase().includes(needle)) continue; + const hit = { path: file.path, line, excerpt: body, scope: file.scope }; + // The excerpt is the whole line. A line too long to fit alone is marked, not cut and not refused. + hits.push(bytes({ commit: index.commit, results: [hit], next: Number.MAX_SAFE_INTEGER }) > QUERY_REPLY_CAP + ? { path: file.path, line, excerpt: null, excerpt_too_long: true, scope: file.scope } : hit); + } + } + return fitPage(hits, { + offset: offsetArg(offset), + build: (picked, next) => ({ commit: index.commit, results: picked, next }), + }); +} + +// manuscript: one chapter file of a book, by its file name, up to 400 lines per call from `from` to `to`. +// Without a chapter it returns the book's first chapter file in path order. Every reply carries +// `next_chapter`, the next chapter file's name in path order or null. A name that matches two visible +// files is refused. Files outside every book (book null) are never chapters of a book. +export async function manuscript(source, { universe: name, book, scope, chapter, from, to } = {}) { + const universe = universeArg(source.mapping, name); + const bookName = textArg(book, 'book'); + const chapterName = chapter === undefined ? undefined : textArg(chapter, 'chapter'); + const start = positionArg(from, 'from', 1); + const end = positionArg(to, 'to', Infinity); + if (end < start) invalid('to must not be before from'); + const { index, files } = await visible(source, universe, { scope: scopeArg(scope), book: bookName, type: 'manuscript' }); + let at = 0; + if (chapterName !== undefined) { + const matches = files.filter(file => file.stem === chapterName); + if (matches.length > 1) raise('query-chapter-ambiguous', `${matches.length} visible files are named ${JSON.stringify(chapterName)}`); + if (!matches.length) return null; + at = files.indexOf(matches[0]); + } + const file = files[at]; + if (!file) return null; + const nextChapter = files[at + 1]?.stem ?? null; + return fitPage(file.lines.slice(start - 1, end), { + limit: MANUSCRIPT_LINES, + build: (picked, next) => ({ + path: file.path, commit: index.commit, scope: file.scope, lines: picked, + next: next === null ? null : start + next, next_chapter: nextChapter, + }), + }); +} + +// universeInfo: a universe's kind, name, books, playground prefixes, type folders and manuscript folders. +// A hybrid adds its sources, constraints and notes. A draft has no books and no canon: its kind, name +// and prefixes are the whole answer beside the mapping's types. It reads only the checked mapping and the +// served commit. +export async function universeInfo(source, { universe: name } = {}) { + const universe = universeArg(source.mapping, name); + const entry = source.mapping.universes[universe]; + const hybrid = entry.kind === 'hybrid'; + return checked({ + commit: source.commit, universe, name: entry.name, kind: entry.kind ?? 'universe', + books: entry.books, playground: entry.playground, types: source.mapping.types, manuscript: source.mapping.manuscript, + ...(hybrid ? { + sources: entry.sources, constraints: entry.constraints, + ...(entry.notes === undefined ? {} : { notes: entry.notes }), + } : {}), + }); +} diff --git a/packages/library/tests/index-build.test.mjs b/packages/library/tests/index-build.test.mjs new file mode 100644 index 000000000..f3c694c4f --- /dev/null +++ b/packages/library/tests/index-build.test.mjs @@ -0,0 +1,91 @@ +// Index tests (C2-05a): the cache key and its bound, failed reads, mapped-only contents, and the rebuild +// time on a 3,000-file fixture. Fixtures are made up and built under the scratch area. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { symlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { clearIndexCache, indexCacheSize, indexFor } from '../src/index-build.mjs'; +import { git, mapping, MARKER } from './helpers.mjs'; +import { note, scratchRepo } from './query-fixture.mjs'; + +const FILES = { + 'Book One/Characters/Mara Vell.md': note('character', 'Mara Vell', MARKER), + 'Book One/Chapters/Chapter 01.md': '# One\n[[Mara Vell]]\n', + 'Inbox Note.md': `${MARKER}\n`, +}; + +// A mapping whose harbor universe has the given display name. Each name gives a new cache key. +const named = name => mapping(e => ({ ...e, universes: { ...e.universes, harbor: { ...e.universes.harbor, name } } })); + +test('the same commit, repo path and mapping return the same index, and only mapped files are in it', async t => { + clearIndexCache(); + const { repo, commit } = scratchRepo(t, FILES); + const a = await indexFor({ repo, commit, mapping: mapping() }); + const b = await indexFor({ repo, commit, mapping: mapping() }); + assert.equal(a, b); + assert.equal(indexCacheSize(), 1); + assert.equal(a.commit, commit); + assert.ok(a.files.has('Book One/Characters/Mara Vell.md')); + assert.ok(!a.files.has('Inbox Note.md')); + assert.deepEqual(a.names.get('mara vell'), ['Book One/Characters/Mara Vell.md']); + assert.deepEqual(a.linksIn.get('Book One/Characters/Mara Vell.md').map(l => l.path), ['Book One/Chapters/Chapter 01.md']); +}); + +test('a different repo path, mapping or commit gets its own index', async t => { + clearIndexCache(); + const { root, repo, commit } = scratchRepo(t, FILES); + const base = await indexFor({ repo, commit, mapping: mapping() }); + const alias = join(root, 'alias'); + symlinkSync(repo, alias); + assert.notEqual(await indexFor({ repo: alias, commit, mapping: mapping() }), base); + assert.notEqual(await indexFor({ repo, commit, mapping: named('Renamed Harbor') }), base); + writeFileSync(join(repo, 'Book One/Places/New.md'), note('place', 'New')); + git(repo, 'add', '-A'); + git(repo, 'commit', '-qm', 'second'); + const next = git(repo, 'rev-parse', 'HEAD'); + const later = await indexFor({ repo, commit: next, mapping: mapping() }); + assert.notEqual(later, base); + assert.equal(later.commit, next); + assert.ok(later.files.has('Book One/Places/New.md')); +}); + +test('the cache holds at most eight indexes and evicts the least recently used', async t => { + clearIndexCache(); + const { repo, commit } = scratchRepo(t, FILES); + const maps = Array.from({ length: 9 }, (_, i) => named(`Harbor ${i}`)); + const first = await indexFor({ repo, commit, mapping: maps[0] }); + for (const m of maps.slice(1)) await indexFor({ repo, commit, mapping: m }); + assert.equal(indexCacheSize(), 8); + assert.notEqual(await indexFor({ repo, commit, mapping: maps[0] }), first); + assert.equal(indexCacheSize(), 8); +}); + +test('a failed read is not kept in the cache', async t => { + clearIndexCache(); + const { repo } = scratchRepo(t, FILES); + await assert.rejects(indexFor({ repo, commit: '0'.repeat(40), mapping: mapping() })); + assert.equal(indexCacheSize(), 0); +}); + +test('a missing or unchecked mapping is refused before any read', async t => { + clearIndexCache(); + const { repo, commit } = scratchRepo(t, FILES); + await assert.rejects(indexFor({ repo, commit, mapping: undefined }), { code: 'library-mapping-invalid' }); + await assert.rejects(indexFor({ repo, commit, mapping: { kind: 'hybrid' } }), { code: 'library-mapping-invalid' }); +}); + +test('a 3,000-file rebuild completes and its time is reported', { timeout: 300_000 }, async t => { + clearIndexCache(); + const folders = ['Characters', 'Places', 'Research']; + const files = {}; + for (let i = 0; i < 3000; i++) { + files[`Book One/${folders[i % 3]}/Note ${i}.md`] = note('character', `Note ${i}`, `Body ${i} [[Note ${(i + 1) % 3000}]]`); + } + const { repo, commit } = scratchRepo(t, files); + const started = performance.now(); + const index = await indexFor({ repo, commit, mapping: mapping() }); + const ms = performance.now() - started; + t.diagnostic(`3,000-file index rebuild: ${ms.toFixed(0)} ms`); + assert.equal(index.paths.length, 3000); +}); diff --git a/packages/library/tests/query-fixture.mjs b/packages/library/tests/query-fixture.mjs new file mode 100644 index 000000000..bf81aa9a0 --- /dev/null +++ b/packages/library/tests/query-fixture.mjs @@ -0,0 +1,31 @@ +// Scratch repos for the query and index tests (C2-05a). They are built under tmpdir(), as helpers.mjs +// repo() does; the gate sets TMPDIR under /mnt/storage/scratch/tmp, never /tmp (RULINGS "tmpdir flag"). +// Bodies are made up. +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { git } from './helpers.mjs'; + +// A repo with one commit holding `files` (path -> text). Removed when the test ends. +export function scratchRepo(t, files) { + const root = mkdtempSync(join(tmpdir(), 'r216-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const repo = join(root, 'books'); + mkdirSync(repo); + git(repo, 'init', '-q', '-b', 'main'); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(repo, path)), { recursive: true }); + writeFileSync(join(repo, path), text); + } + git(repo, 'add', '-A'); + git(repo, 'commit', '-qm', 'fixture'); + return { root, repo, commit: git(repo, 'rev-parse', 'HEAD') }; +} + +// A note: front matter with a type and a name, optional extra front matter lines, then a body. +export const note = (type, name, body = '', extra = []) => ['---', `type: ${type}`, `name: ${name}`, ...extra, '---', body, ''].join('\n'); + +// The one exclusion the hybrid tests use, and the paths it hides. C2-15 (row 267) replaces the reading of +// `exclude` (a path prefix today); swapping these two constants is the whole change to the tests. +export const EXCLUSION = { exclude: 'Book One/Research' }; +export const EXCLUDED_PATHS = ['Book One/Research/Ferry Logs.md']; diff --git a/packages/library/tests/query.test.mjs b/packages/library/tests/query.test.mjs new file mode 100644 index 000000000..0c884705b --- /dev/null +++ b/packages/library/tests/query.test.mjs @@ -0,0 +1,369 @@ +// Query tests for design 3 (C2-05a): universe isolation, scope labels, unmapped files, hybrids, paging +// under the reply cap, refusals and manuscript paging. Fixtures are made up and built under the scratch area. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { REPLY_CAP } from '../../harness/src/tools.mjs'; +import { find, get, search, manuscript, universeInfo, QUERY_REPLY_CAP, LOOKUP_LIMIT, MANUSCRIPT_LINES } from '../src/query.mjs'; +import { mapping, MARKER } from './helpers.mjs'; +import { scratchRepo, note, EXCLUSION, EXCLUDED_PATHS } from './query-fixture.mjs'; + +// A hybrid over harbor and second. Its one constraint is EXCLUSION (query-fixture.mjs) and it declares no +// books or playground. +const WITH_BLEND = e => ({ + ...e, + universes: { + ...e.universes, + blend: { + kind: 'hybrid', name: 'Blend', sources: ['harbor', 'second'], books: [], playground: [], + constraints: [EXCLUSION], notes: 'made-up hybrid', + }, + }, +}); + +// PRD 1.14 levels (C2-01c): harbor gets a universe-level playground prefix outside every book, and a draft +// universe has only playground prefixes. Files there have book null. +const WITH_LEVELS = e => ({ + ...WITH_BLEND(e), + universes: { + ...WITH_BLEND(e).universes, + harbor: { ...e.universes.harbor, playground: ['Book One/Playground/', 'Notes/Harbor/'] }, + idea: { kind: 'draft', name: 'New World', playground: ['Drafts/New World/'] }, + }, +}); +const LEVEL_FILES = { + 'Notes/Harbor/Scraps.md': note('summary', 'Scraps', `${MARKER} loose`), + 'Notes/Harbor/Chapters/Idea Chapter.md': `# Idea\n${MARKER} unbooked chapter\n`, + 'Drafts/New World/Characters/Rin.md': note('character', 'Rin', `${MARKER} draft`), + 'Drafts/New World/Summary.md': note('summary', 'Summary', `${MARKER} draft`), +}; + +const FILES = { + 'Book One/Characters/Mara Vell.md': note('character', 'Mara Vell', `Grey eyes. ${MARKER}`, + ['appears_in: "[[Book One/Chapters/Chapter 01]]"', 'relationships:', ' - "[[Tomas]]"']), + 'Book One/Characters/Tomas.md': note('character', 'Tomas', MARKER), + 'Book One/Chapters/Chapter 01.md': `# One\n${MARKER} [[Mara Vell]] met [[Book One/Places/Harbor Town|the harbor]].\n`, + 'Book One/Chapters/Chapter 02.md': 'Quiet. [[Ferry Logs]] [[Ilse]]\n', + 'Book One/Places/Harbor Town.md': note('place', 'Harbor Town', `Salt. ${MARKER}`), + 'Book One/Research/Ferry Logs.md': note('research', 'Ferry Logs', `Timetable. ${MARKER}`), + 'Book One/Playground/What If.md': note('summary', 'What If', `${MARKER} [[Mara Vell]]`), + 'Second World/Characters/Ilse.md': note('character', 'Ilse', `${MARKER} Ilse.`), + 'Second World/Chapters/01.md': '# Elsewhere\n[[Ilse]] waits.\n', + 'Inbox Note.md': `${MARKER}\n`, + 'Book One/Board.canvas': '{"nodes":[]}\n', +}; + +const src = (ref, map = mapping()) => ({ repo: ref.repo, commit: ref.commit, mapping: map }); +const paths = reply => reply.results.map(r => r.path); + +test('the query reply cap equals the harness reply cap', () => { + assert.equal(QUERY_REPLY_CAP, REPLY_CAP); +}); + +test('find is bound to its universe and refuses a call without one', async t => { + const s = src(scratchRepo(t, FILES)); + const names = paths(await find(s, { universe: 'harbor', query: '' })); + assert.ok(names.length > 0); + assert.ok(names.every(p => p.startsWith('Book One/'))); + assert.ok(!names.includes('Second World/Characters/Ilse.md')); + await assert.rejects(find(s, { query: '' }), { code: 'query-argument-invalid' }); + await assert.rejects(find(s, { universe: 'nowhere', query: '' }), { code: 'query-universe-unknown' }); +}); + +test('every result carries its scope, and playground appears only when asked for', async t => { + const s = src(scratchRepo(t, FILES)); + const canon = (await find(s, { universe: 'harbor', query: '' })).results; + assert.ok(canon.every(r => r.scope === 'canon')); + assert.ok(!canon.some(r => r.path.includes('/Playground/'))); + const play = (await find(s, { universe: 'harbor', query: '', scope: 'playground' })).results; + assert.deepEqual(play.map(r => r.path), ['Book One/Playground/What If.md']); + assert.ok(play.every(r => r.scope === 'playground')); +}); + +test('unmapped files never appear in any reply', async t => { + const s = src(scratchRepo(t, FILES)); + assert.deepEqual((await find(s, { universe: 'harbor', query: 'Inbox' })).results, []); + assert.deepEqual((await find(s, { universe: 'harbor', query: 'Board' })).results, []); + const hits = (await search(s, { universe: 'harbor', text: MARKER })).results; + assert.ok(hits.length > 0); + assert.ok(hits.every(r => r.path !== 'Inbox Note.md' && r.path !== 'Book One/Board.canvas')); +}); + +test('a hybrid sees its sources canon, minus exclusions, and no playground', async t => { + const s = src(scratchRepo(t, FILES), mapping(WITH_BLEND)); + const reply = await find(s, { universe: 'blend', query: '' }); + assert.deepEqual(paths(reply).sort(), [ + 'Book One/Characters/Mara Vell.md', 'Book One/Characters/Tomas.md', 'Book One/Chapters/Chapter 01.md', + 'Book One/Chapters/Chapter 02.md', 'Book One/Places/Harbor Town.md', + 'Second World/Characters/Ilse.md', 'Second World/Chapters/01.md', + ]); + assert.ok(reply.results.every(r => r.scope === 'canon' && ['harbor', 'second'].includes(r.universe))); + assert.deepEqual((await find(s, { universe: 'blend', query: '', scope: 'playground' })).results, []); + assert.deepEqual((await find(s, { universe: 'blend', query: 'Ferry' })).results, []); +}); + +test('a hybrid gets a source file it can see, and not an excluded or foreign one', async t => { + const ref = scratchRepo(t, FILES); + const hybrid = src(ref, mapping(WITH_BLEND)); + const ilse = await get(hybrid, { universe: 'blend', path: 'Second World/Characters/Ilse.md' }); + assert.equal(ilse.scope, 'canon'); + assert.ok(ilse.links_in.some(l => l.path === 'Second World/Chapters/01.md')); + assert.equal(await get(hybrid, { universe: 'blend', path: 'Book One/Research/Ferry Logs.md' }), null); + assert.equal(await get(src(ref), { universe: 'harbor', path: 'Second World/Characters/Ilse.md' }), null); +}); + +test('get lists visible links both ways and drops a target another universe cannot see', async t => { + const ref = scratchRepo(t, FILES); + const harbor = src(ref); + const ch1 = await get(harbor, { universe: 'harbor', path: 'Book One/Chapters/Chapter 01.md' }); + assert.ok(ch1.links_out.some(l => l.path === 'Book One/Characters/Mara Vell.md' && l.line === 2)); + assert.ok(ch1.links_out.some(l => l.path === 'Book One/Places/Harbor Town.md')); + const ch2 = await get(harbor, { universe: 'harbor', path: 'Book One/Chapters/Chapter 02.md' }); + assert.ok(ch2.links_out.some(l => l.path === 'Book One/Research/Ferry Logs.md')); + assert.ok(!ch2.links_out.some(l => l.path === 'Second World/Characters/Ilse.md')); + const mara = await get(harbor, { universe: 'harbor', path: 'Book One/Characters/Mara Vell.md' }); + assert.ok(mara.links_in.some(l => l.path === 'Book One/Chapters/Chapter 01.md')); + assert.ok(mara.links_in.some(l => l.path === 'Book One/Playground/What If.md')); + assert.ok(mara.links_out.some(l => l.path === 'Book One/Characters/Tomas.md' && l.field === 'relationships')); + assert.ok(mara.links_out.some(l => l.path === 'Book One/Chapters/Chapter 01.md' && l.field === 'appears_in')); + const blendCh2 = await get(src(ref, mapping(WITH_BLEND)), { universe: 'blend', path: 'Book One/Chapters/Chapter 02.md' }); + assert.ok(blendCh2.links_out.some(l => l.path === 'Second World/Characters/Ilse.md')); +}); + +test('a long list pages by position, never cuts an item, and returns 50 per page', async t => { + const files = { ...FILES }; + for (let i = 1; i <= 120; i++) files[`Book One/Places/Note ${i}.md`] = note('place', `Note ${i}`); + const s = src(scratchRepo(t, files)); + const seen = []; + let offset = 0; + for (let pages = 0; pages < 10; pages++) { + const reply = await find(s, { universe: 'harbor', query: 'Note', offset }); + assert.ok(JSON.stringify(reply).length <= QUERY_REPLY_CAP); + assert.ok(reply.results.length <= LOOKUP_LIMIT); + if (pages === 0) assert.equal(reply.results.length, LOOKUP_LIMIT); + seen.push(...paths(reply)); + if (reply.next === null) break; + offset = reply.next; + } + assert.equal(seen.length, 120); + assert.equal(new Set(seen).size, 120); +}); + +test('a page stays under the reply cap when items are large', async t => { + const big = 'x'.repeat(2000); + const files = { ...FILES }; + for (let i = 1; i <= 40; i++) files[`Book One/Places/Big ${i}.md`] = note('place', `Big ${i}`, '', [`appears_in: "${big}"`]); + const s = src(scratchRepo(t, files)); + const first = await find(s, { universe: 'harbor', query: 'Big' }); + assert.ok(JSON.stringify(first).length <= QUERY_REPLY_CAP); + assert.ok(first.results.length > 0 && first.results.length < 40); + assert.notEqual(first.next, null); + const seen = []; + let offset = 0; + for (let pages = 0; pages < 10; pages++) { + const reply = await find(s, { universe: 'harbor', query: 'Big', offset }); + assert.ok(JSON.stringify(reply).length <= QUERY_REPLY_CAP); + seen.push(...paths(reply)); + if (reply.next === null) break; + offset = reply.next; + } + assert.equal(new Set(seen).size, 40); +}); + +test('a search line too long to fit alone comes back marked, with no excerpt, and the other hits still return', async t => { + const huge = 'y'.repeat(QUERY_REPLY_CAP + 1000); + const s = src(scratchRepo(t, { ...FILES, 'Book One/Places/Wall.md': note('place', 'Wall', `${huge} ${MARKER}`) })); + const reply = await search(s, { universe: 'harbor', text: 'yyyyyyyy' }); + assert.deepEqual(reply.results, [{ path: 'Book One/Places/Wall.md', line: 5, excerpt: null, excerpt_too_long: true, scope: 'canon' }]); + const mixed = await search(s, { universe: 'harbor', text: MARKER.toLowerCase() }); + assert.ok(mixed.results.some(r => r.excerpt_too_long === true)); + assert.ok(mixed.results.some(r => typeof r.excerpt === 'string' && !('excerpt_too_long' in r))); + assert.ok(JSON.stringify(mixed).length <= QUERY_REPLY_CAP); +}); + +test('a find item that cannot fit alone is refused, not cut', async t => { + const huge = 'y'.repeat(QUERY_REPLY_CAP + 1000); + const s = src(scratchRepo(t, { ...FILES, 'Book One/Places/Wall.md': note('place', 'Wall', '', [`appears_in: "${huge}"`]) })); + await assert.rejects(find(s, { universe: 'harbor', query: 'Wall' }), { code: 'query-item-too-large' }); +}); + +test('get pages its lines, and the next line picks up where the page stopped', async t => { + const body = `${Array.from({ length: 500 }, (_, i) => `Line ${i + 1} ${'z'.repeat(120)}`).join('\n')}\n`; + const s = src(scratchRepo(t, { ...FILES, 'Book One/Chapters/Long.md': body })); + const first = await get(s, { universe: 'harbor', path: 'Book One/Chapters/Long.md' }); + assert.equal(first.lines[0][0], 1); + assert.ok(first.lines.length < 501); + assert.notEqual(first.next, null); + let from = 1, count = 0; + for (let pages = 0; pages < 10; pages++) { + const page = await get(s, { universe: 'harbor', path: 'Book One/Chapters/Long.md', from }); + count += page.lines.length; + if (page.next === null) break; + from = page.next; + } + assert.equal(count, 501); +}); + +test('get refuses a file whose front matter alone is over the cap, and a line alone is over the cap', async t => { + const huge = 'q'.repeat(QUERY_REPLY_CAP + 1000); + const s = src(scratchRepo(t, { + ...FILES, + 'Book One/Places/Heavy.md': note('place', 'Heavy', '', [`appears_in: "${huge}"`]), + 'Book One/Chapters/Wide.md': `${huge}\n`, + })); + await assert.rejects(get(s, { universe: 'harbor', path: 'Book One/Places/Heavy.md' }), { code: 'query-item-too-large' }); + await assert.rejects(get(s, { universe: 'harbor', path: 'Book One/Chapters/Wide.md' }), { code: 'query-item-too-large' }); +}); + +test('get pages the links of a hub note, every page carries the front matter, and the pages add up', async t => { + const files = { ...FILES }; + for (let i = 1; i <= 900; i++) files[`Book One/Chapters/Mention ${i}.md`] = `[[Tomas]] ${'w'.repeat(60)}\n`; + const s = src(scratchRepo(t, files)); + const seen = []; + const PAST_END = 1000; + let linksFrom = 0, pages = 0, lines = 0, from = 1; + for (; pages < 40; pages++) { + const page = await get(s, { universe: 'harbor', path: 'Book One/Characters/Tomas.md', from, links_from: linksFrom }); + assert.ok(JSON.stringify(page).length <= QUERY_REPLY_CAP); + assert.equal(page.frontmatter.name, 'Tomas'); + seen.push(...page.links_in.map(l => l.path)); + if (from !== PAST_END) lines += page.lines.length; + // Once the lines are read, ask past the last one: the page then carries links alone. + from = page.next ?? PAST_END; + if (page.links_next === null && page.next === null) break; + if (page.links_next !== null) linksFrom = page.links_next; + } + assert.ok(pages > 1, 'the hub needed more than one page'); + assert.ok(pages < 40); + assert.equal(seen.filter(p => p.startsWith('Book One/Chapters/Mention ')).length, 900); + assert.equal(new Set(seen).size, seen.length, 'no link comes back twice'); + assert.equal(lines, 6, 'the five lines of the note and the empty one after its last newline'); + const tail = await get(s, { universe: 'harbor', path: 'Book One/Characters/Tomas.md', from: PAST_END }); + assert.deepEqual(tail.lines, []); + assert.equal(tail.next, null); +}); + +test('get rejects a bad links_from', async t => { + const s = src(scratchRepo(t, FILES)); + await assert.rejects(get(s, { universe: 'harbor', path: 'Book One/Characters/Tomas.md', links_from: -1 }), { code: 'query-argument-invalid' }); + const past = await get(s, { universe: 'harbor', path: 'Book One/Characters/Tomas.md', links_from: 99 }); + assert.deepEqual([past.links_out, past.links_in, past.links_next], [[], [], null]); +}); + +test('search returns scoped line hits, case-insensitively, and refuses empty text', async t => { + const s = src(scratchRepo(t, FILES)); + const reply = await search(s, { universe: 'harbor', text: 'marker' }); + assert.ok(reply.results.length > 0); + assert.ok(reply.results.every(r => r.scope === 'canon' && r.path.startsWith('Book One/') && Number.isInteger(r.line))); + assert.ok(reply.results.some(r => r.path === 'Book One/Chapters/Chapter 01.md' && r.line === 2)); + await assert.rejects(search(s, { universe: 'harbor', text: '' }), { code: 'query-argument-invalid' }); +}); + +test('a manuscript call returns one chapter, at most 400 lines, and names its next line', async t => { + const long = `${Array.from({ length: 900 }, (_, i) => `Scene ${i + 1}`).join('\n')}\n`; + const s = src(scratchRepo(t, { ...FILES, 'Book One/Chapters/Long Chapter.md': long })); + const first = await manuscript(s, { universe: 'harbor', book: 'Book One', chapter: 'Long Chapter' }); + assert.equal(first.lines.length, MANUSCRIPT_LINES); + assert.equal(first.next, MANUSCRIPT_LINES + 1); + const second = await manuscript(s, { universe: 'harbor', book: 'Book One', chapter: 'Long Chapter', from: first.next }); + assert.equal(second.lines[0][0], MANUSCRIPT_LINES + 1); +}); + +test('a manuscript call without a chapter returns the first chapter in path order and names the next', async t => { + const s = src(scratchRepo(t, FILES)); + const first = await manuscript(s, { universe: 'harbor', book: 'Book One' }); + assert.equal(first.path, 'Book One/Chapters/Chapter 01.md'); + assert.equal(first.next_chapter, 'Chapter 02'); + const last = await manuscript(s, { universe: 'harbor', book: 'Book One', chapter: 'Chapter 02' }); + assert.equal(last.path, 'Book One/Chapters/Chapter 02.md'); + assert.equal(last.next_chapter, null); + assert.equal(await manuscript(s, { universe: 'harbor', book: 'Book One', chapter: 'Nope' }), null); +}); + +test('a chapter name that matches two visible files is refused', async t => { + const s = src(scratchRepo(t, { ...FILES, 'Book One/Chapters/Part/Chapter 01.md': 'Again.\n' })); + await assert.rejects(manuscript(s, { universe: 'harbor', book: 'Book One', chapter: 'Chapter 01' }), { code: 'query-chapter-ambiguous' }); +}); + +test('a hybrid reads a source chapter through its declared sources only', async t => { + const ref = scratchRepo(t, FILES); + const hybrid = await manuscript(src(ref, mapping(WITH_BLEND)), { universe: 'blend', book: 'Second World', chapter: '01' }); + assert.equal(hybrid.path, 'Second World/Chapters/01.md'); + assert.equal(hybrid.scope, 'canon'); + assert.equal(await manuscript(src(ref), { universe: 'harbor', book: 'Second World', chapter: '01' }), null); +}); + +test('universe info reports a hybrid sources, constraints and notes, and refuses an unknown universe', async t => { + const s = src(scratchRepo(t, FILES), mapping(WITH_BLEND)); + const blend = await universeInfo(s, { universe: 'blend' }); + assert.equal(blend.kind, 'hybrid'); + assert.deepEqual(blend.sources, ['harbor', 'second']); + assert.deepEqual(blend.constraints, [EXCLUSION]); + assert.equal(blend.notes, 'made-up hybrid'); + const harbor = await universeInfo(s, { universe: 'harbor' }); + assert.equal(harbor.kind, 'universe'); + assert.equal(harbor.sources, undefined); + await assert.rejects(universeInfo(s, { universe: 'ghost' }), { code: 'query-universe-unknown' }); +}); + +test('an exclusion hides what it names from a hybrid and from nothing else', async t => { + const ref = scratchRepo(t, FILES); + const hybrid = src(ref, mapping(WITH_BLEND)); + const seen = paths(await find(hybrid, { universe: 'blend', query: '' })); + for (const path of EXCLUDED_PATHS) assert.ok(!seen.includes(path), `${path} stays hidden`); + const own = paths(await find(src(ref), { universe: 'harbor', query: '' })); + for (const path of EXCLUDED_PATHS) assert.ok(own.includes(path), `${path} stays visible to its own universe`); +}); + +test('PRD 1.14: a lookup that names a book leaves book-null files out', async t => { + const s = src(scratchRepo(t, { ...FILES, ...LEVEL_FILES }), mapping(WITH_LEVELS)); + const all = paths(await find(s, { universe: 'harbor', query: '', scope: 'playground' })); + assert.deepEqual(all.sort(), ['Book One/Playground/What If.md', 'Notes/Harbor/Chapters/Idea Chapter.md', 'Notes/Harbor/Scraps.md']); + const booked = await find(s, { universe: 'harbor', query: '', scope: 'playground', book: 'Book One' }); + assert.deepEqual(paths(booked), ['Book One/Playground/What If.md']); + assert.ok(booked.results.every(r => r.book === 'Book One')); + const hits = (await search(s, { universe: 'harbor', text: 'unbooked', scope: 'playground' })).results; + assert.deepEqual(hits.map(r => r.path), ['Notes/Harbor/Chapters/Idea Chapter.md']); + assert.deepEqual((await search(s, { universe: 'harbor', text: 'unbooked', scope: 'playground', book: 'Book One' })).results, []); +}); + +test('PRD 1.14: library_manuscript never reads a book-null file', async t => { + const s = src(scratchRepo(t, { ...FILES, ...LEVEL_FILES }), mapping(WITH_LEVELS)); + // The unbooked file sits under a manuscript folder, so it has type manuscript and book null. + const listed = await find(s, { universe: 'harbor', query: 'Idea', scope: 'playground' }); + assert.deepEqual(listed.results.map(r => [r.path, r.type, r.book]), [['Notes/Harbor/Chapters/Idea Chapter.md', 'manuscript', null]]); + assert.equal(await manuscript(s, { universe: 'harbor', book: 'Book One', scope: 'playground', chapter: 'Idea Chapter' }), null); + assert.equal(await manuscript(s, { universe: 'harbor', book: 'Book One', scope: 'playground' }), null); + assert.equal(await manuscript(s, { universe: 'harbor', book: 'Notes', scope: 'playground', chapter: 'Idea Chapter' }), null); +}); + +test('PRD 1.14: a draft universe has no canon, so its lookups return files only with scope playground', async t => { + const s = src(scratchRepo(t, { ...FILES, ...LEVEL_FILES }), mapping(WITH_LEVELS)); + assert.deepEqual((await find(s, { universe: 'idea', query: '' })).results, []); + assert.deepEqual((await search(s, { universe: 'idea', text: MARKER })).results, []); + const found = await find(s, { universe: 'idea', query: '', scope: 'playground' }); + assert.deepEqual(paths(found).sort(), ['Drafts/New World/Characters/Rin.md', 'Drafts/New World/Summary.md']); + assert.ok(found.results.every(r => r.scope === 'playground' && r.universe === 'idea' && r.book === null)); + const hits = (await search(s, { universe: 'idea', text: 'draft', scope: 'playground' })).results; + assert.equal(hits.length, 2); + const rin = await get(s, { universe: 'idea', path: 'Drafts/New World/Characters/Rin.md' }); + assert.equal(rin.scope, 'playground'); + assert.equal(await get(s, { universe: 'harbor', path: 'Drafts/New World/Characters/Rin.md' }), null); +}); + +test('PRD 1.14: library_universe on a draft returns its kind, name and prefixes', async t => { + const s = src(scratchRepo(t, { ...FILES, ...LEVEL_FILES }), mapping(WITH_LEVELS)); + const draft = await universeInfo(s, { universe: 'idea' }); + assert.equal(draft.kind, 'draft'); + assert.equal(draft.name, 'New World'); + assert.deepEqual(draft.playground, ['Drafts/New World/']); + assert.deepEqual(draft.books, []); + assert.equal(draft.sources, undefined); +}); + +test('a hybrid gets a source playground file as null, while the source gets it with scope playground', async t => { + const ref = scratchRepo(t, FILES); + const path = 'Book One/Playground/What If.md'; + assert.equal(await get(src(ref, mapping(WITH_BLEND)), { universe: 'blend', path }), null); + const own = await get(src(ref), { universe: 'harbor', path }); + assert.equal(own.scope, 'playground'); +}); From 38e322ea0155a3513c3cc73a3db8dcac1247c2e2 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 22 Oct 2026 11:45:58 -0500 Subject: [PATCH 2/3] row224: draft broker library lookups and typed tools (unrun) --- agents/della/work/queue-224/BUILD.md | 141 +++++++++++ packages/bus/src/broker.mjs | 43 +++- packages/bus/src/runtime.mjs | 30 +++ packages/bus/tests/library-lookups.test.mjs | 226 ++++++++++++++++++ packages/cli/src/launcher.mjs | 1 + packages/cli/tests/thread-launch.test.mjs | 22 ++ packages/harness/src/bundle.mjs | 5 +- packages/harness/src/tools.mjs | 36 ++- packages/harness/tests/library-tools.test.mjs | 69 ++++++ 9 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 agents/della/work/queue-224/BUILD.md create mode 100644 packages/bus/tests/library-lookups.test.mjs create mode 100644 packages/harness/tests/library-tools.test.mjs diff --git a/agents/della/work/queue-224/BUILD.md b/agents/della/work/queue-224/BUILD.md new file mode 100644 index 000000000..8c7faf6f3 --- /dev/null +++ b/agents/della/work/queue-224/BUILD.md @@ -0,0 +1,141 @@ +# Row 224 — C2-05b broker lookups and typed tools (#1725) + +Owner: Della; reviewer: Dewey. Assignment sage-2795; scope extension sage-2804. +Base: `4f3fbc25a` (Researcher C2-05a). Branch: `della/row224`. +Brief: C2-05b in `docs/plans/2026-10-20_case2-rows.md` at `aca16404c` +and `51cc0445c`; design section 3, 5.3 M3 and 7.2; C2-05a BUILD rulings. + +## Execution boundary and progress + +Code/docs and local Git only while row 253 holds the gate. No Node (including +syntax checks), HTTPS Git, queue claim, test or push until Sage's CLEAR. +Tests are written first; their RED/GREEN runs are deferred, not claimed. +Queue next is deferred too: its executable uses Node. Sage assigned this row +explicitly. After 253 and 216 land, rebase under a new branch name. + +- Pre-flight: query.mjs consumes `{repo,commit,mapping}`; broker cannot import + Git/filesystem library code because it is in the sandbox import closure. + Runtime injects the host lookup adapter; existing human file/status/sync stay. +- Pre-flight: resolved has no project list and a library is never a checkout. + Bundle needs an explicit library flag from launcher business configuration. +- M3 reading: exact creative base is five lookups plus send_message; retain + explicitly granted broker web_search/web_fetch for research/companion under + landed C2-08 and sage-2804. No generic spec, task, evidence or decision tools. Residual (mos-1308): those + reads lack broker action checks; their typed-tool absence is the only fence + for creative roles. M1/M2 deny bash; broker-side refusal is outside this row. + Mos confirmed this interpretation in mos-1308; design wording fix belongs to Gosalyn. Other definitions retain their current tools. +- Multi-library businesses use an optional project ID; omit it only with one + configured library. Universe is required for all five tools, including get. +- Missing library uses the brief's `library-not-configured`; argument errors + reuse `invalid-request` and C2-05a query codes. No new action or authority. + +## Manifest (draft code, unrun) + +- bus/src/broker.mjs: five lookup verbs, strict args, holder/business rechecks. +- bus/src/runtime.mjs: host lookup adapter, stale-before-read sync, served pin. +- harness/src/tools.mjs: five schemas; creative exact-set filtering. +- harness/src/bundle.mjs: library flag forwarded to typedTools. +- cli/src/launcher.mjs: one flag derived from trusted business projects. +- bus/tests/library-lookups.test.mjs: real query/broker/socket/runtime cases. +- harness/tests/library-tools.test.mjs: schemas, role snapshots, bundle parity. +- cli/tests/thread-launch.test.mjs: flag reaches a real launched bundle with no library checkout. +- agents/della/work/queue-224/BUILD.md: manifest, rulings and deferred verification ledger. + +## Verification ledger + +No test or syntax-check run yet. Acceptance requires full bus/harness suites +and every scripts/test-*.sh on Sage's gate; focused files and affected CLI/ +seat import-closure tests precede that only in an authorized CLEAR. + +## Coordination + +Huey272 reports no runtime/bundle/launcher overlap. Rocko219 changes launcher +THREAD_DEFINITIONS checks only, separate from our buildBundle flag argument. +The launcher argument is required: bundle cannot derive library presence from +checkout IDs (a library has no agent clone). No role, authority or config schema +changes; library project entries are already validated and protected by M9. + +## Mount and pin impact (inspection only) + +No new imports in broker.mjs or tools.mjs; no RUNNER_FILES or sandbox mount +addition. runtime.mjs is host-only and already imports library index.mjs. +Bundle forwards only a boolean, never a root/mapping/slot/credential. The +existing import-closure tests are bus/tests/project-runtime.test.mjs and +seat/tests/sandbox.test.mjs. Grep found no literal sha256 pin for runtime.mjs +or bundle.mjs; bundle tests derive current source digests through SOURCES. +Library sources remain in the existing whole-directory release copy. + +## Runs to schedule only after CLEAR + +Set TMPDIR=/mnt/storage/scratch/tmp. Focused new bus/harness files first, +then existing library.test.mjs, library-sync.test.mjs, project-runtime.test.mjs, +harness creative/tools/spec-tools/bundle files and affected CLI launch tests. +Follow with the required whole bus/harness suites and Sage's full shell gate. +No timing claim yet: use the repository/Sage per-file limits and record every +command, outcome, elapsed time and timeout before acceptance. No focused skip +or pattern run substitutes for whole-file gate evidence. + +## Read-through findings and residuals + +- Library get requires universe although the older design table omitted it; + section 3 isolation rule and C2-05a's accepted API are binding. +- Query type enum matches mapping.TYPES (five values); manuscript is a distinct + lookup and not a find type. Unknown host-path/commit/business/accept fields + are refused; explicit project is restricted to this business's library list. +- Async reads recheck the live holder and current project revision. A stale + read attempts sync, then takes the new served snapshot; a returned fetch + failure/divergence keeps serving the old pin. Busy sync uses the current pin. + Unreadable state, no served commit or unsafe configuration refuse. Query + and sync never hold a SQLite transaction over await. +- Processing halt refuses new agent lookup work and rechecks before returning + async results. Existing human library file/status/sync behavior is unchanged. +- Timer sync and reads still use C2-03a's 60-second contract. No timer tuning or + resource-limit schema is introduced by this row. Q222 creative reply settings + are in C2-06; no reply guard or writing capability is implemented here. +- The generic tool socket's existing 60-second client timeout can precede the + server's 120-second Git timeout. This row does not change the generic timeout; + a timed-out lookup is read-only and retryable, never a write/outcome claim. +- All tests are authored but unrun. No RED, GREEN, syntax, runtime performance, + sandbox, broker/harness suite, shell gate, claim or push evidence exists yet. + +## Candidate review checklist (read-through only) + +- [x] Five verbs mapped to real C2-05a methods, all require universe. +- [x] Default sole library or explicit owned project; no caller repo/commit. +- [x] Human-only file/status/sync gates kept; lookup holder rechecked. +- [x] Async socket dispatch includes lookup verbs through LIBRARY_VERBS. +- [x] Stale/unknown fetch syncs first; results use current immutable served pin. +- [x] Six/eight creative names per mos-1308; other roles retain their tools. +- [x] Launcher -> bundle -> typedTools passes presence without a library clone. +- [x] No imports added to sandbox source closure; no mount change. +- [ ] Execute new tests, RED characterization, full files/suites after CLEAR. +- [ ] Rebase after rows 253/216 land, under a new branch name. +- [ ] Claim/push only in Sage-authorized gap; Dewey review and Sage gate. + +Candidate preservation commits are explicitly unrun under sage-2795; they are +not completion, green-suite or landing claims. No dependencies installed. + +Correction: the first committed-range diff --check found a blank line at EOF +in the new library-tools test, missed while it was untracked. Removed in a +follow-up local commit; the complete range must be checked again. No tests ran. + +## Shared-log receipts for Sage (sage-2815) + +The shared logs are unchanged from the base in this row. Sage appends these +receipts in the landing sweep; they are preserved here, not in the manifest. + +For `BUILD-LOG.md`: + +```text +2026-10-22T16:36:30Z | Della row224 C2-05b #1725 preparation | Before: C2-05a queries exist at 4f3fbc25a but no broker lookup verbs or typed tools. Scope: broker/tools plus Sage-authorized runtime/bundle integration, tests and docs. Existing human-only library reads retained; five agent queries require a live holder and trusted configured source. Code/docs only under row253 gate; no Node/test/push/claim. Runtime ownership coordinated with Rocko219/Huey272. +``` + +For `docs/SESSIONS.md`: + +```text +2026-10-22T16:36:30Z | Della (sage-2795 / sage-2804, Q220) | Row 224 C2-05b #1725, della/row224 base 4f3fbc25a | Read both pinned briefs and C2-05a source; preparing broker/typed tools and authorized host runtime/bundle seams. No queue command, Node, HTTPS Git, claim, test or push during Gander gate. Dewey reviewer; Rocko/Huey runtime coordination sent. Tests and integration verification deferred to CLEAR. +``` + +Local-only draft pins 19c0a7adf/7cf39c390 are superseded by the +log-free candidate. Saved as local branch della/row224-draft-with-logs before +rebuilding the unpublished candidate commit; no remote history changed. diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index c31edd154..547bd9c90 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -96,7 +96,16 @@ export const TASK_PAGE_MAX = 200; // request allowed past the ordinary 64 KiB envelope (server.mjs and client.mjs), up to its content cap. export const PROJECT_FILE_VERBS = Object.freeze(['project.files', 'project.file', 'project.file.save']); // C2-03a: library.sync is the human's fetch, and with `accept` the one way past library-diverged. -export const LIBRARY_VERBS = Object.freeze(['library.file', 'library.status', 'library.sync']); +// Row 224: pure agent lookups are separate from the three human-only verbs. Listing them +// here also makes server.mjs await their host effects instead of serializing a Promise as {}. +const LIBRARY_LOOKUPS = Object.freeze({ + 'library.find': { method: 'find', fields: ['universe', 'query', 'type', 'book', 'scope', 'offset'], required: ['universe', 'query'] }, + 'library.get': { method: 'get', fields: ['universe', 'path', 'from', 'links_from'], required: ['universe', 'path'] }, + 'library.search': { method: 'search', fields: ['universe', 'text', 'book', 'scope', 'offset'], required: ['universe', 'text'] }, + 'library.manuscript': { method: 'manuscript', fields: ['universe', 'book', 'scope', 'chapter', 'from', 'to'], required: ['universe', 'book'] }, + 'library.universe': { method: 'universeInfo', fields: ['universe'], required: ['universe'] }, +}); +export const LIBRARY_VERBS = Object.freeze(['library.file', 'library.status', 'library.sync', ...Object.keys(LIBRARY_LOOKUPS)]); export const PROJECT_SAVE_VERB = 'project.file.save'; export const PROJECT_SAVE_REQUEST_MAX = 1024 * 1024 + 64 * 1024; export const WEB_VERBS =Object.freeze(['web.search', 'web.fetch']); @@ -4601,10 +4610,42 @@ export class Broker { throw this.#refused(s, ProjectFailure && e instanceof ProjectFailure ? new BusError(e.code) : e, request); } } + // Row 224 C2-05b: the agent never supplies a host repo, commit or business. Its current + // holder and business entry are rechecked before/after awaits; runtime owns Git and queries. + async #requestLibraryLookup(s, { verb, args }) { + try { + this.#agent(s); + const business = this.#business(s.business); + const projects = Object.entries(business.projects ?? {}).filter(([, entry]) => entry.kind === 'library'); + if (!projects.length || typeof this.#library?.lookup !== 'function') fail('library-not-configured'); + const spec = LIBRARY_LOOKUPS[verb]; + keys(args, ['project', ...spec.fields], spec.required); + if (!text(args.universe, 160)) fail('invalid-request'); + if (args.project === undefined && projects.length !== 1) fail('invalid-request'); + const project = args.project ?? projects[0][0]; + if (!text(project, 160) || !projects.some(([name]) => name === project)) fail('library-not-configured'); + const revision = JSON.stringify(business.projects[project]); + const authorize = () => { + this.#agent(s); + this.#processingGate(s, verb); + if (JSON.stringify(this.#business(s.business).projects?.[project]) !== revision) fail('library-not-configured'); + }; + const { project: ignored, ...query } = args; + authorize(); + const result = await this.#library.lookup({ business, project }, { method: spec.method, args: query, authorize }); + authorize(); + this.#secretCheck(result); + return result; + } catch (error) { + const Failure = this.#library?.LibraryError; + throw this.#refused(s, Failure && error instanceof Failure ? new BusError(error.code) : error, { verb, args }); + } + } // C2-04 M4: only a verified human reads library text or status. Host Git // functions stay injected, preserving this module's sandbox import closure. async #requestLibrary(s, request) { const { verb, args } = request; + if (Object.hasOwn(LIBRARY_LOOKUPS, verb)) return this.#requestLibraryLookup(s, request); try { this.#human(s); const fields = verb === 'library.file' ? ['project', 'path'] : verb === 'library.sync' ? ['project', 'accept'] : ['project']; diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index daa0bc65e..a16ab2ac6 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -84,6 +84,34 @@ export async function harnessLevels({ log = (e) => process.stderr.write(`plugins } return (name) => levels.get(name) ?? null; } +// Row 224 C2-05b: a host-only adapter. A failed fetch keeps serving immutable old +// objects, as C2-03a requires; no caller may select a repo, commit or sync --accept. +export function createLibraryLookup({ dataRoot, sync, now = () => Date.now() }) { + const methods = { find: libraryModule.find, get: libraryModule.get, search: libraryModule.search, + manuscript: libraryModule.manuscript, universeInfo: libraryModule.universeInfo }; + return async (context, { method, args, authorize }) => { + if (!Object.hasOwn(methods, method)) throw new libraryModule.LibraryError('query-argument-invalid', 'unknown lookup'); + authorize(); + const source = { ...context, dataRoot }; + let state = await libraryModule.libraryStatus(source); + authorize(); + const age = state.lastFetch === null ? Infinity : now() - Date.parse(state.lastFetch); + if (age > libraryModule.SYNC_INTERVAL_MS || age < 0) { + try { await sync(source); } + catch (error) { + // A timer may already be fetching this mirror. Its current served state is still safe. + if (!(error instanceof libraryModule.LibraryError) || error.code !== 'library-sync-busy') throw error; + } + authorize(); + state = await libraryModule.libraryStatus(source); + } + if (!state.served) throw new libraryModule.LibraryError('library-commit', 'library has never been synced'); + authorize(); + return methods[method]({ repo: libraryModule.mirrorPath(source), commit: state.served, + mapping: libraryModule.checkMapping(context.business.projects[context.project]) }, args); + }; +} + export async function startBroker({ dataRoot, businesses, @@ -273,6 +301,8 @@ export async function startBroker({ store, businesses, library: { + lookup: createLibraryLookup({ dataRoot, sync: context => + processing.work(() => libraryModule.sync(context, syncOptions(context.business))) }), readFile: context => libraryModule.readLibraryFile({ ...context, dataRoot }), status: context => libraryModule.libraryStatus({ ...context, dataRoot }), sync: (context, { accept }) => processing.work(() => libraryModule.sync({ ...context, dataRoot }, { ...syncOptions(context.business), accept })), diff --git a/packages/bus/tests/library-lookups.test.mjs b/packages/bus/tests/library-lookups.test.mjs new file mode 100644 index 000000000..d01d0e9ef --- /dev/null +++ b/packages/bus/tests/library-lookups.test.mjs @@ -0,0 +1,226 @@ +// Row 224: exercise broker authorization, real C2-05a queries and host served snapshots. +// All bodies and remotes are made up. External sync is replaced only at its trusted host seam. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { Store } from '../src/store.mjs'; +import { Broker } from '../src/broker.mjs'; +import { createLibraryLookup, startBroker } from '../src/runtime.mjs'; +import { serve } from '../src/server.mjs'; +import { Client } from '../src/client.mjs'; +import * as library from '../../library/src/index.mjs'; +import { scratchRepo, note } from '../../library/tests/query-fixture.mjs'; +import { mappingEntry, git } from '../../library/tests/helpers.mjs'; + +const NOW = Date.parse('2026-10-22T16:00:00.000Z'); +const VERBS = ['library.find', 'library.get', 'library.search', 'library.manuscript', 'library.universe']; +const GET = { universe: 'harbor', path: 'Book One/Characters/Tomas.md' }; +const ARGS = [ + { universe: 'harbor', query: 'Tomas' }, GET, + { universe: 'harbor', text: 'ferry' }, { universe: 'harbor', book: 'Book One' }, { universe: 'harbor' }, +]; +const TEXTS = { + 'Book One/Characters/Tomas.md': note('character', 'Tomas', 'He runs the ferry. [[Elsewhere]]'), + 'Book One/Chapters/01.md': '# A ferry\nOne line.\n', + 'Book One/Playground/What If.md': note('character', 'Maybe Tomas', 'Unwritten ferry.'), + 'Second World/Characters/Elsewhere.md': note('character', 'Elsewhere', 'Other ferry.'), +}; + +async function world(t, { projects, adapter, claimed = true, now = () => NOW } = {}) { + const f = scratchRepo(t, TEXTS), dataRoot = join(f.root, 'data'); + mkdirSync(dataRoot); + const business = { id: 'acme', human: 'jason', vars: { 'bus.tester': true }, + projects: projects ?? { books: mappingEntry() }, arbiters: { technical: 'coder', delivery: 'coder' }, + roles: { coder: { definition: 'coder', authority: { withinRole: [], crossRole: [] } } } }; + const context = { business, project: 'books', dataRoot }; + let mirror = null, state = null, fetches = 0; + if (business.projects.books?.kind === 'library') { + mirror = await library.ensureMirror(context); + git(f.repo, '-c', 'protocol.file.allow=always', 'push', '-q', mirror, 'HEAD:refs/heads/served'); + state = { version: 1, branch: 'main', served: f.commit, lastFetch: new Date(NOW).toISOString(), + lastError: null, diverged: null, problems: [] }; + writeFileSync(join(mirror, library.SYNC_STATE_FILE), JSON.stringify(state), { mode: 0o600 }); + } + const setState = patch => { + state = { ...state, ...patch }; + writeFileSync(join(mirror, library.SYNC_STATE_FILE), JSON.stringify(state), { mode: 0o600 }); + }; + const lookup = createLibraryLookup({ dataRoot, now, sync: async () => { + fetches++; + setState({ lastFetch: new Date(now()).toISOString() }); + return { state, code: null }; + } }); + const store = new Store(dataRoot); + t.after(() => store.close()); + const broker = new Broker({ store, businesses: { acme: business }, + library: adapter ?? { lookup, LibraryError: library.LibraryError } }); + const agent = broker.bindLaunch({ business: 'acme', role: 'coder', run: 'r224', harness: 'pi' }); + if (claimed) broker.request(agent, { verb: 'role.claim' }); + const human = broker.bindHuman({ business: 'acme', human: 'jason', via: 'cli', outsideAgent: true }); + const reader = broker.bindReader({ business: 'acme' }), tester = broker.bindTester({ business: 'acme' }); + const call = (verb, args = {}, cap = agent) => Promise.resolve().then(() => broker.request(cap, { verb, args })); + return { ...f, dataRoot, business, context, mirror, store, broker, agent, human, reader, tester, + call, lookup, setState, fetches: () => fetches }; +} + +test('all five agent verbs read the configured served commit through real queries', async t => { + const f = await world(t); + const [find, get, search, manuscript, universe] = await Promise.all(VERBS.map((verb, i) => f.call(verb, ARGS[i]))); + for (const result of [find, get, search, manuscript, universe]) assert.equal(result.commit, f.commit); + assert.deepEqual(find.results.map(x => x.path), ['Book One/Characters/Tomas.md']); + assert.match(JSON.stringify(get.lines), /He runs the ferry/); + assert.deepEqual(get.links_out, []); // The only target is in another universe. + assert.ok(search.results.length > 0); + assert.ok(search.results.every(x => !x.path.startsWith('Second World/') && x.scope === 'canon')); + assert.equal(manuscript.path, 'Book One/Chapters/01.md'); + assert.deepEqual(universe.books, ['Book One']); + assert.equal(f.fetches(), 0); + const playground = await f.call('library.find', { universe: 'harbor', query: '', scope: 'playground' }); + assert.deepEqual(playground.results.map(x => x.path), ['Book One/Playground/What If.md']); + assert.equal(await f.call('library.get', { universe: 'harbor', path: 'Second World/Characters/Elsewhere.md' }), null); +}); + +test('each verb refuses missing universe and unknown keys before host work', async t => { + let calls = 0; + const f = await world(t, { adapter: { lookup: () => { calls++; } } }); + for (const [i, verb] of VERBS.entries()) { + const { universe, ...missing } = ARGS[i]; + await assert.rejects(f.call(verb, missing), { code: 'invalid-request' }); + for (const extra of [{ repo: '/elsewhere' }, { commit: 'b'.repeat(40) }, { business: 'other' }, { accept: 'b'.repeat(40) }]) + await assert.rejects(f.call(verb, { ...ARGS[i], ...extra }), { code: 'invalid-request' }); + await assert.rejects(f.call(verb, { ...ARGS[i], universe: '' }), { code: 'invalid-request' }); + } + assert.equal(calls, 0); +}); + +test('no library, missing host adapter and undeclared project refuse library-not-configured', async t => { + const noLibrary = await world(t, { projects: {} }); + const noAdapter = await world(t, { adapter: {} }); + const f = await world(t); + for (const [i, verb] of VERBS.entries()) { + await assert.rejects(noLibrary.call(verb, ARGS[i]), { code: 'library-not-configured' }); + await assert.rejects(noAdapter.call(verb, ARGS[i]), { code: 'library-not-configured' }); + await assert.rejects(f.call(verb, { ...ARGS[i], project: 'other' }), { code: 'library-not-configured' }); + } + f.business.projects.second = mappingEntry(); + await assert.rejects(f.call('library.get', GET), { code: 'invalid-request' }); + assert.equal((await f.call('library.get', { ...GET, project: 'books' })).commit, f.commit); +}); + +test('reads need the current holder on each call; human/tester/reader are not agents', async t => { + const f = await world(t, { claimed: false }); + for (const [i, verb] of VERBS.entries()) { + await assert.rejects(f.call(verb, ARGS[i]), { code: 'not-holder' }); + await assert.rejects(f.call(verb, ARGS[i], f.human), { code: 'agent-required' }); + await assert.rejects(f.call(verb, ARGS[i], f.tester), { code: 'tester-refused' }); + await assert.rejects(f.call(verb, ARGS[i], f.reader), { code: 'read-only' }); + } + f.broker.request(f.agent, { verb: 'role.claim' }); + assert.equal((await f.call('library.get', GET)).commit, f.commit); + f.broker.request(f.agent, { verb: 'role.release' }); + await assert.rejects(f.call('library.get', GET), { code: 'not-holder' }); +}); + +test('releasing a claim during an async lookup fences the response', { timeout: 30_000 }, async t => { + let release, entered; + const started = new Promise(resolve => { entered = resolve; }); + const f = await world(t, { adapter: { lookup: async () => { + entered(); await new Promise(resolve => { release = resolve; }); return { lines: [[1, 'not returned']] }; + } } }); + const pending = f.call('library.get', GET); + await started; + f.broker.request(f.agent, { verb: 'role.release' }); + release(); + await assert.rejects(pending, { code: 'not-holder' }); +}); + +test('a business library removed during an async read cannot leak its result', { timeout: 30_000 }, async t => { + let release, entered; + const started = new Promise(resolve => { entered = resolve; }); + const f = await world(t, { adapter: { lookup: async () => { + entered(); await new Promise(resolve => { release = resolve; }); return { lines: [[1, 'not returned']] }; + } } }); + const pending = f.call('library.get', GET); + await started; + delete f.business.projects.books; + release(); + await assert.rejects(pending, { code: 'library-not-configured' }); +}); + +test('freshness boundary, unknown last fetch and failed fetch still use immutable served state', async t => { + const f = await world(t); + f.setState({ lastFetch: new Date(NOW - 60_000).toISOString() }); + await f.call('library.get', GET); + assert.equal(f.fetches(), 0); + f.setState({ lastFetch: new Date(NOW - 60_001).toISOString() }); + await f.call('library.get', GET); + assert.equal(f.fetches(), 1); + f.setState({ lastFetch: null }); + await f.call('library.universe', { universe: 'harbor' }); + assert.equal(f.fetches(), 2); + let attempted = 0; + const failing = createLibraryLookup({ dataRoot: f.dataRoot, now: () => NOW, sync: async () => { + attempted++; + f.setState({ lastError: { code: 'library-fetch-failed', at: new Date(NOW).toISOString() } }); + return { code: 'library-fetch-failed' }; + } }); + f.setState({ lastFetch: null }); + const result = await failing(f.context, { method: 'get', args: GET, authorize() {} }); + assert.equal(attempted, 1); + assert.equal(result.commit, f.commit); +}); + +test('an unreadable sync state fails closed; a busy sync keeps serving the existing pin', async t => { + const f = await world(t); + const lookup = createLibraryLookup({ dataRoot: f.dataRoot, now: () => NOW, sync: async () => { + throw new library.LibraryError('library-sync-busy', 'another fetch'); + } }); + f.setState({ lastFetch: null }); + assert.equal((await lookup(f.context, { method: 'get', args: GET, authorize() {} })).commit, f.commit); + writeFileSync(join(f.mirror, library.SYNC_STATE_FILE), '{bad json', { mode: 0o600 }); + await assert.rejects(f.call('library.get', GET), { code: 'library-state-invalid' }); +}); + +test('a stale lookup sees the commit published by its sync, not the prior served commit', async t => { + const f = await world(t); + writeFileSync(join(f.repo, GET.path), note('character', 'Tomas', 'Changed ferry.')); + git(f.repo, 'add', '-A'); git(f.repo, 'commit', '-qm', 'second fixture'); + const next = git(f.repo, 'rev-parse', 'HEAD'); + const lookup = createLibraryLookup({ dataRoot: f.dataRoot, now: () => NOW, sync: async () => { + git(f.repo, '-c', 'protocol.file.allow=always', 'push', '-q', f.mirror, 'HEAD:refs/heads/served'); + f.setState({ served: next, lastFetch: new Date(NOW).toISOString() }); + return { code: null }; + } }); + f.setState({ lastFetch: null }); + const result = await lookup(f.context, { method: 'get', args: GET, authorize() {} }); + assert.equal(result.commit, next); + assert.match(JSON.stringify(result.lines), /Changed ferry/); +}); + +test('real socket awaits lookup results and rechecks rejected query arguments', { timeout: 30_000 }, async t => { + const f = await world(t); + const path = join(f.root, 'lookup.sock'); + const server = await serve({ broker: f.broker, path }); t.after(() => server.close()); + const client = new Client({ path, cap: f.agent }); + const result = await client.call('library.get', GET); + assert.equal(result.commit, f.commit); + await assert.rejects(client.call('library.find', { universe: 'harbor', query: '', scope: 'anything' }), /query-argument-invalid/); +}); + +test('runtime actually injects the lookup adapter, without exposing human library.file', { timeout: 30_000 }, async t => { + const f = scratchRepo(t, TEXTS), dataRoot = join(f.root, 'runtime'); mkdirSync(dataRoot); + const business = { id: 'acme', human: 'jason', projects: { books: mappingEntry() }, + arbiters: { technical: 'coder', delivery: 'coder' }, + roles: { coder: { definition: 'coder', authority: { withinRole: [], crossRole: [] } } } }; + const runtime = await startBroker({ dataRoot, businesses: { acme: business } }); t.after(() => runtime.close()); + const mirror = library.mirrorPath({ business, project: 'books', dataRoot }); + git(f.repo, '-c', 'protocol.file.allow=always', 'push', '-q', mirror, 'HEAD:refs/heads/served'); + writeFileSync(join(mirror, library.SYNC_STATE_FILE), JSON.stringify({ version: 1, branch: 'main', served: f.commit, + lastFetch: new Date().toISOString(), lastError: null, diverged: null, problems: [] }), { mode: 0o600 }); + const cap = runtime.broker.bindLaunch({ business: 'acme', role: 'coder', run: 'r224-runtime', harness: 'pi' }); + runtime.broker.request(cap, { verb: 'role.claim' }); + assert.equal((await runtime.broker.request(cap, { verb: 'library.get', args: GET })).commit, f.commit); + await assert.rejects(runtime.broker.request(cap, { verb: 'library.file', args: { project: 'books', path: GET.path } }), + { code: 'human-required' }); +}); diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index 26f337a2d..826b5cbff 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -439,6 +439,7 @@ export function createLauncher({ contract: readFileSync(resolved.contract, "utf8"), arbiter: Object.values(b.arbiters).includes(instance), tracker, + library: Object.values(b.projects ?? {}).some(project => project.kind === "library"), credentials, session: { run, launchedBy, workspace, toolSocket, turnMarker, checkouts }, harness, diff --git a/packages/cli/tests/thread-launch.test.mjs b/packages/cli/tests/thread-launch.test.mjs index 74a34e1a9..26b119b70 100644 --- a/packages/cli/tests/thread-launch.test.mjs +++ b/packages/cli/tests/thread-launch.test.mjs @@ -27,6 +27,7 @@ import { bootConfig, loadSystem } from "../src/config.mjs"; import { startTimeOf } from "../src/host.mjs"; import { createLauncher } from "../src/launcher.mjs"; import { fakeHarnesses, fixture, tmp } from "./helpers.mjs"; +import { mappingEntry } from "../../library/tests/helpers.mjs"; // Row 264a (#1769): a thread made by hand gets the default session the 3k-3l migration gives a // legacy thread, with the thread's id. const DEFAULT_SESSION = [ @@ -442,3 +443,24 @@ test("REQ-LIMIT-4 a thread run at its token limit stops at the end of its turn: assert.deepEqual(threadRuns(f, A), []); assert.equal(host.runtime.threadsState("acme").find((x) => x.thread === A).holder, null); }); + +// Row224: the launcher must derive library presence from business projects, not +// from session checkouts (a library never has an agent clone). +test("C2-05b a real launched bundle gets lookups for a library project without a library checkout", { skip, timeout: 60000 }, async t => { + const prepared = prepare(t, { idle: 10, edit: doc => { + doc.projects.books = mappingEntry(); + return doc; + } }); + const { f } = prepared; + const host = await start(t, prepared); + const project = host.call("project.create", { name: "library work", repo: "stack" }).id; + const thread = host.call("thread.create", { project, name: "lookup" }).id; + await ask(host.call, thread, "inspect bundle"); + const run = launchLog(f).find(e => e.event === "launch" && e.thread === thread).run; + const tools = JSON.parse(readFileSync(join(f.dataRoot, "launches", "acme", run, "bundle", "tools.json"), "utf8")); + assert.deepEqual(tools.filter(tool => tool.name.startsWith("library_")).map(tool => tool.name), + ["library_find", "library_get", "library_search", "library_manuscript", "library_universe"]); + const [session] = threadRuns(f, thread); + assert.ok(session); + assert.ok(!session.checkouts?.some(checkout => checkout.project === "books")); +}); diff --git a/packages/harness/src/bundle.mjs b/packages/harness/src/bundle.mjs index 2fb792d5d..ab0c72fa7 100644 --- a/packages/harness/src/bundle.mjs +++ b/packages/harness/src/bundle.mjs @@ -292,6 +292,7 @@ function prompt({ contract, resolved, session, tools, untyped, builtins, persona // resolved resolveInstance(...) output // contract the role contract's text // arbiter true when the business names this instance as an arbiter +// library true when the trusted business configuration has a library project // tracker true when the broker has task verbs for the business // credentials the broker's credentialStatus(...) for the instance (metadata only) // session { run, launchedBy, workspace, toolSocket, turnMarker, checkouts } @@ -309,11 +310,11 @@ function prompt({ contract, resolved, session, tools, untyped, builtins, persona // (row 213); an instance with persona.sha256 and no personaDir refuses // Returns { manifest, files, policy, tools, env } with absolute paths; env is the module's // adapter variables, which the host writes into session.json. -export function buildBundle({ dir, resolved, contract, arbiter = false, tracker = false, credentials = [], session, harness, route = "primary", routeReason = null, harnessVersion = null, models = null, land = false, skillsDir = null, personaDir = null }) { +export function buildBundle({ dir, resolved, contract, arbiter = false, tracker = false, library = false, credentials = [], session, harness, route = "primary", routeReason = null, harnessVersion = null, models = null, land = false, skillsDir = null, personaDir = null }) { if (route !== "primary" && route !== "fallback") throw new BundleError(`route ${JSON.stringify(route)} isn't primary or fallback`); const model = sessionModel(resolved, models, route); const web = webPlan(resolved, model.harness, harness.builtinWeb()); - const tools = typedTools({ resolved, arbiter, tracker, projects: (session.checkouts ?? []).map((c) => c.project), webTyped: !web.builtin, land }); + const tools = typedTools({ resolved, arbiter, tracker, library, projects: (session.checkouts ?? []).map((c) => c.project), webTyped: !web.builtin, land }); const untyped = untypedActions(resolved, tools); const persona = readPersona(resolved, personaDir); mkdirSync(dir, { recursive: true, mode: 0o700 }); diff --git a/packages/harness/src/tools.mjs b/packages/harness/src/tools.mjs index 1c038a366..8bee29e1d 100644 --- a/packages/harness/src/tools.mjs +++ b/packages/harness/src/tools.mjs @@ -78,9 +78,10 @@ const TASK_TOOLS = { const MESSAGE_CLASSES = ["REQUEST", "ASSIGNMENT", "REVIEW-REQUEST", "REVIEW-RESULT", "RESULT", "INFO", "DECISION", "REACTION", "BLOCK"]; -// typedTools({ resolved, arbiter, tracker, projects, land }) -> [{ name, verb, action, description, parameters }] +// typedTools({ resolved, arbiter, tracker, projects, land, library }) -> [{ name, verb, action, description, parameters }] // resolved resolveInstance(...) output for the session's instance // arbiter true when the business names this instance as an arbiter +// library true when the trusted business configuration has a library project // tracker true when the broker has task verbs for the business // projects the project ids the session has a clone of // land true when the role and thread hold a merge decision a landing could use (landHeld) @@ -88,7 +89,7 @@ const MESSAGE_CLASSES = ["REQUEST", "ASSIGNMENT", "REVIEW-REQUEST", "REVIEW-RESU // `verb` is the broker verb; the `launch` tool has verb "launch" and goes // to the host instead, and "working.push", "working.fetch" and // "working.land" go to the host's git socket. -export function typedTools({ resolved, arbiter = false, tracker = false, projects = [], webTyped = true, land = false }) { +export function typedTools({ resolved, arbiter = false, tracker = false, projects = [], webTyped = true, land = false, library = false }) { const held = new Set([...resolved.limits.authority.withinRole, ...resolved.limits.authority.crossRole]); const tools = []; const add = (name, verb, action, description, parameters) => tools.push({ name, verb, action, description, parameters }); @@ -222,6 +223,32 @@ export function typedTools({ resolved, arbiter = false, tracker = false, project url: str("The https url to fetch", { maxLength: 2048 }), }, ["url"])); } + // Row224: no library clone or token reaches a session. The broker chooses the served + // commit and requires a universe on every call; cursors page without cutting replies. + if (library) { + const universe = str("Universe ID in the configured library", { minLength: 1 }); + const project = str("Library project ID; omit only when the business has one library"); + const scope = str("canon by default; playground is opt-in", { enum: ["canon", "playground"] }); + const book = str("Book folder from library_universe"); + const offset = { type: "integer", minimum: 0, description: "next from the previous page" }; + const line = { type: "integer", minimum: 1, description: "One-based line number" }; + const base = { universe, project }; + add("library_find", "library.find", null, "Find names in one universe, at the served commit. At most 50 results; continue with next as offset.", obj({ + ...base, query: str("Case-insensitive name or filename substring; empty matches all"), + type: str("Mapped type", { enum: ["character", "place", "research", "tracker", "summary"] }), + book, scope, offset, + }, ["universe", "query"])); + add("library_get", "library.get", null, "Read a mapped note and visible links in one universe at the served commit. next pages lines; links_next pages links. An explicit own-universe playground path is readable.", obj({ + ...base, path: str("Mapped note path relative to the library"), from: line, links_from: offset, + }, ["universe", "path"])); + add("library_search", "library.search", null, "Search text in one universe at the served commit. At most 50 hits; use next as offset. Playground is opt-in.", obj({ + ...base, text: str("Nonempty case-insensitive substring", { minLength: 1 }), book, scope, offset, + }, ["universe", "text"])); + add("library_manuscript", "library.manuscript", null, "Read one chapter of a book at the served commit, at most 400 lines. Omit chapter for the first; continue with next or next_chapter.", obj({ + ...base, book, scope, chapter: str("Chapter filename without extension"), from: line, to: line, + }, ["universe", "book"])); + add("library_universe", "library.universe", null, "Read a universe's mapping at the served commit: books, types, playground prefixes, and hybrid sources/constraints/notes or draft metadata.", obj(base, ["universe"])); + } add("read_spec", "spec.read", null, "Read a project's committed north-star spec and its file history. commit is an optional full reachable commit ID. The file commit, rather than HEAD, is what a decision cites. confirmed says the file commit was authored by the business's human.", obj({ project: str("Project ID from the business file"), commit: str("Optional full historical commit ID", { pattern: "^[0-9a-f]{40}$" }), @@ -277,6 +304,11 @@ export function typedTools({ resolved, arbiter = false, tracker = false, project if (held.has("task.create")) add("list_requirements", "requirements", null, "List the approved PRD's requirement ids, each with a one-line title. task_create takes only these. When none fits a request, raise a prd.approve decision instead of creating the task.", obj({})); } + // mos-1308 / M3: no generic spec, evidence, tracker, decision or write tools. The + // C2-08 web grant still supplies broker web reads to companion and research only. + if (CREATIVE_DEFINITIONS.includes(resolved.definition)) + return tools.filter(tool => tool.name === "send_message" || tool.name.startsWith("library_") || + tool.name === "web_search" || tool.name === "web_fetch"); return tools; } diff --git a/packages/harness/tests/library-tools.test.mjs b/packages/harness/tests/library-tools.test.mjs new file mode 100644 index 000000000..771b53956 --- /dev/null +++ b/packages/harness/tests/library-tools.test.mjs @@ -0,0 +1,69 @@ +// Row224 M3: exact creative tool sets preserve the existing C2-08 web grants. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { typedTools } from '../src/tools.mjs'; +import { buildBundle } from '../src/bundle.mjs'; +import { HARNESS, REPO_SKILLS, resolvedFor, scratch } from './helpers.mjs'; + +const LOOKUPS = ['library_find', 'library_get', 'library_search', 'library_manuscript', 'library_universe']; +const CREATIVE = ['companion', 'research', 'ideas', 'proofreader', 'fact-checker', 'continuity']; +const role = (t, definition, harness = 'pi') => resolvedFor(t, 'bean', d => { + d.roles.bean = { definition, vars: { harness, model: 'claude-sonnet-5-5', thinking: 'high' } }; +}); +const expected = definition => ['send_message', ...(['companion', 'research'].includes(definition) + ? ['web_search', 'web_fetch'] : []), ...LOOKUPS]; + +test('M3: per-role snapshots contain six names, or eight with the C2-08 web grant', t => { + for (const definition of CREATIVE) { + const tools = typedTools({ resolved: role(t, definition), library: true, tracker: true, arbiter: true }); + assert.deepEqual(tools.map(t => t.name), expected(definition), definition); + assert.ok(tools.every(t => t.parameters.additionalProperties === false)); + } +}); + +test('every noncreative role gets lookups when a library is configured, not otherwise', t => { + for (const instance of ['coder', 'reviewer', 'pm', 'chief']) { + const resolved = resolvedFor(t, instance); + const ordinary = typedTools({ resolved }).map(t => t.name); + assert.ok(!ordinary.some(name => name.startsWith('library_'))); + const tools = typedTools({ resolved, library: true }); + assert.deepEqual(tools.filter(t => t.name.startsWith('library_')).map(t => t.name), LOOKUPS); + assert.deepEqual(tools.filter(t => !t.name.startsWith('library_')).map(t => t.name), ordinary); + } +}); + +test('lookup schemas require a universe even for get, include paging, and cannot select repo or commit', t => { + const tools = typedTools({ resolved: role(t, 'ideas'), library: true }); + const required = [ ['universe', 'query'], ['universe', 'path'], ['universe', 'text'], ['universe', 'book'], ['universe'] ]; + for (const [i, tool] of tools.filter(t => t.name.startsWith('library_')).entries()) { + assert.deepEqual(tool.parameters.required, required[i]); + assert.equal(tool.verb, LOOKUPS[i].replace('_', '.')); + assert.equal(tool.action, null); + assert.equal(tool.parameters.additionalProperties, false); + for (const key of ['repo', 'commit', 'business', 'accept']) assert.equal(Object.hasOwn(tool.parameters.properties, key), false); + } + const byName = Object.fromEntries(tools.map(t => [t.name, t])); + assert.deepEqual(byName.library_find.parameters.properties.scope.enum, ['canon', 'playground']); + assert.equal(byName.library_find.parameters.properties.offset.minimum, 0); + assert.equal(byName.library_get.parameters.properties.from.minimum, 1); + assert.equal(byName.library_get.parameters.properties.links_from.minimum, 0); + assert.equal(byName.library_manuscript.parameters.properties.chapter.type, 'string'); +}); + +test('bundle flag reaches both harness tool manifests and creative prompt without generic tools', t => { + for (const definition of CREATIVE) for (const harness of ['pi', 'claude-code']) { + const dir = scratch(t), workspace = join(dir, 'ws'); mkdirSync(workspace); + const b = buildBundle({ dir: join(dir, 'bundle'), resolved: role(t, definition, harness), library: true, + contract: '# Creative\n', tracker: true, arbiter: true, skillsDir: REPO_SKILLS, + session: { run: 'r224', launchedBy: 'chief', workspace, toolSocket: join(dir, 't.sock'), turnMarker: join(dir, 'turn.done') }, + harness: HARNESS[harness] }); + assert.deepEqual(b.tools.map(t => t.name), expected(definition), `${definition} ${harness}`); + const serialized = JSON.parse(readFileSync(b.files.tools, 'utf8')); + assert.deepEqual(serialized.map(t => t.name), expected(definition)); + const prompt = readFileSync(b.files.prompt, 'utf8'); + assert.match(prompt, /library_find/); + assert.doesNotMatch(prompt, /Bus tools: [^\n]*(read_spec|list_agents|inbox|trail|write_record|raise_decision|list_tasks)/); + } +}); From 23816cf285c952ddc47393130f8396a1993d7d99 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 22 Oct 2026 11:50:19 -0500 Subject: [PATCH 3/3] row224: pace failed inline sync attempts (unrun) --- agents/della/work/queue-224/BUILD.md | 33 +++++++++++++++++-- packages/bus/src/runtime.mjs | 6 +++- packages/bus/tests/library-lookups.test.mjs | 15 +++++++-- packages/harness/tests/library-tools.test.mjs | 2 ++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/agents/della/work/queue-224/BUILD.md b/agents/della/work/queue-224/BUILD.md index 8c7faf6f3..4e3368350 100644 --- a/agents/della/work/queue-224/BUILD.md +++ b/agents/della/work/queue-224/BUILD.md @@ -43,7 +43,7 @@ explicitly. After 253 and 216 land, rebase under a new branch name. ## Verification ledger -No test or syntax-check run yet. Acceptance requires full bus/harness suites +No test or syntax-check run yet. Acceptance requires full bus/cli/harness suites and every scripts/test-*.sh on Sage's gate; focused files and affected CLI/ seat import-closure tests precede that only in an authorized CLEAR. @@ -70,7 +70,7 @@ Library sources remain in the existing whole-directory release copy. Set TMPDIR=/mnt/storage/scratch/tmp. Focused new bus/harness files first, then existing library.test.mjs, library-sync.test.mjs, project-runtime.test.mjs, harness creative/tools/spec-tools/bundle files and affected CLI launch tests. -Follow with the required whole bus/harness suites and Sage's full shell gate. +Follow with the required whole bus/cli/harness suites and Sage's full shell gate. No timing claim yet: use the repository/Sage per-file limits and record every command, outcome, elapsed time and timeout before acceptance. No focused skip or pattern run substitutes for whole-file gate evidence. @@ -139,3 +139,32 @@ For `docs/SESSIONS.md`: Local-only draft pins 19c0a7adf/7cf39c390 are superseded by the log-free candidate. Saved as local branch della/row224-draft-with-logs before rebuilding the unpublished candidate commit; no remote history changed. + +## Dewey dewey-0852 and Sage sage-2819 — focused corrections + +B1: inline freshness uses max(lastFetch, lastError.at), not lastFetch alone. +An error within 60 seconds counts as a recent attempt, so later lookups serve +the old served pin without another fetch. Keep inline sync: the brief requires +it and businesses without timed libraries have no sync timer. The extended +failed-fetch test makes reads at +1, +59,999 and +60,000 ms (one attempt total), +then +60,001 ms (second attempt). All reads must return the old commit. + +Required mutant M-B1: remove lastError.at from recentAttempt. The repeated +reads must fail the attempted-count assertion. This mutant has NOT been run. + +Residual (Sage will defer): the first inline fetch in each outage window can +still exceed the tool timeout (60 seconds, or the runner broker client’s +30 seconds). Bounding that fetch is out of scope for 224. Subsequent reads +are paced by persisted lastError.at. No claim of first-read outage latency. + +N1: whole bus, cli and harness suites are required, plus the shell gate. +If the new thread-launch integration test skips due to sandboxProblem(), +name the skip; a skip is not launcher integration proof. +N2: cli/tests/thread-launch.test.mjs now depends on library/tests/helpers.mjs +mappingEntry() and library/tests/fixtures/mapping.json. Changes to that fixture +must rerun the CLI launch file. +N3: harness/tests/library-tools.test.mjs compares the type enum to mapping.mjs +TYPES, so adding a mapped type cannot silently leave the typed tool behind. + +These edits are local code/docs only. No Node, RED/GREEN run, mutant, claim, +HTTPS Git or push. New local pin goes to Dewey/Mos/Sage for focused review. diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index a16ab2ac6..859d0a630 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -95,7 +95,11 @@ export function createLibraryLookup({ dataRoot, sync, now = () => Date.now() }) const source = { ...context, dataRoot }; let state = await libraryModule.libraryStatus(source); authorize(); - const age = state.lastFetch === null ? Infinity : now() - Date.parse(state.lastFetch); + // sage-2819 / dewey-0852: failed attempts also pace inline retries. Otherwise an + // outage makes every lookup refetch while lastFetch remains old or null. + const recentAttempt = Math.max(state.lastFetch === null ? -Infinity : Date.parse(state.lastFetch), + state.lastError === null ? -Infinity : Date.parse(state.lastError.at)); + const age = now() - recentAttempt; if (age > libraryModule.SYNC_INTERVAL_MS || age < 0) { try { await sync(source); } catch (error) { diff --git a/packages/bus/tests/library-lookups.test.mjs b/packages/bus/tests/library-lookups.test.mjs index d01d0e9ef..a612af4a7 100644 --- a/packages/bus/tests/library-lookups.test.mjs +++ b/packages/bus/tests/library-lookups.test.mjs @@ -159,16 +159,25 @@ test('freshness boundary, unknown last fetch and failed fetch still use immutabl f.setState({ lastFetch: null }); await f.call('library.universe', { universe: 'harbor' }); assert.equal(f.fetches(), 2); - let attempted = 0; - const failing = createLibraryLookup({ dataRoot: f.dataRoot, now: () => NOW, sync: async () => { + let attempted = 0, clock = NOW; + const failing = createLibraryLookup({ dataRoot: f.dataRoot, now: () => clock, sync: async () => { attempted++; - f.setState({ lastError: { code: 'library-fetch-failed', at: new Date(NOW).toISOString() } }); + f.setState({ lastError: { code: 'library-fetch-failed', at: new Date(clock).toISOString() } }); return { code: 'library-fetch-failed' }; } }); f.setState({ lastFetch: null }); const result = await failing(f.context, { method: 'get', args: GET, authorize() {} }); assert.equal(attempted, 1); assert.equal(result.commit, f.commit); + for (const elapsed of [1, 59_999, 60_000]) { + clock = NOW + elapsed; + const retry = await failing(f.context, { method: 'get', args: GET, authorize() {} }); + assert.equal(retry.commit, f.commit, 'serve the old pin during the failure cooldown'); + assert.equal(attempted, 1, 'recent lastError prevents another sync even though lastFetch is null'); + } + clock = NOW + 60_001; + assert.equal((await failing(f.context, { method: 'get', args: GET, authorize() {} })).commit, f.commit); + assert.equal(attempted, 2, 'retry only after the failed-attempt interval expires'); }); test('an unreadable sync state fails closed; a busy sync keeps serving the existing pin', async t => { diff --git a/packages/harness/tests/library-tools.test.mjs b/packages/harness/tests/library-tools.test.mjs index 771b53956..9208a208f 100644 --- a/packages/harness/tests/library-tools.test.mjs +++ b/packages/harness/tests/library-tools.test.mjs @@ -5,6 +5,7 @@ import { mkdirSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { typedTools } from '../src/tools.mjs'; import { buildBundle } from '../src/bundle.mjs'; +import { TYPES } from '../../library/src/mapping.mjs'; import { HARNESS, REPO_SKILLS, resolvedFor, scratch } from './helpers.mjs'; const LOOKUPS = ['library_find', 'library_get', 'library_search', 'library_manuscript', 'library_universe']; @@ -46,6 +47,7 @@ test('lookup schemas require a universe even for get, include paging, and cannot } const byName = Object.fromEntries(tools.map(t => [t.name, t])); assert.deepEqual(byName.library_find.parameters.properties.scope.enum, ['canon', 'playground']); + assert.deepEqual(byName.library_find.parameters.properties.type.enum, [...TYPES]); assert.equal(byName.library_find.parameters.properties.offset.minimum, 0); assert.equal(byName.library_get.parameters.properties.from.minimum, 1); assert.equal(byName.library_get.parameters.properties.links_from.minimum, 0);