From d3c7bbf839d9c3bf50f154b18e7e4472fbd005ed Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Tue, 20 Oct 2026 16:29:31 -0500 Subject: [PATCH] gosalyn: C2-01 packages/library, mapping, scope, commit reader with R1-R7, link resolver, renderItems, library survey (row 212, #416) --- packages/cli/tests/stack-copy.test.mjs | 2 +- packages/library/README.md | 128 +++++++++++ packages/library/package.json | 11 + packages/library/src/index.mjs | 8 + packages/library/src/links.mjs | 79 +++++++ packages/library/src/mapping.mjs | 132 +++++++++++ packages/library/src/problems.mjs | 27 +++ packages/library/src/reader.mjs | 219 +++++++++++++++++++ packages/library/src/render.mjs | 59 +++++ packages/library/src/scope.mjs | 21 ++ packages/library/src/survey.mjs | 126 +++++++++++ packages/library/tests/fixtures/mapping.json | 12 + packages/library/tests/helpers.mjs | 54 +++++ packages/library/tests/links.test.mjs | 50 +++++ packages/library/tests/mapping.test.mjs | 82 +++++++ packages/library/tests/reader.test.mjs | 151 +++++++++++++ packages/library/tests/render.test.mjs | 80 +++++++ packages/library/tests/survey.test.mjs | 82 +++++++ scripts/mosaic | 6 + scripts/stack-copy.sh | 2 +- 20 files changed, 1329 insertions(+), 2 deletions(-) create mode 100644 packages/library/README.md create mode 100644 packages/library/package.json create mode 100644 packages/library/src/index.mjs create mode 100644 packages/library/src/links.mjs create mode 100644 packages/library/src/mapping.mjs create mode 100644 packages/library/src/problems.mjs create mode 100644 packages/library/src/reader.mjs create mode 100644 packages/library/src/render.mjs create mode 100644 packages/library/src/scope.mjs create mode 100644 packages/library/src/survey.mjs create mode 100644 packages/library/tests/fixtures/mapping.json create mode 100644 packages/library/tests/helpers.mjs create mode 100644 packages/library/tests/links.test.mjs create mode 100644 packages/library/tests/mapping.test.mjs create mode 100644 packages/library/tests/reader.test.mjs create mode 100644 packages/library/tests/render.test.mjs create mode 100644 packages/library/tests/survey.test.mjs diff --git a/packages/cli/tests/stack-copy.test.mjs b/packages/cli/tests/stack-copy.test.mjs index 86e65e588..02078a767 100644 --- a/packages/cli/tests/stack-copy.test.mjs +++ b/packages/cli/tests/stack-copy.test.mjs @@ -35,7 +35,7 @@ export function setup(t, realPin=false){ test('shared copy: npm ci, pushed reachability, readonly/existing copy, webui delegation',t=>{ const s=setup(t),r=s.run('stack-copy.sh',['copy',s.id,'--copies',s.copies]);assert.equal(r.status,0,r.stderr); const copy=join(s.copies,s.id);assert.equal(statSync(copy).mode&0o777,0o555); - for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs'])assert.ok(existsSync(join(copy,f)),f); + for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/library/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs'])assert.ok(existsSync(join(copy,f)),f); const npm=join(s.bin,'npm');writeFileSync(npm,'#!/bin/sh\nexit 99\n');chmodSync(npm,0o755); const existing=s.run('webui-service.sh',['copy',s.id,'--copies',s.copies]);assert.equal(existing.status,0);assert.equal(existing.stdout,`exists: ${copy}\n`);unlinkSync(npm); git(s.repo,'update-ref','refs/remotes/origin/refactor',git(s.repo,'hash-object','-w','package.json')); diff --git a/packages/library/README.md b/packages/library/README.md new file mode 100644 index 000000000..200011cfe --- /dev/null +++ b/packages/library/README.md @@ -0,0 +1,128 @@ +# packages/library + +Reads a writing repo by a mapping of folders to books, types, canon and playground. This is Case 2 +row C2-01 (`docs/plans/2026-10-20_case2-rows.md`), built from design sections 1.1 to 1.3 and 7.4 in +`agents/gosalyn/work/case2/DESIGN.md` on `gosalyn/case2-design`. +It is pure code with no state. Business file loading (C2-04), sync (C2-03a), the index (C2-05a) and the +guard (C2-06) build on it. + +## Mapping + +`checkMapping(entry, where, { fail })` checks a `kind: "library"` project entry and returns a +deep-frozen copy. `where` names the entry in messages. `fail(code, message)` defaults to throwing a +`LibraryError`. C2-04 passes its own so a business file refusal carries the file name. + +```json +{ + "kind": "library", + "remote": "https://git.example.invalid/her/books.git", + "branch": "main", + "vaultPrefix": "books", + "types": { "Characters": "character", "Places": "place", "Research": "research" }, + "manuscript": ["Chapters"], + "universes": { + "harbor": { "name": "Harbor", "books": ["Book One"], "playground": ["Book One/Playground/"] } + } +} +``` + +- `remote` is https with no credentials, query or fragment. `branch` is a plain ref name. +- `vaultPrefix` is optional, one folder name. +- `types` maps a folder name to `character`, `place`, `research`, `tracker` or `summary`. + `manuscript` lists folder names and shares none with `types`. +- A universe has `name`, `books` and `playground`. A hybrid adds `kind: "hybrid"`, two or more + non-hybrid `sources`, `constraints` (each one `{ "exclude": ... }` or + `{ "pin": { "entity", "field", "value" } }`) and `notes`. + +Refusals: + +| Code | When | +|---|---| +| `library-playground-outside-book` | a playground prefix is not strictly inside a book of its own universe | +| `library-mapping-invalid` | any other shape fault: unknown key, bad remote or branch, a book folder in two universes, a book folder that holds another, a playground prefix that holds another, an unknown hybrid source | + +## Scope + +`classify(mapping, path)` decides scope from the path alone. Only paths ending in lowercase `.md` are +mapped. A path under a playground prefix is `playground`. Any other path under a book folder is +`canon`. Everything else is `unmapped`. The type is the first folder after the book, or after the +playground prefix: a `types` name, `manuscript`, or `null`. + +## Reader + +`readLibrary({ repo, commit, mapping })` reads one commit, never a working tree. `repo` may be a +clone or a bare mirror. `commit` must be a full 40-hex id that names a commit, else `library-commit`. +Git runs through `git()` and `gitBytes()` from `packages/project/src/git.mjs`, so the same safe config, +environment and ceiling directory apply. A failed git call refuses `library-git`. + +It returns `{ commit, files, problems }`, deep-frozen and sorted. Each file has its path, scope, +universe, book, type, numbered lines, `bodyLine`, parsed `frontmatter` or `null`, `frontmatterRaw` +and `links`. Front matter `appears_in` and `relationships` wikilinks count as links. + +Problems carry `{ code, path, line, message }`: + +| Rule | Meaning | File indexed | +|---|---|---| +| R1 | not a regular `100644` blob of at most 1 MiB, or not UTF-8, or holds a NUL | no | +| R2 | path is unmapped | no | +| R3 | front matter not fully parsed, `line` is where parsing stopped | yes, front matter kept raw | +| R4 | link resolves to no file or to more than one | yes, link dropped | +| R5 | canon file links into the playground | yes | +| R6 | canon file links into another universe, a hybrid's sources excepted | yes | +| R7 | two paths differ only in case | no, neither | + +The front matter parser takes scalars, quoted strings, flow lists, block lists and wikilinks. Nested +maps, anchors, block scalars and duplicate keys stop it with R3. + +## Links + +`resolver(paths, { vaultPrefix })` tries four steps in order, each with the target as written and +then with `.md` added: + +1. `root`: the path from the repo root. +2. `vault-prefix`: the same path with `vaultPrefix/` stripped. +3. `relative`: relative to the linking file, never above the repo root. +4. `shortest`: a target with no `/` that matches the base name of exactly one indexed file. + +It resolves only against indexed files. A link to a `.canvas` file or an attachment is R4. A link into +another repo resolves to nothing. `extractLinks` skips fenced and inline code. + +## renderItems + +`renderItems(body, { scopeOf, changed })` turns a reply body into the plain text both channels show. +It is pure. Item kinds are `fact`, `finding`, `correction` (with `from` and `to`), `question` and +`idea`. Each cite is a path with commit, line range and quote, or a url with a quote. + +``` +1. Correction: Chapter 1 says grey. + From: "green eyes" + To: "grey eyes" + [canon] Book One/Characters/Mara Vell.md:9 @bbbbbbb "green eyes" (changed since) +``` + +A body of `{ "items": [], "notice": ... }` gives `No cited answer`, `No answer` or `The turn failed`. +Any other shape gives the fixed line `This reply could not be shown.` and none of the body's text. + +## Survey + +```sh +mosaic library survey [--mapping ] +``` + +It reads the repo's HEAD commit and prints folder names, file counts by extension, front matter key +sets and link-form counts. It never prints file text, front matter values, file names or link targets. +A key that is not a short identifier prints as `(other key)`, since it could be her words. With +`--mapping` it adds scope counts, resolver step counts and problem counts by rule. + +Exit codes follow `packages/project`: 0 done, 2 the repo can't be read (`library-git`, +`library-commit`), 3 a bad mapping, 4 usage. `scripts/mosaic` sends `library survey` here. Other +library verbs will be bus verbs. + +## Tests + +```sh +node --test packages/library/tests/ +``` + +Fixtures are made-up repos built per test in a temp dir. A marker string stands in for her text, and +the survey tests check that it never appears in output. diff --git a/packages/library/package.json b/packages/library/package.json new file mode 100644 index 000000000..8d1b70675 --- /dev/null +++ b/packages/library/package.json @@ -0,0 +1,11 @@ +{ + "name": "@mosaic/library", + "version": "0.1.0", + "private": true, + "type": "module", + "license": "UNLICENSED", + "exports": "./src/index.mjs", + "scripts": { + "test": "node --test tests/*.test.mjs" + } +} diff --git a/packages/library/src/index.mjs b/packages/library/src/index.mjs new file mode 100644 index 000000000..a76a34554 --- /dev/null +++ b/packages/library/src/index.mjs @@ -0,0 +1,8 @@ +// packages/library: her writing repo read by a mapping (Case 2, design 1 and 7.4). See README.md. +export { LibraryError, RULES, problem, sortProblems } from './problems.mjs'; +export { checkMapping, TYPES } from './mapping.mjs'; +export { classify } from './scope.mjs'; +export { extractLinks, parseWikilink, resolver, formOf } from './links.mjs'; +export { readLibrary, listTree, readTexts, parseFile, parseFrontMatter, headCommit, MAX_BLOB, LINK_FIELDS } from './reader.mjs'; +export { renderItems, KINDS, NOTICES, UNREADABLE } from './render.mjs'; +export { survey, formatSurvey, surveyMain } from './survey.mjs'; diff --git a/packages/library/src/links.mjs b/packages/library/src/links.mjs new file mode 100644 index 000000000..4d8d69b3a --- /dev/null +++ b/packages/library/src/links.mjs @@ -0,0 +1,79 @@ +// Wikilinks and the link resolver of design 1.3. Links are [[target]], [[target|label]] and +// [[target#heading]], with ![[...]] embeds read the same way. Links inside fenced code blocks and +// inline code spans are not links, as in Obsidian. +import { posix } from 'node:path'; + +const WIKILINK = /(!?)\[\[([^\[\]\n]+?)\]\]/g; +const MARKDOWN_LINK = /(? link.markdown ? 'markdown' : link.embed ? 'embed' : !link.file ? 'self' + : link.label !== null ? 'label' : link.heading !== null ? 'heading' : 'plain'; + +// Links in body lines. `lines` is [[number, text], ...]. Markdown links are returned only with +// `markdown: true`, for the survey's counts. The resolver follows wikilinks only. +export function extractLinks(lines, { markdown = false } = {}) { + const found = []; + let fence = null; + for (const [line, text] of lines) { + const open = FENCE.exec(text); + if (fence) { if (open && open[1][0] === fence[0] && open[1].length >= fence.length) fence = null; continue; } + if (open) { fence = open[1]; continue; } + const plain = text.replace(/(`+)[^`]*?\1/g, match => ' '.repeat(match.length)); + for (const m of plain.matchAll(WIKILINK)) found.push({ ...parts(m[2], m[1] === '!'), line }); + if (markdown) { + for (const m of plain.matchAll(MARKDOWN_LINK)) { + if (!/^[a-z][a-z0-9+.-]*:/i.test(m[1]) && !m[1].startsWith('#')) + found.push({ file: m[1], heading: null, label: null, embed: false, markdown: true, line }); + } + } + } + return found; +} + +// A resolver over the indexed paths. resolve(from, file) tries, in order: a path from the repo +// root, the same path with `vaultPrefix/` stripped, a path relative to the linking file, and +// Obsidian's shortest-path form, a base name that matches exactly one indexed file. Each step tries +// the target as written and then with `.md` added. A target in another of her repos is not in the +// index, so it resolves to nothing. +export function resolver(paths, { vaultPrefix } = {}) { + const known = new Set(paths), byBase = new Map(); + for (const path of known) { + const base = posix.basename(path); + for (const key of new Set([base, base.replace(/\.md$/, '')])) byBase.set(key, [...(byBase.get(key) ?? []), path]); + } + const hit = candidate => [candidate, `${candidate}.md`].find(c => known.has(c)); + return function resolve(from, file) { + if (typeof file !== 'string' || !file) return { path: null, step: 'missing' }; + let path = hit(file); + if (path) return { path, step: 'root' }; + if (vaultPrefix && file.startsWith(`${vaultPrefix}/`) && (path = hit(file.slice(vaultPrefix.length + 1)))) + return { path, step: 'vault-prefix' }; + const relative = posix.normalize(posix.join(posix.dirname(from), file)); + if (relative !== '..' && !relative.startsWith('../') && !relative.startsWith('/') && (path = hit(relative))) + return { path, step: 'relative' }; + if (!file.includes('/')) { + const matches = byBase.get(file) ?? []; + if (matches.length === 1) return { path: matches[0], step: 'shortest' }; + if (matches.length > 1) return { path: null, step: 'ambiguous', count: matches.length }; + } + return { path: null, step: 'missing' }; + }; +} diff --git a/packages/library/src/mapping.mjs b/packages/library/src/mapping.mjs new file mode 100644 index 000000000..914a32196 --- /dev/null +++ b/packages/library/src/mapping.mjs @@ -0,0 +1,132 @@ +// The library project entry of a business file (design 1.1): her folders mapped onto universes, +// books, type folders, manuscript folders and playground prefixes. A pure validator. Business file +// loading wires it in C2-04, which passes its own `fail` so the refusal reads as a business file +// problem. Agents can't write this entry (M9, C2-04). +import { raise } from './problems.mjs'; + +export const TYPES = Object.freeze(['character', 'place', 'research', 'tracker', 'summary']); +const ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +const BRANCH = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/; +const CONSTRAINT_KEYS = ['exclude', 'pin']; + +// A folder segment from her vault: spaces and capitals are fine, separators, control characters +// and dot names are not. +function segment(value) { + return typeof value === 'string' && value.length > 0 && value.length <= 255 && value !== '.' && value !== '..' + && !/[\/\\\u0000-\u001f\u007f]/.test(value) && value.trim() === value; +} +function folder(value) { + return typeof value === 'string' && value.length <= 1024 && value.split('/').every(segment); +} +const holds = (outer, inner) => inner.startsWith(outer.endsWith('/') ? outer : `${outer}/`); + +export function checkMapping(entry, where = 'library', { fail = raise } = {}) { + const bad = message => fail('library-mapping-invalid', `${where} ${message}`); + const object = (value, name) => { + if (value === null || typeof value !== 'object' || Array.isArray(value)) bad(`${name} must be an object`); + }; + const only = (value, keys, name) => { + for (const key of Object.keys(value)) if (!keys.includes(key)) bad(`${name} has unknown key ${JSON.stringify(key)}`); + }; + const list = (value, name, check, what) => { + if (!Array.isArray(value)) bad(`${name} must be a list`); + for (const item of value) if (!check(item)) bad(`${name} holds ${JSON.stringify(item)}, which is not ${what}`); + if (new Set(value).size !== value.length) bad(`${name} names an entry twice`); + return [...value]; + }; + + object(entry, ''); + only(entry, ['kind', 'remote', 'branch', 'vaultPrefix', 'universes', 'types', 'manuscript'], 'entry'); + if (entry.kind !== 'library') bad('kind must be "library"'); + let url; + try { url = new URL(entry.remote); } catch { bad('remote must be an https:// URL'); } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash || entry.remote.includes('\0')) + bad('remote must be an https:// URL with no credentials, query or fragment'); + if (typeof entry.branch !== 'string' || !BRANCH.test(entry.branch) || entry.branch.includes('..') + || entry.branch.includes('//') || entry.branch.endsWith('/') || entry.branch.endsWith('.lock')) + bad('branch must be a plain branch name'); + if (Object.hasOwn(entry, 'vaultPrefix') && !segment(entry.vaultPrefix)) bad('vaultPrefix must be one folder name'); + + object(entry.types, 'types'); + for (const [name, type] of Object.entries(entry.types)) { + if (!segment(name)) bad(`types has ${JSON.stringify(name)}, which is not one folder name`); + if (!TYPES.includes(type)) bad(`types.${name} must be one of ${TYPES.join(', ')}`); + } + const manuscript = list(entry.manuscript, 'manuscript', segment, 'one folder name'); + for (const name of manuscript) if (Object.hasOwn(entry.types, name)) bad(`manuscript folder ${JSON.stringify(name)} is also a type folder`); + + object(entry.universes, 'universes'); + if (!Object.keys(entry.universes).length) bad('universes must name at least one universe'); + const universes = {}, bookOwner = new Map(), prefixes = []; + for (const [id, universe] of Object.entries(entry.universes)) { + const name = `universes.${id}`; + if (!ID.test(id)) bad(`universe id ${JSON.stringify(id)} is not a plain id`); + object(universe, name); + const hybrid = universe.kind === 'hybrid'; + only(universe, hybrid ? ['kind', 'name', 'sources', 'books', 'playground', 'constraints', 'notes'] : ['name', 'books', 'playground'], name); + if (typeof universe.name !== 'string' || !universe.name.trim() || universe.name.length > 128 || /[\u0000-\u001f]/.test(universe.name)) + bad(`${name}.name must be a short display name`); + const books = list(universe.books, `${name}.books`, folder, 'a folder path'); + if (!hybrid && !books.length) bad(`${name}.books must name at least one book folder`); + for (const book of books) { + if (bookOwner.has(book)) bad(`book folder ${JSON.stringify(book)} is in two universes`); + bookOwner.set(book, id); + } + const playground = list(universe.playground ?? [], `${name}.playground`, + value => typeof value === 'string' && value.endsWith('/') && folder(value.slice(0, -1)), 'a folder path ending in /'); + for (const prefix of playground) { + // Every playground file must have a universe and a book (1.1, B1). + if (!books.some(book => holds(book, prefix) && prefix !== `${book}/`)) + fail('library-playground-outside-book', `${where} ${name}.playground ${JSON.stringify(prefix)} is not inside one of its books`); + prefixes.push(prefix); + } + const out = { name: universe.name, books, playground }; + if (hybrid) { + out.kind = 'hybrid'; + out.sources = list(universe.sources, `${name}.sources`, value => typeof value === 'string' && ID.test(value), 'a universe id'); + if (out.sources.length < 2) bad(`${name}.sources must name two or more universes`); + if (!Array.isArray(universe.constraints ?? [])) bad(`${name}.constraints must be a list`); + out.constraints = (universe.constraints ?? []).map((constraint, i) => { + const at = `${name}.constraints[${i}]`; + object(constraint, at); + only(constraint, CONSTRAINT_KEYS, at); + if (Object.keys(constraint).length !== 1) bad(`${at} must hold exactly one of ${CONSTRAINT_KEYS.join(', ')}`); + if (Object.hasOwn(constraint, 'exclude')) { + if (!folder(constraint.exclude)) bad(`${at}.exclude must be a source path`); + return { exclude: constraint.exclude }; + } + object(constraint.pin, `${at}.pin`); + only(constraint.pin, ['entity', 'field', 'value'], `${at}.pin`); + if (!folder(constraint.pin.entity) || typeof constraint.pin.field !== 'string' || !constraint.pin.field + || !['string', 'number', 'boolean'].includes(typeof constraint.pin.value)) + bad(`${at}.pin needs an entity path, a field name and a scalar value`); + return { pin: { ...constraint.pin } }; + }); + if (Object.hasOwn(universe, 'notes') && (typeof universe.notes !== 'string' || universe.notes.length > 4000)) + bad(`${name}.notes must be text of at most 4000 characters`); + if (Object.hasOwn(universe, 'notes')) out.notes = universe.notes; + } + universes[id] = out; + } + for (const [id, universe] of Object.entries(universes)) { + for (const source of universe.sources ?? []) { + if (!Object.hasOwn(universes, source) || universes[source].kind === 'hybrid') + bad(`universes.${id}.sources names ${JSON.stringify(source)}, which is not a non-hybrid universe`); + } + } + const books = [...bookOwner.keys()]; + for (const a of books) for (const b of books) if (a !== b && holds(a, b)) bad(`book folder ${JSON.stringify(a)} holds book folder ${JSON.stringify(b)}`); + for (const a of prefixes) for (const b of prefixes) if (a !== b && holds(a, b)) bad(`playground ${JSON.stringify(a)} holds playground ${JSON.stringify(b)}`); + if (new Set(prefixes).size !== prefixes.length) bad('a playground prefix is listed twice'); + + return deepFreeze({ + kind: 'library', remote: entry.remote, branch: entry.branch, + ...(Object.hasOwn(entry, 'vaultPrefix') ? { vaultPrefix: entry.vaultPrefix } : {}), + universes, types: { ...entry.types }, manuscript, + }); +} + +function deepFreeze(value) { + if (value && typeof value === 'object') { for (const item of Object.values(value)) deepFreeze(item); Object.freeze(value); } + return value; +} diff --git a/packages/library/src/problems.mjs b/packages/library/src/problems.mjs new file mode 100644 index 000000000..df69d0c3c --- /dev/null +++ b/packages/library/src/problems.mjs @@ -0,0 +1,27 @@ +// Errors and the R-rule problems of design 1.3. A problem never holds back a commit: it names a +// path and, where there is one, a line, and the index leaves the file or link out. +export class LibraryError extends Error { + constructor(code, message) { super(`${code}: ${message}`); this.code = code; } +} +export const raise = (code, message) => { throw new LibraryError(code, message); }; + +export const RULES = Object.freeze({ + R1: 'not a regular blob up to 1 MiB of UTF-8 text', + R2: 'path is unmapped', + R3: 'front matter not fully parsed', + R4: 'link resolves to no file or to more than one', + R5: 'canon file refers into the playground', + R6: 'canon file refers into another universe', + R7: 'paths differ only in case', +}); + +export function problem(code, path, line, message = RULES[code]) { + if (!Object.hasOwn(RULES, code)) raise('library-problem-code', `unknown problem code ${code}`); + return Object.freeze({ code, path, ...(Number.isSafeInteger(line) ? { line } : {}), message }); +} + +// Stable order for status output and tests: path, then line, then code. +export function sortProblems(list) { + return [...list].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 + : (a.line ?? 0) - (b.line ?? 0) || (a.code < b.code ? -1 : a.code > b.code ? 1 : 0)); +} diff --git a/packages/library/src/reader.mjs b/packages/library/src/reader.mjs new file mode 100644 index 000000000..752856ce0 --- /dev/null +++ b/packages/library/src/reader.mjs @@ -0,0 +1,219 @@ +// The commit reader of design 1.3. It reads a commit of her repo, never a working tree: it lists the +// tree with `git ls-tree -r -z` and reads blobs with `cat-file`, through the same bounded git() +// and gitBytes() as packages/project, so hooks, protocols and the environment are fixed the same +// way. Nothing in the repo is checked out or run (D7). Each R-rule is a problem, never a refusal +// of the commit; the affected file or link is left out of the index. +import { git, gitBytes } from '../../project/src/git.mjs'; +import { classify } from './scope.mjs'; +import { extractLinks, parseWikilink, resolver } from './links.mjs'; +import { LibraryError, problem, raise, sortProblems } from './problems.mjs'; + +export const MAX_BLOB = 1024 * 1024; +const MAX_LISTING = 32 * 1024 * 1024; +const SHA = /^[0-9a-f]{40}$/; +const ENTRY = /^([0-7]{6}) (blob|tree|commit) ([0-9a-f]{40}) +(-|\d+)\t(.+)$/s; +export const LINK_FIELDS = Object.freeze(['appears_in', 'relationships']); +const READERS = 8; + +async function local(promise, message) { + try { return await promise; } catch (e) { raise('library-git', `${message} (${e.code ?? 'error'})`); } +} + +export async function headCommit(repo) { + return local(git(repo, ['rev-parse', '--verify', '--end-of-options', 'HEAD^{commit}']), 'repository has no readable HEAD commit'); +} + +// Every entry of the commit's tree: { mode, type, oid, size, path }. Sizes come from `-l`, so no +// blob is opened to list it. +export async function listTree(repo, commit) { + if (typeof commit !== 'string' || !SHA.test(commit)) raise('library-commit', 'expected a full commit id'); + const kind = await local(git(repo, ['cat-file', '-t', commit]), 'commit is not in the repository'); + if (kind !== 'commit') raise('library-commit', 'not a commit'); + const bytes = await local(gitBytes(repo, ['ls-tree', '-r', '-z', '-l', '--full-tree', commit], MAX_LISTING), 'tree listing refused'); + let text; + try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); } + catch { raise('library-git', 'tree listing is not UTF-8'); } + const entries = text.split('\0'); + if (entries.pop() !== '') raise('library-git', 'invalid tree listing'); + return entries.map(entry => { + const m = ENTRY.exec(entry); + if (!m) raise('library-git', 'invalid tree listing'); + return { mode: m[1], type: m[2], oid: m[3], size: m[4] === '-' ? null : Number(m[4]), path: m[5] }; + }); +} + +// R1: only a 100644 blob up to 1 MiB of UTF-8 text without NUL is read. +export const readable = entry => entry.mode === '100644' && entry.type === 'blob' + && Number.isSafeInteger(entry.size) && entry.size <= MAX_BLOB; + +// Text of each readable entry, by path. An entry whose bytes aren't UTF-8 text is missing from the +// map, and the caller reports it under R1. +export async function readTexts(repo, entries) { + const texts = new Map(), queue = [...entries]; + const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }); + const worker = async () => { + for (let entry = queue.shift(); entry; entry = queue.shift()) { + const bytes = await local(gitBytes(repo, ['cat-file', 'blob', entry.oid], MAX_BLOB), 'blob read refused'); + let value; + try { value = decoder.decode(bytes); } catch { continue; } + if (!value.includes('\0')) texts.set(entry.path, value); + } + }; + await Promise.all(Array.from({ length: READERS }, worker)); + return texts; +} + +const QUOTED = /^"(?:[^"\\]|\\.)*"$|^'(?:[^']|'')*'$/; +const PLAIN_START = /^[^\s\[\]{}&*!|>%@`"'#,]/; +function scalar(raw) { + const value = raw.trim(); + if (parseWikilink(value)) return { ok: true, value }; + if (QUOTED.test(value)) { + if (value[0] === "'") return { ok: true, value: value.slice(1, -1).replace(/''/g, "'") }; + try { return { ok: true, value: JSON.parse(value) }; } catch { return { ok: false }; } + } + if (value === '' || value === '~' || value === 'null') return { ok: true, value: null }; + if (!PLAIN_START.test(value) || /: |\s#/.test(value) || value.endsWith(':')) return { ok: false }; + if (value === 'true' || value === 'false') return { ok: true, value: value === 'true' }; + if (/^-?(0|[1-9]\d*)(\.\d+)?$/.test(value)) return { ok: true, value: Number(value) }; + return { ok: true, value }; +} +function flow(raw) { + const inner = raw.trim().slice(1, -1), items = []; + let current = '', quote = null; + for (const ch of inner) { + if (quote) { current += ch; if (ch === quote) quote = null; continue; } + if (ch === '"' || ch === "'") { quote = ch; current += ch; continue; } + if (ch === '[' || ch === ']' || ch === '{' || ch === '}') return { ok: false }; + if (ch === ',') { items.push(current); current = ''; continue; } + current += ch; + } + if (quote) return { ok: false }; + if (current.trim() || items.length) items.push(current); + const values = []; + for (const item of items) { const s = scalar(item); if (!s.ok || !item.trim()) return { ok: false }; values.push(s.value); } + return { ok: true, value: values }; +} +function value(raw) { + const text = raw.trim(); + if (text.startsWith('[') && !parseWikilink(text)) return text.endsWith(']') ? flow(text) : { ok: false }; + return scalar(text); +} + +// Front matter of design 1.3 (R3): scalars, quoted strings, flow lists and block lists of scalars or +// wikilinks. Anything else, a nested map, a block scalar or an anchor for example, leaves the whole +// front matter as raw text with one R3 problem at the line where parsing stopped. +// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. +export function parseFrontMatter(lines) { + if (lines[0]?.replace(/^/, '').trimEnd() !== '---') return { frontmatter: null, raw: null, bodyLine: 1, stop: null }; + const close = lines.findIndex((text, i) => i > 0 && /^(---|\.\.\.)\s*$/.test(text)); + if (close < 0) return { frontmatter: null, raw: null, bodyLine: 1, stop: 1 }; + const raw = lines.slice(1, close).join('\n'), data = {}; + let list = null; + for (let i = 1; i < close; i++) { + const text = lines[i], line = i + 1; + if (!text.trim() || /^\s*#/.test(text)) continue; + const item = /^(\s*)- (.*)$|^(\s*)-$/.exec(text); + if (item && list) { + const parsed = scalar(item[2] ?? ''); + if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; + data[list].push(parsed.value); + continue; + } + const pair = /^([^\s:#\-][^:]*?|"[^"]+"|'[^']+')\s*:(?:\s+(.*))?$/.exec(text); + if (!pair) return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; + const key = pair[1].replace(/^(["'])(.*)\1$/, '$2'); + if (Object.hasOwn(data, key) || key === '__proto__') return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; + if (pair[2] === undefined || !pair[2].trim()) { + const next = lines[i + 1]; + if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; list = key; } + else { data[key] = null; list = null; } + continue; + } + list = null; + const parsed = value(pair[2]); + if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; + data[key] = parsed.value; + } + return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null }; +} + +export const splitLines = text => text.replace(/^/, '').split('\n').map(l => l.endsWith('\r') ? l.slice(0, -1) : l); + +// One parsed file: its lines (all of them, numbered from 1), front matter and the links it holds. +// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. +export function parseFile(text, { markdown = false } = {}) { + const lines = splitLines(text); + const fm = parseFrontMatter(lines); + const numbered = lines.map((t, i) => [i + 1, t]); + const links = extractLinks(numbered.slice(fm.bodyLine - 1), { markdown }); + if (fm.frontmatter) { + for (const field of LINK_FIELDS) { + const raw = fm.frontmatter[field]; + const values = Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw]; + for (const v of values) { + const link = typeof v === 'string' ? parseWikilink(v.trim()) : null; + if (link) links.push({ ...link, field, line: frontMatterLine(lines, fm.bodyLine, field) }); + } + } + } + return { lines: numbered, bodyLine: fm.bodyLine, frontmatter: fm.frontmatter, frontmatterRaw: fm.raw, stop: fm.stop, links }; +} +function frontMatterLine(lines, bodyLine, field) { + const i = lines.slice(0, bodyLine - 1).findIndex(t => new RegExp(`^["']?${field}["']?\\s*:`).test(t)); + return i < 0 ? undefined : i + 1; +} + +// Read a commit under a checked mapping (mapping.mjs). Returns the indexed files and every +// R1-R7 problem: { commit, files: [...], problems: [...] }, frozen, sorted by path. +export async function readLibrary({ repo, commit, mapping }) { + if (!mapping || mapping.kind !== 'library') throw new LibraryError('library-mapping-invalid', 'expected a checked library mapping'); + const entries = await listTree(repo, commit); + const problems = [], twins = new Set(), byLower = new Map(); + for (const e of entries) byLower.set(e.path.toLowerCase(), [...(byLower.get(e.path.toLowerCase()) ?? []), e.path]); + for (const group of byLower.values()) if (group.length > 1) for (const path of group) twins.add(path); + + const candidates = []; + for (const entry of entries) { + const where = classify(mapping, entry.path); + if (where.scope === 'unmapped') { problems.push(problem('R2', entry.path)); continue; } + if (twins.has(entry.path)) { problems.push(problem('R7', entry.path)); continue; } + if (!readable(entry)) { problems.push(problem('R1', entry.path)); continue; } + candidates.push({ entry, where }); + } + const texts = await readTexts(repo, candidates.map(c => c.entry)); + const files = []; + for (const { entry, where } of candidates) { + if (!texts.has(entry.path)) { problems.push(problem('R1', entry.path)); continue; } + const parsed = parseFile(texts.get(entry.path)); + if (parsed.stop !== null) problems.push(problem('R3', entry.path, parsed.stop)); + files.push({ path: entry.path, ...where, name: entry.path.split('/').pop().replace(/\.md$/, ''), ...parsed }); + } + + const resolve = resolver(files.map(f => f.path), { vaultPrefix: mapping.vaultPrefix }); + const byPath = new Map(files.map(f => [f.path, f])); + for (const file of files) { + const kept = []; + for (const link of file.links) { + if (!link.file) continue; + const target = resolve(file.path, link.file); + if (!target.path) { problems.push(problem('R4', file.path, link.line)); continue; } + const to = byPath.get(target.path); + if (file.scope === 'canon') { + if (to.scope === 'playground') problems.push(problem('R5', file.path, link.line)); + const sources = mapping.universes[file.universe].sources ?? []; + if (to.universe !== file.universe && !sources.includes(to.universe)) problems.push(problem('R6', file.path, link.line)); + } + kept.push({ ...link, path: target.path, step: target.step }); + } + file.links = kept; + delete file.stop; + } + files.sort((a, b) => a.path < b.path ? -1 : 1); + return deepFreeze({ commit, files, problems: sortProblems(problems) }); +} + +function deepFreeze(value) { + if (value && typeof value === 'object' && !Object.isFrozen(value)) { Object.freeze(value); for (const v of Object.values(value)) deepFreeze(v); } + return value; +} diff --git a/packages/library/src/render.mjs b/packages/library/src/render.mjs new file mode 100644 index 000000000..398ee0090 --- /dev/null +++ b/packages/library/src/render.mjs @@ -0,0 +1,59 @@ +// renderItems(body): how a stored cited reply looks (design 7.4), shared by the Console chat (C2-10b) +// and the Discord relay (C2-18). Pure: no git, no clock. The caller passes what it knows about cites. +// The guard (C2-06) checks limits before a reply is stored. This only refuses to show a body that +// doesn't have the reply shape, and then it shows a fixed line, never the body's text. +export const KINDS = Object.freeze({ fact: 'Fact', finding: 'Finding', correction: 'Correction', question: 'Question', idea: 'Idea' }); +export const NOTICES = Object.freeze({ 'no-cited-answer': 'No cited answer', 'no-answer': 'No answer', 'turn-failed': 'The turn failed' }); +export const UNREADABLE = 'This reply could not be shown.'; +const SHA = /^[0-9a-f]{40}$/; + +const plain = value => value !== null && typeof value === 'object' && !Array.isArray(value); +const line = n => Number.isSafeInteger(n) && n > 0; +function cite(value, { scopeOf, changed }) { + if (!plain(value) || typeof value.quote !== 'string') return null; + if (Object.hasOwn(value, 'url')) { + if (typeof value.url !== 'string' || Object.hasOwn(value, 'path')) return null; + return { badge: 'web', url: value.url, ref: value.url, quote: value.quote, changed: false }; + } + const { path, commit, lines } = value; + if (typeof path !== 'string' || !SHA.test(commit ?? '') || !Array.isArray(lines) || lines.length !== 2 + || !line(lines[0]) || !line(lines[1]) || lines[1] < lines[0]) return null; + const badge = scopeOf?.(path); + const ref = `${path}:${lines[0] === lines[1] ? lines[0] : `${lines[0]}-${lines[1]}`} @${commit.slice(0, 7)}`; + return { badge: badge === 'canon' || badge === 'playground' ? badge : null, path, commit, lines: [...lines], ref, + quote: value.quote, changed: changed?.({ path, commit }) === true }; +} +const quoted = text => JSON.stringify(text); + +// body: the stored message body (a JSON string or the parsed object). +// options.scopeOf(path) -> 'canon' | 'playground' | undefined, for the scope badge. +// options.changed({path, commit}) -> true when the file's last commit is newer than the cited one. +// Returns { kind: 'items', items, text } | { kind: 'notice', notice, text } | { kind: 'unreadable', text }. +export function renderItems(body, options = {}) { + const unreadable = { kind: 'unreadable', text: UNREADABLE }; + let value = body; + if (typeof body === 'string') { try { value = JSON.parse(body); } catch { return unreadable; } } + if (!plain(value) || !Array.isArray(value.items)) return unreadable; + const keys = Object.keys(value); + if (value.items.length === 0) { + if (keys.length !== 2 || !Object.hasOwn(NOTICES, value.notice)) return unreadable; + return { kind: 'notice', notice: value.notice, text: NOTICES[value.notice] }; + } + if (keys.length !== 1) return unreadable; + const items = []; + for (const item of value.items) { + if (!plain(item) || !Object.hasOwn(KINDS, item.kind) || typeof item.text !== 'string' || !Array.isArray(item.cites)) return unreadable; + const correction = item.kind === 'correction'; + if (correction && (typeof item.from !== 'string' || typeof item.to !== 'string')) return unreadable; + const cites = item.cites.map(c => cite(c, options)); + if (cites.some(c => c === null)) return unreadable; + items.push({ kind: item.kind, label: KINDS[item.kind], text: item.text, + ...(correction ? { from: item.from, to: item.to } : {}), cites }); + } + const text = items.map((item, i) => [ + `${i + 1}. ${item.label}: ${item.text}`, + ...(item.kind === 'correction' ? [` From: ${quoted(item.from)}`, ` To: ${quoted(item.to)}`] : []), + ...item.cites.map(c => ` ${c.badge ? `[${c.badge}] ` : ''}${c.ref} ${quoted(c.quote)}${c.changed ? ' (changed since)' : ''}`), + ].join('\n')).join('\n'); + return { kind: 'items', items, text }; +} diff --git a/packages/library/src/scope.mjs b/packages/library/src/scope.mjs new file mode 100644 index 000000000..5a95c4fce --- /dev/null +++ b/packages/library/src/scope.mjs @@ -0,0 +1,21 @@ +// The scope rule of design 1.2. Scope comes from the path alone (D2): no file content can change it. +// Paths are repo-relative and compared exactly, case included (R7 handles case-only twins). +const inside = (folder, path) => path.startsWith(folder.endsWith('/') ? folder : `${folder}/`); + +// { scope: 'playground' | 'canon', universe, book, type, folder } or { scope: 'unmapped' }. +// `type` is the mapped type of the first folder under the book (canon) or under the playground +// prefix (playground), 'manuscript' for a manuscript folder, else null. `folder` is that folder name. +export function classify(mapping, path) { + if (typeof path !== 'string' || !path.endsWith('.md')) return { scope: 'unmapped' }; + for (const [universe, entry] of Object.entries(mapping.universes)) { + const book = entry.books.find(b => inside(b, path)); + if (!book) continue; + const prefix = entry.playground.find(p => inside(p, path)); + const rest = path.slice((prefix ?? `${book}/`).length).split('/'); + const folder = rest.length > 1 ? rest[0] : null; + const type = folder === null ? null : Object.hasOwn(mapping.types, folder) ? mapping.types[folder] + : mapping.manuscript.includes(folder) ? 'manuscript' : null; + return { scope: prefix ? 'playground' : 'canon', universe, book, type, folder }; + } + return { scope: 'unmapped' }; +} diff --git a/packages/library/src/survey.mjs b/packages/library/src/survey.mjs new file mode 100644 index 000000000..3e72623fa --- /dev/null +++ b/packages/library/src/survey.mjs @@ -0,0 +1,126 @@ +// `mosaic library survey [--mapping ]` (design 1.1, 10 step 7). It prints what a mapping +// needs: folder names, file counts by extension, front matter key sets and link-form counts. It +// never prints file text, front matter values, file names or link targets. With a mapping it adds +// scope counts, resolver step counts and R-problem counts by code. It reads the repo's HEAD commit +// through the commit reader, never the working tree. +import { readFileSync, realpathSync, statSync } from 'node:fs'; +import { resolve as resolvePath } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { checkMapping } from './mapping.mjs'; +import { classify } from './scope.mjs'; +import { formOf } from './links.mjs'; +import { headCommit, listTree, parseFile, readable, readLibrary, readTexts } from './reader.mjs'; +import { LibraryError, RULES } from './problems.mjs'; + +export const USAGE = 'usage: mosaic library survey [--mapping ]'; +// Keys are printed only in the short identifier forms property names take. A key with spaces or of +// any other shape could be a phrase of hers, so it is counted as one "(other key)". +const KEY = /^[A-Za-z_][A-Za-z0-9_-]{0,31}$/; +const OTHER_KEY = '(other key)'; +const FORMS = ['plain', 'label', 'heading', 'embed', 'self', 'markdown']; +const STEPS = ['root', 'vault-prefix', 'relative', 'shortest']; +const extension = path => /\.md$/.test(path) ? 'md' : /\.canvas$/.test(path) ? 'canvas' : /\.base$/.test(path) ? 'base' : 'other'; + +// The survey as data. `mapping` is a checked mapping or undefined. +export async function survey({ repo, commit, mapping }) { + const head = commit ?? await headCommit(repo); + const entries = await listTree(repo, head); + const folders = new Map(); + for (const entry of entries) { + const slash = entry.path.lastIndexOf('/'); + const name = slash < 0 ? '' : entry.path.slice(0, slash); + const counts = folders.get(name) ?? { md: 0, canvas: 0, base: 0, other: 0 }; + counts[entry.type === 'blob' ? extension(entry.path) : 'other']++; + folders.set(name, counts); + } + const markdown = entries.filter(e => e.path.endsWith('.md')); + const texts = await readTexts(repo, markdown.filter(readable)); + const keySets = new Map(), forms = Object.fromEntries(FORMS.map(f => [f, 0])); + const fields = { appears_in: { wikilink: 0, name: 0 }, relationships: { wikilink: 0, name: 0 } }; + let noFrontMatter = 0, unparsed = 0; + for (const text of texts.values()) { + const file = parseFile(text, { markdown: true }); + if (file.stop !== null) unparsed++; + else if (!file.frontmatter) noFrontMatter++; + else { + const set = [...new Set(Object.keys(file.frontmatter).map(k => KEY.test(k) ? k : OTHER_KEY))].sort().join(', '); + keySets.set(set, (keySets.get(set) ?? 0) + 1); + for (const field of Object.keys(fields)) { + const raw = file.frontmatter[field]; + for (const v of Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw]) + fields[field][typeof v === 'string' && /^\s*!?\[\[[^\]]+\]\]\s*$/.test(v) ? 'wikilink' : 'name']++; + } + } + for (const link of file.links) if (!link.field) forms[formOf(link)]++; + } + const result = { + commit: head, entries: entries.length, + folders: [...folders].sort(([a], [b]) => a < b ? -1 : 1).map(([name, counts]) => ({ name, ...counts })), + markdown: { files: markdown.length, read: texts.size, noFrontMatter, unparsed }, + keySets: [...keySets].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([keys, count]) => ({ keys, count })), + forms, fields, + }; + if (mapping) { + const scopes = { canon: 0, playground: 0, unmapped: 0 }; + for (const entry of entries) scopes[classify(mapping, entry.path).scope]++; + const library = await readLibrary({ repo, commit: head, mapping }); + const steps = Object.fromEntries(STEPS.map(s => [s, 0])); + for (const file of library.files) for (const link of file.links) steps[link.step]++; + const problems = Object.fromEntries(Object.keys(RULES).map(code => [code, 0])); + for (const p of library.problems) problems[p.code]++; + steps.unresolved = problems.R4; + Object.assign(result, { scopes, indexed: library.files.length, steps, problems }); + } + return result; +} + +// Folder names are quoted as JSON strings, so no name can carry a control sequence to the terminal. +export function formatSurvey(s) { + const out = [`commit ${s.commit}`, `entries ${s.entries}`, 'folders (direct children: md canvas base other)']; + for (const f of s.folders) out.push(` ${f.name === '' ? '(root)' : JSON.stringify(f.name)} ${f.md} ${f.canvas} ${f.base} ${f.other}`); + out.push(`markdown ${s.markdown.files} files, ${s.markdown.read} read, ${s.markdown.noFrontMatter} without front matter, ${s.markdown.unparsed} with front matter not fully parsed`); + out.push('front matter key sets'); + for (const k of s.keySets) out.push(` ${k.count} [${k.keys}]`); + out.push(`body links ${FORMS.map(f => `${f} ${s.forms[f]}`).join(', ')}`); + for (const [field, c] of Object.entries(s.fields)) out.push(`field ${field} wikilink ${c.wikilink}, name ${c.name}`); + if (s.scopes) { + out.push(`scopes canon ${s.scopes.canon}, playground ${s.scopes.playground}, unmapped ${s.scopes.unmapped}; indexed ${s.indexed}`); + out.push(`resolved ${STEPS.map(st => `${st} ${s.steps[st]}`).join(', ')}, unresolved ${s.steps.unresolved}`); + out.push(`problems ${Object.entries(s.problems).map(([code, n]) => `${code} ${n}`).join(', ')}`); + } + return `${out.join('\n')}\n`; +} + +// Exit codes as packages/project's CLI: 0 done, 2 the repo can't be read, 3 a bad mapping, 4 usage. +export async function surveyMain(argv, io = { stdout: process.stdout, stderr: process.stderr }) { + const fail = (code, message) => { io.stderr.write(`library survey: ${message}\n`); return code; }; + const [verb, ...rest] = argv; + if (verb !== 'survey') return fail(4, USAGE); + let repo, mappingFile; + for (let i = 0; i < rest.length; i++) { + if (rest[i] === '--mapping' && i + 1 < rest.length && mappingFile === undefined) mappingFile = rest[++i]; + else if (!rest[i].startsWith('-') && repo === undefined) repo = rest[i]; + else return fail(4, USAGE); + } + if (repo === undefined) return fail(4, USAGE); + let mapping; + if (mappingFile !== undefined) { + let entry; + try { entry = JSON.parse(readFileSync(mappingFile, 'utf8')); } + catch { return fail(3, `library-mapping-invalid: ${mappingFile} is not a readable JSON file`); } + try { mapping = checkMapping(entry, mappingFile); } + catch (e) { if (e instanceof LibraryError) return fail(3, e.message); throw e; } + } + const path = resolvePath(repo); + try { if (!statSync(path).isDirectory()) return fail(2, `library-git: ${repo} is not a directory`); } + catch { return fail(2, `library-git: ${repo} is not a directory`); } + try { io.stdout.write(formatSurvey(await survey({ repo: path, mapping }))); } + catch (e) { if (e instanceof LibraryError) return fail(2, e.message); throw e; } + return 0; +} + +const entry = () => { try { return import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href; } catch { return false; } }; +if (process.argv[1] && entry()) { + const code = await surveyMain(process.argv.slice(2)); + if (code !== 0) process.exitCode = code; +} diff --git a/packages/library/tests/fixtures/mapping.json b/packages/library/tests/fixtures/mapping.json new file mode 100644 index 000000000..90f2734e9 --- /dev/null +++ b/packages/library/tests/fixtures/mapping.json @@ -0,0 +1,12 @@ +{ + "kind": "library", + "remote": "https://git.example.invalid/writer/books.git", + "branch": "main", + "vaultPrefix": "books", + "universes": { + "harbor": {"name": "The Harbor", "books": ["Book One"], "playground": ["Book One/Playground/"]}, + "second": {"name": "Second World", "books": ["Second World"]} + }, + "types": {"Characters": "character", "Places": "place", "Research": "research", "Tracker": "tracker", "Summaries": "summary"}, + "manuscript": ["Chapters"] +} diff --git a/packages/library/tests/helpers.mjs b/packages/library/tests/helpers.mjs new file mode 100644 index 000000000..635efd5c8 --- /dev/null +++ b/packages/library/tests/helpers.mjs @@ -0,0 +1,54 @@ +// A repo the test builds in her layout's shape: spaces and capitals in folder names, a .canvas file +// and an unmapped top-level note. Bodies are made up; MARKER stands in for her text. +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, symlinkSync, chmodSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { checkMapping } from '../src/mapping.mjs'; + +export const MARKER = 'MARKER-7f3a-HER-TEXT'; +export const FIXTURES = fileURLToPath(new URL('./fixtures/', import.meta.url)); +export const mappingEntry = () => JSON.parse(readFileSync(join(FIXTURES, 'mapping.json'), 'utf8')); +export const mapping = (change = e => e) => checkMapping(change(mappingEntry())); + +export const git = (repo, ...args) => execFileSync('git', ['-c', 'user.name=fixture', '-c', 'user.email=fixture@mosaic.invalid', + '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=/dev/null', '-C', repo, ...args], { encoding: 'utf8' }).trim(); + +export const BASE = { + 'Book One/Characters/Mara Vell.md': [ + '---', 'type: character', 'name: Mara Vell', 'appears_in: "[[Book One/Chapters/Chapter 01]]"', + 'relationships:', ' - "[[Tomas]]"', ' - The Ferryman', '---', `Grey eyes. ${MARKER}`, ''].join('\n'), + 'Book One/Characters/Tomas.md': ['---', 'type: character', 'name: Tomas', 'appears_in: ["[[Chapter 01]]"]', '---', MARKER, ''].join('\n'), + 'Book One/Chapters/Chapter 01.md': [ + '# One', `${MARKER} [[Mara Vell]] met [[Book One/Places/Harbor Town|the harbor]].`, + 'See [[../Places/Harbor Town#Docks]] and [[books/Book One/Characters/Tomas]].', + '```', '[[Not A Link]]', '```', 'Inline `[[Not Either]]` and [a note](Notes.md).', ''].join('\n'), + 'Book One/Places/Harbor Town.md': ['---', 'type: place', '---', `Salt. ${MARKER}`, ''].join('\n'), + 'Book One/Playground/What If.md': ['---', 'name: What If', '---', `${MARKER} [[Mara Vell]]`, ''].join('\n'), + 'Book One/Board.canvas': '{"nodes":[]}\n', + 'Book One/Playground/Sketch.canvas': '{"nodes":[]}\n', + 'Inbox Note.md': `${MARKER}\n`, + 'Second World/Chapters/01.md': `# Elsewhere ${MARKER}\n`, +}; + +// repo({files, extra}) writes files (path -> text, or { link } / { mode, text }), commits, and +// returns { repo, commit }. +export function repo(t, files = BASE) { + const root = mkdtempSync(join(tmpdir(), 'library-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const dir = join(root, 'books'); + mkdirSync(dir); + git(dir, 'init', '-q', '-b', 'main'); + for (const [path, value] of Object.entries(files)) { + const file = join(dir, path); + mkdirSync(dirname(file), { recursive: true }); + if (typeof value === 'object' && value.link) symlinkSync(value.link, file); + else if (typeof value === 'object' && value.bytes) writeFileSync(file, value.bytes); + else writeFileSync(file, typeof value === 'object' ? value.text : value); + if (typeof value === 'object' && value.mode) chmodSync(file, value.mode); + } + git(dir, 'add', '-A'); + git(dir, 'commit', '-qm', 'fixture'); + return { root, repo: dir, commit: git(dir, 'rev-parse', 'HEAD') }; +} diff --git a/packages/library/tests/links.test.mjs b/packages/library/tests/links.test.mjs new file mode 100644 index 000000000..8cef702ed --- /dev/null +++ b/packages/library/tests/links.test.mjs @@ -0,0 +1,50 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { extractLinks, parseWikilink, resolver } from '../src/index.mjs'; + +const PATHS = ['Book One/Characters/Mara Vell.md', 'Book One/Characters/Tomas.md', 'Book One/Places/Harbor Town.md', + 'Book One/Chapters/Chapter 01.md', 'Book One/Places/Notes.md', 'Book One/Characters/Notes.md', 'Mara Vell.md']; +const resolve = resolver(PATHS, { vaultPrefix: 'books' }); +const FROM = 'Book One/Chapters/Chapter 01.md'; + +test('resolver step 1: a path from the repo root, with or without .md', () => { + assert.deepEqual(resolve(FROM, 'Book One/Places/Harbor Town'), { path: 'Book One/Places/Harbor Town.md', step: 'root' }); + assert.deepEqual(resolve(FROM, 'Book One/Places/Harbor Town.md'), { path: 'Book One/Places/Harbor Town.md', step: 'root' }); + // Root wins over shortest: Mara Vell.md at the root, not the character page. + assert.deepEqual(resolve(FROM, 'Mara Vell'), { path: 'Mara Vell.md', step: 'root' }); +}); + +test('resolver step 2: the same path with vaultPrefix/ stripped', () => { + assert.deepEqual(resolve(FROM, 'books/Book One/Characters/Tomas'), { path: 'Book One/Characters/Tomas.md', step: 'vault-prefix' }); + assert.deepEqual(resolver(PATHS)(FROM, 'books/Book One/Characters/Tomas'), { path: null, step: 'missing' }); +}); + +test('resolver step 3: a path relative to the linking file, never above the repo root', () => { + assert.deepEqual(resolve(FROM, '../Places/Harbor Town'), { path: 'Book One/Places/Harbor Town.md', step: 'relative' }); + assert.deepEqual(resolve('Book One/Places/Harbor Town.md', './Notes'), { path: 'Book One/Places/Notes.md', step: 'relative' }); + assert.deepEqual(resolve(FROM, '../../../Book One/Places/Harbor Town'), { path: null, step: 'missing' }); +}); + +test('resolver step 4: a base name that matches exactly one indexed file', () => { + assert.deepEqual(resolve(FROM, 'Tomas'), { path: 'Book One/Characters/Tomas.md', step: 'shortest' }); + assert.deepEqual(resolve(FROM, 'Harbor Town.md'), { path: 'Book One/Places/Harbor Town.md', step: 'shortest' }); + assert.deepEqual(resolve(FROM, 'Notes'), { path: null, step: 'ambiguous', count: 2 }); + assert.deepEqual(resolve(FROM, 'Nobody'), { path: null, step: 'missing' }); + // A partial path is not a base name. + assert.deepEqual(resolve(FROM, 'Places/Harbor Town'), { path: null, step: 'missing' }); +}); + +test('a link into another of her repos resolves to nothing', () => { + assert.deepEqual(resolve(FROM, 'bean-brain/Journal/Today'), { path: null, step: 'missing' }); + assert.deepEqual(resolve(FROM, '../bean-brain/Journal/Today'), { path: null, step: 'missing' }); +}); + +test('wikilink forms, embeds, and links skipped inside code', () => { + assert.deepEqual(parseWikilink('[[A/B#Head|Label]]'), { file: 'A/B', heading: 'Head', label: 'Label', embed: false }); + assert.deepEqual(parseWikilink('![[pic.png]]'), { file: 'pic.png', heading: null, label: null, embed: true }); + assert.equal(parseWikilink('[[a]] and more'), null); + const lines = [[1, '[[One]] [[Two|2]] [[Three#H]] ![[Four]] [[#Self]]'], [2, '```js'], [3, '[[Fenced]]'], [4, '```'], + [5, 'Inline `[[Code]]` and [md](Five.md) and [web](https://example.invalid/x)']]; + assert.deepEqual(extractLinks(lines).map(l => [l.file, l.line]), [['One', 1], ['Two', 1], ['Three', 1], ['Four', 1], ['', 1]]); + assert.deepEqual(extractLinks(lines, { markdown: true }).filter(l => l.markdown).map(l => l.file), ['Five.md']); +}); diff --git a/packages/library/tests/mapping.test.mjs b/packages/library/tests/mapping.test.mjs new file mode 100644 index 000000000..fe0992a5c --- /dev/null +++ b/packages/library/tests/mapping.test.mjs @@ -0,0 +1,82 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { checkMapping, classify, LibraryError } from '../src/index.mjs'; +import { mapping, mappingEntry } from './helpers.mjs'; + +const refuses = (change, code, pattern) => assert.throws(() => checkMapping(change(mappingEntry())), + e => e instanceof LibraryError && e.code === code && (!pattern || pattern.test(e.message))); + +test('the fixture mapping checks and comes back frozen', () => { + const m = mapping(); + assert.equal(m.universes.harbor.playground[0], 'Book One/Playground/'); + assert.ok(Object.isFrozen(m.universes.harbor.books)); + assert.throws(() => { m.types.Extra = 'place'; }); +}); + +test('a playground prefix outside its universe books refuses library-playground-outside-book', () => { + refuses(e => { e.universes.harbor.playground = ['Second World/Playground/']; return e; }, 'library-playground-outside-book'); + refuses(e => { e.universes.harbor.playground = ['Elsewhere/']; return e; }, 'library-playground-outside-book'); + // The whole book as its playground would leave no canon: not inside the book. + refuses(e => { e.universes.harbor.playground = ['Book One/']; return e; }, 'library-playground-outside-book'); +}); + +test('no playground prefix holds another, and no book folder holds another', () => { + refuses(e => { e.universes.harbor.playground = ['Book One/Playground/', 'Book One/Playground/Deep/']; return e; }, + 'library-mapping-invalid', /holds playground/); + refuses(e => { e.universes.second.books = ['Second World', 'Book One/Part Two']; return e; }, 'library-mapping-invalid', /holds book folder/); + refuses(e => { e.universes.second.books = ['Book One']; return e; }, 'library-mapping-invalid', /in two universes/); +}); + +test('shape refusals: kind, remote, branch, unknown keys, types, manuscript, folder names', () => { + refuses(e => ({ ...e, kind: 'code' }), 'library-mapping-invalid', /kind/); + refuses(e => ({ ...e, remote: 'http://git.example.invalid/x.git' }), 'library-mapping-invalid', /https/); + refuses(e => ({ ...e, remote: 'https://user:secret@git.example.invalid/x.git' }), 'library-mapping-invalid', /credentials/); + refuses(e => ({ ...e, remote: 'git@git.example.invalid:x.git' }), 'library-mapping-invalid', /https/); + refuses(e => ({ ...e, branch: 'main..x' }), 'library-mapping-invalid', /branch/); + refuses(e => ({ ...e, root: '/tmp' }), 'library-mapping-invalid', /unknown key "root"/); + refuses(e => { e.universes.harbor.extra = 1; return e; }, 'library-mapping-invalid', /unknown key/); + refuses(e => { e.types.Characters = 'villain'; return e; }, 'library-mapping-invalid', /types.Characters/); + refuses(e => { e.types['A/B'] = 'place'; return e; }, 'library-mapping-invalid', /one folder name/); + refuses(e => ({ ...e, manuscript: ['Characters'] }), 'library-mapping-invalid', /also a type folder/); + refuses(e => { e.universes.harbor.books = ['../Book One']; return e; }, 'library-mapping-invalid', /folder path/); + refuses(e => ({ ...e, vaultPrefix: 'a/b' }), 'library-mapping-invalid', /vaultPrefix/); + refuses(e => ({ ...e, universes: {} }), 'library-mapping-invalid', /at least one universe/); + refuses(e => { e.universes['bad id'] = e.universes.second; delete e.universes.second; return e; }, 'library-mapping-invalid', /plain id/); +}); + +test('a hybrid universe is a shape the query code can take (1.4), with sources that must exist', () => { + const hybrid = e => { e.universes.cross = { kind: 'hybrid', name: 'Crossover', sources: ['harbor', 'second'], books: ['Crossover'], + constraints: [{ exclude: 'second/Characters' }, { pin: { entity: 'Book One/Characters/Mara Vell.md', field: 'eyes', value: 'grey' } }], + notes: 'Set after book one.' }; return e; }; + const m = checkMapping(hybrid(mappingEntry())); + assert.deepEqual(m.universes.cross.sources, ['harbor', 'second']); + refuses(e => { hybrid(e); e.universes.cross.sources = ['harbor', 'nowhere']; return e; }, 'library-mapping-invalid', /non-hybrid universe/); + refuses(e => { hybrid(e); e.universes.cross.sources = ['harbor']; return e; }, 'library-mapping-invalid', /two or more/); + refuses(e => { hybrid(e); e.universes.cross.constraints = [{ exclude: 'x', pin: {} }]; return e; }, 'library-mapping-invalid', /exactly one/); +}); + +test('a caller-supplied fail receives the code, as business file loading will pass it (C2-04)', () => { + const seen = []; + const fail = (code, message) => { seen.push(code); throw new Error(message); }; + const entry = mappingEntry(); + entry.universes.harbor.playground = ['Elsewhere/']; + assert.throws(() => checkMapping(entry, 'businesses/acme.json projects.books', { fail }), /businesses\/acme.json projects.books/); + assert.deepEqual(seen, ['library-playground-outside-book']); +}); + +test('scope comes from the path alone (design 1.2)', () => { + const m = mapping(); + assert.deepEqual(classify(m, 'Book One/Characters/Mara Vell.md'), + { scope: 'canon', universe: 'harbor', book: 'Book One', type: 'character', folder: 'Characters' }); + assert.deepEqual(classify(m, 'Book One/Chapters/Chapter 01.md'), + { scope: 'canon', universe: 'harbor', book: 'Book One', type: 'manuscript', folder: 'Chapters' }); + assert.deepEqual(classify(m, 'Book One/Playground/What If.md'), + { scope: 'playground', universe: 'harbor', book: 'Book One', type: null, folder: null }); + assert.deepEqual(classify(m, 'Book One/Playground/Characters/New.md'), + { scope: 'playground', universe: 'harbor', book: 'Book One', type: 'character', folder: 'Characters' }); + assert.deepEqual(classify(m, 'Book One/Notes.md'), { scope: 'canon', universe: 'harbor', book: 'Book One', type: null, folder: null }); + assert.equal(classify(m, 'Second World/Chapters/01.md').universe, 'second'); + for (const path of ['Inbox Note.md', 'Book One/Board.canvas', 'Book One/Playground/Sketch.canvas', 'Book One/Board.base', + 'Book One/cover.png', 'Book One/Upper.MD', 'Book Onex/a.md', 'book one/a.md', 'Elsewhere/Book One/a.md']) + assert.deepEqual(classify(m, path), { scope: 'unmapped' }, path); +}); diff --git a/packages/library/tests/reader.test.mjs b/packages/library/tests/reader.test.mjs new file mode 100644 index 000000000..1913e2974 --- /dev/null +++ b/packages/library/tests/reader.test.mjs @@ -0,0 +1,151 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { readLibrary, parseFrontMatter, LibraryError, MAX_BLOB } from '../src/index.mjs'; +import { BASE, git, mapping, repo } from './helpers.mjs'; + +const codes = (result, path) => result.problems.filter(p => p.path === path).map(p => p.code); +const file = (result, path) => result.files.find(f => f.path === path); + +test('the fixture reads clean: every resolver step, appears_in and relationships wikilinks indexed', async t => { + const r = repo(t); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.equal(result.commit, r.commit); + assert.deepEqual(result.files.map(f => [f.path, f.scope]), [ + ['Book One/Chapters/Chapter 01.md', 'canon'], ['Book One/Characters/Mara Vell.md', 'canon'], + ['Book One/Characters/Tomas.md', 'canon'], ['Book One/Places/Harbor Town.md', 'canon'], + ['Book One/Playground/What If.md', 'playground'], ['Second World/Chapters/01.md', 'canon']]); + const chapter = file(result, 'Book One/Chapters/Chapter 01.md'); + assert.deepEqual(chapter.links.map(l => [l.path, l.step, l.line]), [ + ['Book One/Characters/Mara Vell.md', 'shortest', 2], ['Book One/Places/Harbor Town.md', 'root', 2], + ['Book One/Places/Harbor Town.md', 'relative', 3], ['Book One/Characters/Tomas.md', 'vault-prefix', 3]]); + const mara = file(result, 'Book One/Characters/Mara Vell.md'); + assert.deepEqual(mara.frontmatter, { type: 'character', name: 'Mara Vell', appears_in: '[[Book One/Chapters/Chapter 01]]', + relationships: ['[[Tomas]]', 'The Ferryman'] }); + assert.deepEqual(mara.links.map(l => [l.field, l.path, l.line]), [ + ['appears_in', 'Book One/Chapters/Chapter 01.md', 4], ['relationships', 'Book One/Characters/Tomas.md', 5]]); + assert.deepEqual(file(result, 'Book One/Characters/Tomas.md').links.map(l => [l.field, l.path]), + [['appears_in', 'Book One/Chapters/Chapter 01.md']]); + assert.equal(mara.bodyLine, 9); + assert.equal(mara.lines[8][0], 9); + assert.equal(mara.type, 'character'); + // Only the unmapped paths are problems: the top-level note and both .canvas files. + assert.deepEqual(result.problems.map(p => [p.code, p.path]), [ + ['R2', 'Book One/Board.canvas'], ['R2', 'Book One/Playground/Sketch.canvas'], ['R2', 'Inbox Note.md']]); + assert.ok(Object.isFrozen(result.files[0].links)); +}); + +test('R1: executables, symlinks, gitlinks, blobs over 1 MiB and non-UTF-8 text are not indexed', async t => { + const r = repo(t, { ...BASE, + 'Book One/Places/Run.md': { text: 'x\n', mode: 0o755 }, + 'Book One/Places/Link.md': { link: 'Harbor Town.md' }, + 'Book One/Places/Big.md': 'x'.repeat(MAX_BLOB + 1), + 'Book One/Places/Edge.md': 'y'.repeat(MAX_BLOB), + 'Book One/Places/Latin.md': { bytes: Buffer.from([0x63, 0x61, 0x66, 0xe9, 0x0a]) }, + 'Book One/Places/Nul.md': 'a\0b\n' }); + git(r.repo, 'update-index', '--add', '--cacheinfo', `160000,${r.commit},Book One/Places/Sub.md`); + git(r.repo, 'commit', '-qm', 'gitlink'); + const result = await readLibrary({ repo: r.repo, commit: git(r.repo, 'rev-parse', 'HEAD'), mapping: mapping() }); + for (const name of ['Run', 'Link', 'Big', 'Latin', 'Nul', 'Sub']) { + assert.deepEqual(codes(result, `Book One/Places/${name}.md`), ['R1'], name); + assert.equal(file(result, `Book One/Places/${name}.md`), undefined, name); + } + assert.ok(file(result, 'Book One/Places/Edge.md'), 'exactly 1 MiB is read'); +}); + +test('R2: unmapped paths are reported and not indexed', async t => { + const r = repo(t); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + for (const path of ['Inbox Note.md', 'Book One/Board.canvas', 'Book One/Playground/Sketch.canvas']) { + assert.deepEqual(codes(result, path), ['R2']); + assert.equal(file(result, path), undefined); + } +}); + +test('R3: front matter not fully parsed stays raw, with the line where parsing stopped', async t => { + const r = repo(t, { ...BASE, + 'Book One/Places/Nested.md': ['---', 'type: place', 'coords:', ' lat: 1', '---', 'Body [[Tomas]]', ''].join('\n'), + 'Book One/Places/Block.md': ['---', 'summary: |', ' text', '---', ''].join('\n'), + 'Book One/Places/Open.md': ['---', 'type: place', 'never closed', ''].join('\n') }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + const nested = file(result, 'Book One/Places/Nested.md'); + assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Places/Nested.md'), [{ code: 'R3', path: 'Book One/Places/Nested.md', line: 4, message: 'front matter not fully parsed' }]); + assert.equal(nested.frontmatter, null); + assert.equal(nested.frontmatterRaw, 'type: place\ncoords:\n lat: 1'); + assert.deepEqual(nested.links.map(l => l.path), ['Book One/Characters/Tomas.md'], 'still indexed by body'); + assert.equal(result.problems.find(p => p.path === 'Book One/Places/Block.md').line, 2); + assert.equal(result.problems.find(p => p.path === 'Book One/Places/Open.md').line, 1); + assert.equal(file(result, 'Book One/Places/Open.md').bodyLine, 1); +}); + +test('front matter forms the reader accepts and refuses', () => { + const parse = text => parseFrontMatter(text.split('\n')); + assert.deepEqual(parse('---\na: plain words\nb: "quoted: yes"\nc: \'it\'\'s\'\nd: [x, "y z", 3]\ne:\n- one\n- "[[Two]]"\nf: [[Wiki]]\ng: true\nh:\n---').frontmatter, + { a: 'plain words', b: 'quoted: yes', c: "it's", d: ['x', 'y z', 3], e: ['one', '[[Two]]'], f: '[[Wiki]]', g: true, h: null }); + for (const bad of ['a: {x: 1}', 'a: &anchor x', 'a: [x, [y]]', 'a: x: y', 'a: 1\na: 2', ' a: 1', 'a: "open', '- loose']) + assert.notEqual(parse(`---\n${bad}\n---`).stop, null, bad); + assert.deepEqual(parse('no front matter'), { frontmatter: null, raw: null, bodyLine: 1, stop: null }); +}); + +test('R4: a link to no file, to more than one, or into another repo is reported and not followed', async t => { + const r = repo(t, { ...BASE, + 'Book One/Research/Notes.md': 'a\n', 'Book One/Places/Notes.md': 'b\n', + 'Book One/Chapters/Links.md': ['[[Nobody]]', '[[Notes]]', '[[bean-brain/Journal/Today]]', '[[Tomas]]', ''].join('\n') }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Chapters/Links.md').map(p => [p.code, p.line]), [['R4', 1], ['R4', 2], ['R4', 3]]); + assert.deepEqual(file(result, 'Book One/Chapters/Links.md').links.map(l => l.path), ['Book One/Characters/Tomas.md']); +}); + +test('R5: a canon file that refers into the playground is reported, and the link still goes by path', async t => { + const r = repo(t, { ...BASE, 'Book One/Places/Dock.md': 'See [[What If]].\n' }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.deepEqual(codes(result, 'Book One/Places/Dock.md'), ['R5']); + assert.deepEqual(file(result, 'Book One/Places/Dock.md').links.map(l => l.path), ['Book One/Playground/What If.md']); + // The playground file links to canon: no problem, R5 is only about canon. + assert.deepEqual(codes(result, 'Book One/Playground/What If.md'), []); +}); + +test('R6: a canon file that refers into another universe is reported; a hybrid may refer into its sources', async t => { + const r = repo(t, { ...BASE, 'Book One/Places/Far.md': 'Like [[Second World/Chapters/01]].\n', + 'Crossover/Chapters/A.md': 'From [[Mara Vell]] and [[Second World/Chapters/01]].\n' }); + const hybrid = mapping(e => { e.universes.cross = { kind: 'hybrid', name: 'Cross', sources: ['harbor', 'second'], books: ['Crossover'] }; return e; }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: hybrid }); + assert.deepEqual(codes(result, 'Book One/Places/Far.md'), ['R6']); + assert.deepEqual(file(result, 'Book One/Places/Far.md').links.map(l => l.path), ['Second World/Chapters/01.md']); + assert.deepEqual(codes(result, 'Crossover/Chapters/A.md'), []); +}); + +test('R7: two paths that differ only in case are both left out', async t => { + const r = repo(t, BASE); + writeFileSync(join(r.root, 'twin'), 'twin\n'); + const blob = git(r.repo, 'hash-object', '-w', join(r.root, 'twin')); + git(r.repo, 'update-index', '--add', '--cacheinfo', `100644,${blob},Book One/Characters/tomas.md`); + git(r.repo, 'commit', '-qm', 'case twin'); + const result = await readLibrary({ repo: r.repo, commit: git(r.repo, 'rev-parse', 'HEAD'), mapping: mapping() }); + for (const path of ['Book One/Characters/Tomas.md', 'Book One/Characters/tomas.md']) { + assert.deepEqual(codes(result, path), ['R7'], path); + assert.equal(file(result, path), undefined); + } +}); + +test('the reader reads the commit, never the working tree, and refuses a bad commit id', async t => { + const r = repo(t); + writeFileSync(join(r.repo, 'Book One/Places/Harbor Town.md'), 'unsaved [[Nobody]]\n'); + writeFileSync(join(r.repo, 'Book One/Places/Untracked.md'), 'x\n'); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.equal(file(result, 'Book One/Places/Harbor Town.md').lines[0][1], '---'); + assert.equal(file(result, 'Book One/Places/Untracked.md'), undefined); + for (const commit of ['HEAD', r.commit.slice(0, 12), '0'.repeat(40), git(r.repo, 'rev-parse', 'HEAD^{tree}')]) + await assert.rejects(readLibrary({ repo: r.repo, commit, mapping: mapping() }), e => e instanceof LibraryError, commit); + await assert.rejects(readLibrary({ repo: r.repo, commit: r.commit, mapping: {} }), /library-mapping-invalid/); +}); + +test('a bare mirror reads the same as the clone', async t => { + const r = repo(t); + const bare = join(r.root, 'mirror.git'); + git(r.root, 'clone', '-q', '--bare', r.repo, bare); + const a = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + const b = await readLibrary({ repo: bare, commit: r.commit, mapping: mapping() }); + assert.deepEqual(b, a); +}); diff --git a/packages/library/tests/render.test.mjs b/packages/library/tests/render.test.mjs new file mode 100644 index 000000000..b07e26d25 --- /dev/null +++ b/packages/library/tests/render.test.mjs @@ -0,0 +1,80 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { renderItems, UNREADABLE } from '../src/index.mjs'; + +const A = 'a'.repeat(40), B = 'b'.repeat(40); +const cite = (path, lines, quote, commit = A) => ({ path, commit, lines, quote }); +const options = { + scopeOf: path => path.includes('/Playground/') ? 'playground' : path.endsWith('.md') ? 'canon' : undefined, + changed: ({ path, commit }) => path === 'Book One/Characters/Mara Vell.md' && commit === A, +}; +const render = items => renderItems(JSON.stringify({ items }), options); + +// One snapshot per item kind: the plain text both channels show. +const SNAPSHOTS = { + fact: [{ kind: 'fact', text: 'Tomas is the ferryman.', cites: [cite('Book One/Characters/Tomas.md', [3, 3], 'the ferryman', B)] }], + finding: [{ kind: 'finding', text: 'Eye colour differs.', cites: [cite('Book One/Chapters/Chapter 01.md', [41, 41], 'her grey eyes'), + cite('Book One/Characters/Mara Vell.md', [9, 10], 'green eyes')] }], + correction: [{ kind: 'correction', text: 'Chapter 1 says grey.', from: 'green eyes', to: 'grey eyes', + cites: [cite('Book One/Characters/Mara Vell.md', [9, 9], 'green eyes', B)] }], + question: [{ kind: 'question', text: 'Is the harbor north of the town?', cites: [cite('Book One/Places/Harbor Town.md', [2, 4], 'the harbor', B)] }], + idea: [{ kind: 'idea', text: 'A storm could close the harbor.', cites: [cite('Book One/Playground/What If.md', [5, 5], 'a storm', B), + { url: 'https://example.invalid/tides', quote: 'spring tides' }] }], +}; +const EXPECTED = { + fact: '1. Fact: Tomas is the ferryman.\n [canon] Book One/Characters/Tomas.md:3 @bbbbbbb "the ferryman"', + finding: '1. Finding: Eye colour differs.\n [canon] Book One/Chapters/Chapter 01.md:41 @aaaaaaa "her grey eyes"\n' + + ' [canon] Book One/Characters/Mara Vell.md:9-10 @aaaaaaa "green eyes" (changed since)', + correction: '1. Correction: Chapter 1 says grey.\n From: "green eyes"\n To: "grey eyes"\n' + + ' [canon] Book One/Characters/Mara Vell.md:9 @bbbbbbb "green eyes"', + question: '1. Question: Is the harbor north of the town?\n [canon] Book One/Places/Harbor Town.md:2-4 @bbbbbbb "the harbor"', + idea: '1. Idea: A storm could close the harbor.\n [playground] Book One/Playground/What If.md:5 @bbbbbbb "a storm"\n' + + ' [web] https://example.invalid/tides "spring tides"', +}; + +for (const kind of Object.keys(SNAPSHOTS)) { + test(`renderItems snapshot: ${kind}`, () => { + const out = render(SNAPSHOTS[kind]); + assert.equal(out.kind, 'items'); + assert.equal(out.text, EXPECTED[kind]); + assert.equal(out.items[0].kind, kind); + }); +} + +test('renderItems numbers several items and takes a parsed body as well as a string', () => { + const items = [...SNAPSHOTS.fact, ...SNAPSHOTS.question]; + const out = renderItems({ items }, options); + assert.equal(out.text, `${EXPECTED.fact}\n${EXPECTED.question.replace(/^1\./, '2.')}`); + assert.deepEqual(out.items[1].cites[0], { badge: 'canon', path: 'Book One/Places/Harbor Town.md', commit: B, lines: [2, 4], + ref: 'Book One/Places/Harbor Town.md:2-4 @bbbbbbb', quote: 'the harbor', changed: false }); +}); + +test('without options a path cite has no badge and is never marked changed', () => { + assert.equal(renderItems({ items: SNAPSHOTS.finding }).text, + '1. Finding: Eye colour differs.\n Book One/Chapters/Chapter 01.md:41 @aaaaaaa "her grey eyes"\n' + + ' Book One/Characters/Mara Vell.md:9-10 @aaaaaaa "green eyes"'); +}); + +const NOTICE_SNAPSHOTS = { 'no-cited-answer': 'No cited answer', 'no-answer': 'No answer', 'turn-failed': 'The turn failed' }; +for (const [notice, text] of Object.entries(NOTICE_SNAPSHOTS)) { + test(`renderItems snapshot: notice ${notice}`, () => { + assert.deepEqual(renderItems(JSON.stringify({ items: [], notice }), options), { kind: 'notice', notice, text }); + }); +} + +test('a body without the reply shape shows a fixed line and none of its text', () => { + const secret = 'free prose the guard would refuse'; + const bodies = [secret, JSON.stringify(secret), '{"items": [], "notice": "other"}', JSON.stringify({ items: [], notice: 'no-answer', extra: secret }), + JSON.stringify({ items: [] }), JSON.stringify({ items: [{ kind: 'story', text: secret, cites: [] }] }), + JSON.stringify({ items: [{ kind: 'fact', text: secret }] }), JSON.stringify({ items: SNAPSHOTS.fact, prose: secret }), + JSON.stringify({ items: [{ kind: 'correction', text: secret, cites: [] }] }), + JSON.stringify({ items: [{ kind: 'fact', text: secret, cites: [cite('a.md', [2, 1], 'q')] }] }), + JSON.stringify({ items: [{ kind: 'fact', text: secret, cites: [{ path: 'a.md', commit: 'abc', lines: [1, 1], quote: 'q' }] }] }), + JSON.stringify({ items: [{ kind: 'fact', text: secret, cites: [{ url: 'https://x.invalid', path: 'a.md', quote: 'q' }] }] }), + null, 42, '[]']; + for (const body of bodies) { + const out = renderItems(body, options); + assert.deepEqual(out, { kind: 'unreadable', text: UNREADABLE }, String(body)); + assert.ok(!JSON.stringify(out).includes(secret)); + } +}); diff --git a/packages/library/tests/survey.test.mjs b/packages/library/tests/survey.test.mjs new file mode 100644 index 000000000..48a120813 --- /dev/null +++ b/packages/library/tests/survey.test.mjs @@ -0,0 +1,82 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { surveyMain } from '../src/index.mjs'; +import { BASE, FIXTURES, MARKER, repo } from './helpers.mjs'; + +const ROOT = fileURLToPath(new URL('../../..', import.meta.url)); +async function run(args) { + let stdout = '', stderr = ''; + const code = await surveyMain(args, { stdout: { write: s => { stdout += s; } }, stderr: { write: s => { stderr += s; } } }); + return { code, stdout, stderr }; +} +// Strings from the fixture that are text, values, file names or link targets: none may be printed. +const NEVER = [MARKER, 'Grey eyes', 'The Ferryman', 'Mara Vell', 'Tomas', 'Harbor Town', 'What If', 'Inbox Note', 'Chapter 01', + 'Sketch', 'Board', 'Not A Link', 'the harbor', 'Docks', 'character']; + +test('the survey prints folder names, counts, key sets and link-form counts, and never text', async t => { + const r = repo(t, { ...BASE, 'Book One/Places/Odd.md': ['---', `"${MARKER} key": 1`, '---', ''].join('\n') }); + const out = await run(['survey', r.repo]); + assert.equal(out.code, 0, out.stderr); + assert.equal(out.stdout, [ + `commit ${r.commit}`, + 'entries 10', + 'folders (direct children: md canvas base other)', + ' (root) 1 0 0 0', + ' "Book One" 0 1 0 0', + ' "Book One/Chapters" 1 0 0 0', + ' "Book One/Characters" 2 0 0 0', + ' "Book One/Places" 2 0 0 0', + ' "Book One/Playground" 1 1 0 0', + ' "Second World/Chapters" 1 0 0 0', + 'markdown 8 files, 8 read, 3 without front matter, 0 with front matter not fully parsed', + 'front matter key sets', + ' 1 [(other key)]', + ' 1 [appears_in, name, relationships, type]', + ' 1 [appears_in, name, type]', + ' 1 [name]', + ' 1 [type]', + 'body links plain 3, label 1, heading 1, embed 0, self 0, markdown 1', + 'field appears_in wikilink 2, name 0', + 'field relationships wikilink 1, name 1', + ''].join('\n')); + for (const s of NEVER) assert.ok(!out.stdout.includes(s), s); +}); + +test('with --mapping the survey adds scope, resolver step and problem counts', async t => { + const r = repo(t); + const out = await run(['survey', r.repo, '--mapping', join(FIXTURES, 'mapping.json')]); + assert.equal(out.code, 0, out.stderr); + const tail = out.stdout.trimEnd().split('\n').slice(-3); + assert.deepEqual(tail, [ + 'scopes canon 5, playground 1, unmapped 3; indexed 6', + 'resolved root 2, vault-prefix 1, relative 2, shortest 3, unresolved 0', + 'problems R1 0, R2 3, R3 0, R4 0, R5 0, R6 0, R7 0']); + for (const s of NEVER) assert.ok(!out.stdout.includes(s), s); +}); + +test('survey exit codes: usage 4, bad mapping 3, unreadable repo 2', async t => { + const r = repo(t); + for (const args of [[], ['scan', r.repo], ['survey'], ['survey', r.repo, '--mapping'], ['survey', r.repo, '--other'], ['survey', r.repo, r.repo]]) + assert.equal((await run(args)).code, 4, args.join(' ')); + const bad = join(r.root, 'bad.json'); + writeFileSync(bad, JSON.stringify({ kind: 'library' })); + const refused = await run(['survey', r.repo, '--mapping', bad]); + assert.equal(refused.code, 3); + assert.match(refused.stderr, /library-mapping-invalid/); + writeFileSync(bad, 'not json'); + assert.equal((await run(['survey', r.repo, '--mapping', bad])).code, 3); + assert.equal((await run(['survey', join(r.root, 'missing')])).code, 2); + assert.equal((await run(['survey', r.root])).code, 2, 'a directory that is not a repo'); +}); + +test('scripts/mosaic library survey reaches the survey', t => { + const r = repo(t); + const out = spawnSync(join(ROOT, 'scripts/mosaic'), ['library', 'survey', r.repo], { encoding: 'utf8', timeout: 30000 }); + assert.equal(out.status, 0, out.stderr); + assert.match(out.stdout, new RegExp(`^commit ${r.commit}\n`)); + assert.ok(!out.stdout.includes(MARKER)); +}); diff --git a/scripts/mosaic b/scripts/mosaic index 58d184b94..2f68cdc92 100755 --- a/scripts/mosaic +++ b/scripts/mosaic @@ -34,6 +34,12 @@ if [ "${1:-}" = business ]; then shift exec node "$REPO/packages/business/src/cli.mjs" "$@" fi +# `mosaic library survey`: names and counts from a writing repo, never its text +# (packages/library/README.md). Other library verbs are bus verbs. +if [ "${1:-}" = library ] && [ "${2:-}" = survey ]; then + shift + exec node "$REPO/packages/library/src/survey.mjs" "$@" +fi case "${1:-}" in usage|project|limits|inbox|decide|tasks|agents|trail|talk|stop|launches|bus|plugins|credentials|name|names|follow|threads|thread) exec node "$REPO/packages/cli/src/cli.mjs" "$@" ;; esac diff --git a/scripts/stack-copy.sh b/scripts/stack-copy.sh index 31f9ea060..31e3258f3 100755 --- a/scripts/stack-copy.sh +++ b/scripts/stack-copy.sh @@ -33,7 +33,7 @@ COPY_PATHS=( packages/conversation/src packages/discord/src packages/discord/bin packages/discord/extension packages/harness/src packages/seat/src packages/tasks/src packages/tasks/fields.json packages/tasks/requirements.json packages/webui/src packages/webui/README.md packages/cli/README.md - packages/queue/src + packages/queue/src packages/library/src # Whole directory: every in-repo plug-in module ships with the copy (row 131). packages/plugins )