// The lookup queries of design 3 (C2-05a): find, get, search, manuscript and universeInfo. Each takes a // source {repo, commit, mapping} and an argument object, and returns a reply that fits QUERY_REPLY_CAP. // Universe isolation is here, in sees(): a file reaches a reply only when sees() allows it. Scope comes // from the path (D2). Unmapped files are not in the index, so they never appear. Lists page by position // and never cut an item. A search line too long to fit alone comes back as a marked hit with no excerpt; // a find item, a file's front matter or one of its lines that cannot fit alone is refused. import { raise } from './problems.mjs'; import { TYPES } from './mapping.mjs'; import { indexFor } from './index-build.mjs'; import { listTree } from './reader.mjs'; import { excluded } from './scope.mjs'; // The same cap as packages/harness/src/tools.mjs REPLY_CAP. The query tests assert the two stay equal. export const QUERY_REPLY_CAP = 40_000; export const LOOKUP_LIMIT = 50; export const MANUSCRIPT_LINES = 400; const SCOPES = Object.freeze(['canon', 'playground', 'supporting']); const MANUSCRIPT_SCOPES = Object.freeze(['canon', 'playground']); const SCOPE_RANK = Object.freeze({ supporting: 0, playground: 1, canon: 2 }); const bytes = value => JSON.stringify(value).length; const invalid = message => raise('query-argument-invalid', message); function textArg(value, name) { if (typeof value !== 'string') invalid(`${name} must be text`); return value; } function scopeArg(value, allowed = SCOPES) { if (value === undefined) return 'canon'; if (!allowed.includes(value)) invalid(`scope must be one of ${allowed.join(", ")}`); return value; } function positionArg(value, name, fallback) { if (value === undefined) return fallback; if (!Number.isSafeInteger(value) || value < 1) invalid(`${name} must be a line number of 1 or more`); return value; } function offsetArg(value) { if (value === undefined) return 0; if (!Number.isSafeInteger(value) || value < 0) invalid('offset must be a whole number of 0 or more'); return value; } function universeArg(mapping, value) { if (typeof value !== 'string' || !value) invalid('universe is required'); if (!Object.hasOwn(mapping.universes, value)) raise('query-universe-unknown', `this library has no universe ${JSON.stringify(value)}`); return value; } // Whether universe U may see a file. Its own files, in either scope. A hybrid also sees the canon of // its declared sources, minus what its exclude constraints cover (scope.mjs excluded(), C2-15). A hybrid // never sees a source's playground. Pin constraints hide nothing; the reader reports them as R8. export function sees(mapping, universe, file) { if (file.universe === universe) return true; const entry = mapping.universes[universe]; if (entry.kind !== 'hybrid' || !entry.sources.includes(file.universe) || file.scope !== 'canon') return false; return !excluded(mapping, universe, file.path); } // The files one universe may see in one scope, with optional book and type filters, in path order. async function visible(source, universe, { scope, book, type }) { const index = await indexFor(source); const files = index.paths .map(path => index.files.get(path)) .filter(file => file.scope === scope && sees(index.mapping, universe, file) && (book === undefined || file.book === book) && (type === undefined || file.type === type)); return { index, files }; } // A reply whose size is over the cap is refused, never cut. function checked(reply) { if (bytes(reply) > QUERY_REPLY_CAP) raise('query-item-too-large', `the reply is over the ${QUERY_REPLY_CAP}-character cap`); return reply; } // Picks items from `offset`, at most `limit`, while the reply still fits. build(picked, next) returns // the reply; `next` is the position of the first item not picked, or null when none is left. An item // that cannot fit even alone is refused: skipping it would hide it, and cutting it would break it. function fitPage(items, { offset = 0, limit = LOOKUP_LIMIT, build }) { const picked = []; let at = offset; while (at < items.length && picked.length < limit) { // The size check uses items.length for `next`, the widest number a cursor can take, so a page never runs over. if (bytes(build([...picked, items[at]], items.length)) > QUERY_REPLY_CAP) { if (!picked.length) raise('query-item-too-large', `the item at position ${at} alone is over the ${QUERY_REPLY_CAP}-character cap`); break; } picked.push(items[at]); at += 1; } return checked(build(picked, at < items.length ? at : null)); } // find: the files whose name or file name holds `query`, case-insensitive. Up to 50 per page. export async function find(source, { universe: name, query, type, book, scope, offset } = {}) { const universe = universeArg(source.mapping, name); const needle = textArg(query, 'query').toLowerCase(); if (type !== undefined && !TYPES.includes(type)) invalid(`type must be one of ${TYPES.join(', ')}`); const { index, files } = await visible(source, universe, { scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'), type, }); const hits = files.filter(file => file.name.toLowerCase().includes(needle) || file.stem.toLowerCase().includes(needle)); return fitPage(hits, { offset: offsetArg(offset), build: (picked, next) => ({ commit: index.commit, results: picked.map(file => ({ path: file.path, type: file.type, name: file.name, appears_in: file.frontmatter?.appears_in ?? null, scope: file.scope, universe: file.universe, book: file.book, })), next, }), }); } // get: one file in the requested scope (canon default), its served-tree blob, front matter, // lines and visible links both ways. Links keep all visible scopes, ordered supporting/playground/canon. // Returns null when the file is missing or not visible, so a caller cannot tell the two apart. Lines page // by `from` (reply `next`), links page together by `links_from` (reply `links_next`); `links_out` come // before `links_in`. Every page carries the front matter. Links take at most half the cap, so a hub note // with many backlinks stays readable and still leaves room for lines. A page with lines left carries at // least one line, so links shrink when a line needs the room; read on with `from: next` and // `links_from: links_next`. A `from` past the last line returns links alone. Only front matter, or a // line that cannot fit with no links beside it, is refused. export async function get(source, { universe: name, path, scope, from, links_from: linksFrom } = {}) { const universe = universeArg(source.mapping, name); const requestedScope = scopeArg(scope); const startLine = positionArg(from, 'from', 1); const linkStart = offsetArg(linksFrom); if (typeof path !== 'string') invalid('path must be text'); const index = await indexFor(source); const file = index.files.get(path); if (!file || file.scope !== requestedScope || !sees(index.mapping, universe, file)) return null; const reachable = links => links.filter(link => { const other = index.files.get(link.path); return other !== undefined && sees(index.mapping, universe, other); }).sort((a, b) => SCOPE_RANK[index.files.get(a.path).scope] - SCOPE_RANK[index.files.get(b.path).scope]); const outs = reachable(index.linksOut.get(file.path) ?? []); const ins = reachable(index.linksIn.get(file.path) ?? []); const links = [...outs.map(link => ({ out: true, link })), ...ins.map(link => ({ out: false, link }))]; // The reader/index does not keep tree oids. Resolve only in this immutable served // snapshot, never HEAD; listTree already checks/bounds the tree and its object IDs. const entry = (await listTree(source.repo, index.commit)).find(entry => entry.path === file.path); if (!entry || entry.mode !== '100644' || entry.type !== 'blob') raise('library-commit', 'indexed file is absent from the served tree'); const blob = entry.oid; const lines = file.lines.slice(startLine - 1); const linkRest = Math.max(0, links.length - linkStart); const page = (nLines, nLinks) => { const taken = links.slice(linkStart, linkStart + nLinks); return { path: file.path, commit: index.commit, blob, scope: file.scope, frontmatter: file.frontmatter, lines: lines.slice(0, nLines), links_out: taken.filter(item => item.out).map(item => item.link), links_in: taken.filter(item => !item.out).map(item => item.link), next: nLines < lines.length ? startLine + nLines : null, links_next: nLinks < linkRest ? linkStart + nLinks : null, }; }; // How many leading items fit in `room`, by each item's own size. const count = (items, room) => { let used = 0, n = 0; while (n < items.length && used + bytes(items[n]) + 1 <= room) used += bytes(items[n++]) + 1; return n; }; const base = bytes(page(0, 0)); if (base > QUERY_REPLY_CAP) raise('query-item-too-large', `the front matter of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); if (lines.length && bytes(page(1, 0)) > QUERY_REPLY_CAP) { raise('query-item-too-large', `line ${lines[0][0]} of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); } const rest = links.slice(linkStart); let nLinks = count(rest, (lines.length ? Math.floor(QUERY_REPLY_CAP / 2) : QUERY_REPLY_CAP) - base); if (!nLinks && rest.length) nLinks = count(rest, QUERY_REPLY_CAP - base); if (!nLinks && rest.length) raise('query-item-too-large', `a link of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`); while (lines.length && nLinks > 0 && bytes(page(1, nLinks)) > QUERY_REPLY_CAP) nLinks = Math.floor(nLinks / 2); let nLines = Math.max(lines.length ? 1 : 0, count(lines, QUERY_REPLY_CAP - bytes(page(0, nLinks)))); while (nLines > 1 && bytes(page(nLines, nLinks)) > QUERY_REPLY_CAP) nLines -= 1; return checked(page(nLines, nLinks)); } // search: lines whose text holds `text`, case-insensitive. The excerpt is the whole line. Up to 50 per page. // Each hit names its source universe, as find does: a hybrid's hits come from its own files and its sources' canon. export async function search(source, { universe: name, text, book, scope, offset } = {}) { const universe = universeArg(source.mapping, name); const needle = textArg(text, 'text').toLowerCase(); if (!needle) invalid('text must not be empty'); const { index, files } = await visible(source, universe, { scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'), }); const hits = []; for (const file of files) { for (const [line, body] of file.lines) { if (!body.toLowerCase().includes(needle)) continue; const hit = { path: file.path, line, excerpt: body, scope: file.scope, universe: file.universe }; // The excerpt is the whole line. A line too long to fit alone is marked, not cut and not refused. hits.push(bytes({ commit: index.commit, results: [hit], next: Number.MAX_SAFE_INTEGER }) > QUERY_REPLY_CAP ? { path: file.path, line, excerpt: null, excerpt_too_long: true, scope: file.scope, universe: file.universe } : hit); } } return fitPage(hits, { offset: offsetArg(offset), build: (picked, next) => ({ commit: index.commit, results: picked, next }), }); } // manuscript: one chapter file of a book, by its file name, up to 400 lines per call from `from` to `to`. // Without a chapter it returns the book's first chapter file in path order. Every reply carries // `next_chapter`, the next chapter file's name in path order or null. A name that matches two visible // files is refused. Files outside every book (book null) are never chapters of a book. export async function manuscript(source, { universe: name, book, scope, chapter, from, to } = {}) { const universe = universeArg(source.mapping, name); const bookName = textArg(book, 'book'); const chapterName = chapter === undefined ? undefined : textArg(chapter, 'chapter'); const start = positionArg(from, 'from', 1); const end = positionArg(to, 'to', Infinity); if (end < start) invalid('to must not be before from'); const { index, files } = await visible(source, universe, { scope: scopeArg(scope, MANUSCRIPT_SCOPES), book: bookName, type: 'manuscript' }); let at = 0; if (chapterName !== undefined) { const matches = files.filter(file => file.stem === chapterName); if (matches.length > 1) raise('query-chapter-ambiguous', `${matches.length} visible files are named ${JSON.stringify(chapterName)}`); if (!matches.length) return null; at = files.indexOf(matches[0]); } const file = files[at]; if (!file) return null; const nextChapter = files[at + 1]?.stem ?? null; return fitPage(file.lines.slice(start - 1, end), { limit: MANUSCRIPT_LINES, build: (picked, next) => ({ path: file.path, commit: index.commit, scope: file.scope, lines: picked, next: next === null ? null : start + next, next_chapter: nextChapter, }), }); } // universeInfo: a universe's kind, name, books, playground prefixes, type folders and manuscript folders. // A hybrid adds its sources, constraints and notes. A draft has no books and no canon: its kind, name // and prefixes are the whole answer beside the mapping's types. It reads only the checked mapping and the // served commit. export async function universeInfo(source, { universe: name } = {}) { const universe = universeArg(source.mapping, name); const entry = source.mapping.universes[universe]; const hybrid = entry.kind === 'hybrid'; return checked({ commit: source.commit, universe, name: entry.name, kind: entry.kind ?? 'universe', books: entry.books, playground: entry.playground, types: source.mapping.types, manuscript: source.mapping.manuscript, ...(hybrid ? { sources: entry.sources, constraints: entry.constraints, ...(entry.notes === undefined ? {} : { notes: entry.notes }), } : {}), }); }