stack/packages/library/src/query.mjs

// The lookup queries of design 3 (C2-05a): find, get, search, manuscript and universeInfo. Each takes a
// source {repo, commit, mapping} and an argument object, and returns a reply that fits QUERY_REPLY_CAP.
// Universe isolation is here, in sees(): a file reaches a reply only when sees() allows it. Scope comes
// from the path (D2). Unmapped files are not in the index, so they never appear. Lists page by position
// and never cut an item. A search line too long to fit alone comes back as a marked hit with no excerpt;
// a find item, a file's front matter or one of its lines that cannot fit alone is refused.
import { raise } from './problems.mjs';
import { TYPES } from './mapping.mjs';
import { indexFor } from './index-build.mjs';
import { listTree } from './reader.mjs';
import { excluded } from './scope.mjs';

// The same cap as packages/harness/src/tools.mjs REPLY_CAP. The query tests assert the two stay equal.
export const QUERY_REPLY_CAP = 40_000;
export const LOOKUP_LIMIT = 50;
export const MANUSCRIPT_LINES = 400;
const SCOPES = Object.freeze(['canon', 'playground', 'supporting']);
const MANUSCRIPT_SCOPES = Object.freeze(['canon', 'playground']);
const SCOPE_RANK = Object.freeze({ supporting: 0, playground: 1, canon: 2 });

const bytes = value => JSON.stringify(value).length;
const invalid = message => raise('query-argument-invalid', message);

function textArg(value, name) {
  if (typeof value !== 'string') invalid(`${name} must be text`);
  return value;
}
function scopeArg(value, allowed = SCOPES) {
  if (value === undefined) return 'canon';
  if (!allowed.includes(value)) invalid(`scope must be one of ${allowed.join(", ")}`);
  return value;
}
function positionArg(value, name, fallback) {
  if (value === undefined) return fallback;
  if (!Number.isSafeInteger(value) || value < 1) invalid(`${name} must be a line number of 1 or more`);
  return value;
}
function offsetArg(value) {
  if (value === undefined) return 0;
  if (!Number.isSafeInteger(value) || value < 0) invalid('offset must be a whole number of 0 or more');
  return value;
}
function universeArg(mapping, value) {
  if (typeof value !== 'string' || !value) invalid('universe is required');
  if (!Object.hasOwn(mapping.universes, value)) raise('query-universe-unknown', `this library has no universe ${JSON.stringify(value)}`);
  return value;
}

// Whether universe U may see a file. Its own files, in either scope. A hybrid also sees the canon of
// its declared sources, minus what its exclude constraints cover (scope.mjs excluded(), C2-15). A hybrid
// never sees a source's playground. Pin constraints hide nothing; the reader reports them as R8.
export function sees(mapping, universe, file) {
  if (file.universe === universe) return true;
  const entry = mapping.universes[universe];
  if (entry.kind !== 'hybrid' || !entry.sources.includes(file.universe) || file.scope !== 'canon') return false;
  return !excluded(mapping, universe, file.path);
}

// The files one universe may see in one scope, with optional book and type filters, in path order.
async function visible(source, universe, { scope, book, type }) {
  const index = await indexFor(source);
  const files = index.paths
    .map(path => index.files.get(path))
    .filter(file => file.scope === scope && sees(index.mapping, universe, file)
      && (book === undefined || file.book === book) && (type === undefined || file.type === type));
  return { index, files };
}

// A reply whose size is over the cap is refused, never cut.
function checked(reply) {
  if (bytes(reply) > QUERY_REPLY_CAP) raise('query-item-too-large', `the reply is over the ${QUERY_REPLY_CAP}-character cap`);
  return reply;
}

// Picks items from `offset`, at most `limit`, while the reply still fits. build(picked, next) returns
// the reply; `next` is the position of the first item not picked, or null when none is left. An item
// that cannot fit even alone is refused: skipping it would hide it, and cutting it would break it.
function fitPage(items, { offset = 0, limit = LOOKUP_LIMIT, build }) {
  const picked = [];
  let at = offset;
  while (at < items.length && picked.length < limit) {
    // The size check uses items.length for `next`, the widest number a cursor can take, so a page never runs over.
    if (bytes(build([...picked, items[at]], items.length)) > QUERY_REPLY_CAP) {
      if (!picked.length) raise('query-item-too-large', `the item at position ${at} alone is over the ${QUERY_REPLY_CAP}-character cap`);
      break;
    }
    picked.push(items[at]);
    at += 1;
  }
  return checked(build(picked, at < items.length ? at : null));
}

// find: the files whose name or file name holds `query`, case-insensitive. Up to 50 per page.
export async function find(source, { universe: name, query, type, book, scope, offset } = {}) {
  const universe = universeArg(source.mapping, name);
  const needle = textArg(query, 'query').toLowerCase();
  if (type !== undefined && !TYPES.includes(type)) invalid(`type must be one of ${TYPES.join(', ')}`);
  const { index, files } = await visible(source, universe, {
    scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'), type,
  });
  const hits = files.filter(file => file.name.toLowerCase().includes(needle) || file.stem.toLowerCase().includes(needle));
  return fitPage(hits, {
    offset: offsetArg(offset),
    build: (picked, next) => ({
      commit: index.commit,
      results: picked.map(file => ({
        path: file.path, type: file.type, name: file.name, appears_in: file.frontmatter?.appears_in ?? null,
        scope: file.scope, universe: file.universe, book: file.book,
      })),
      next,
    }),
  });
}

// get: one file in the requested scope (canon default), its served-tree blob, front matter,
// lines and visible links both ways. Links keep all visible scopes, ordered supporting/playground/canon.
// Returns null when the file is missing or not visible, so a caller cannot tell the two apart. Lines page
// by `from` (reply `next`), links page together by `links_from` (reply `links_next`); `links_out` come
// before `links_in`. Every page carries the front matter. Links take at most half the cap, so a hub note
// with many backlinks stays readable and still leaves room for lines. A page with lines left carries at
// least one line, so links shrink when a line needs the room; read on with `from: next` and
// `links_from: links_next`. A `from` past the last line returns links alone. Only front matter, or a
// line that cannot fit with no links beside it, is refused.
export async function get(source, { universe: name, path, scope, from, links_from: linksFrom } = {}) {
  const universe = universeArg(source.mapping, name);
  const requestedScope = scopeArg(scope);
  const startLine = positionArg(from, 'from', 1);
  const linkStart = offsetArg(linksFrom);
  if (typeof path !== 'string') invalid('path must be text');
  const index = await indexFor(source);
  const file = index.files.get(path);
  if (!file || file.scope !== requestedScope || !sees(index.mapping, universe, file)) return null;
  const reachable = links => links.filter(link => {
    const other = index.files.get(link.path);
    return other !== undefined && sees(index.mapping, universe, other);
  }).sort((a, b) => SCOPE_RANK[index.files.get(a.path).scope] - SCOPE_RANK[index.files.get(b.path).scope]);
  const outs = reachable(index.linksOut.get(file.path) ?? []);
  const ins = reachable(index.linksIn.get(file.path) ?? []);
  const links = [...outs.map(link => ({ out: true, link })), ...ins.map(link => ({ out: false, link }))];
  // The reader/index does not keep tree oids. Resolve only in this immutable served
  // snapshot, never HEAD; listTree already checks/bounds the tree and its object IDs.
  const entry = (await listTree(source.repo, index.commit)).find(entry => entry.path === file.path);
  if (!entry || entry.mode !== '100644' || entry.type !== 'blob') raise('library-commit', 'indexed file is absent from the served tree');
  const blob = entry.oid;
  const lines = file.lines.slice(startLine - 1);
  const linkRest = Math.max(0, links.length - linkStart);
  const page = (nLines, nLinks) => {
    const taken = links.slice(linkStart, linkStart + nLinks);
    return {
      path: file.path, commit: index.commit, blob, scope: file.scope, frontmatter: file.frontmatter,
      lines: lines.slice(0, nLines),
      links_out: taken.filter(item => item.out).map(item => item.link),
      links_in: taken.filter(item => !item.out).map(item => item.link),
      next: nLines < lines.length ? startLine + nLines : null,
      links_next: nLinks < linkRest ? linkStart + nLinks : null,
    };
  };
  // How many leading items fit in `room`, by each item's own size.
  const count = (items, room) => {
    let used = 0, n = 0;
    while (n < items.length && used + bytes(items[n]) + 1 <= room) used += bytes(items[n++]) + 1;
    return n;
  };
  const base = bytes(page(0, 0));
  if (base > QUERY_REPLY_CAP) raise('query-item-too-large', `the front matter of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`);
  if (lines.length && bytes(page(1, 0)) > QUERY_REPLY_CAP) {
    raise('query-item-too-large', `line ${lines[0][0]} of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`);
  }
  const rest = links.slice(linkStart);
  let nLinks = count(rest, (lines.length ? Math.floor(QUERY_REPLY_CAP / 2) : QUERY_REPLY_CAP) - base);
  if (!nLinks && rest.length) nLinks = count(rest, QUERY_REPLY_CAP - base);
  if (!nLinks && rest.length) raise('query-item-too-large', `a link of ${file.path} alone is over the ${QUERY_REPLY_CAP}-character cap`);
  while (lines.length && nLinks > 0 && bytes(page(1, nLinks)) > QUERY_REPLY_CAP) nLinks = Math.floor(nLinks / 2);
  let nLines = Math.max(lines.length ? 1 : 0, count(lines, QUERY_REPLY_CAP - bytes(page(0, nLinks))));
  while (nLines > 1 && bytes(page(nLines, nLinks)) > QUERY_REPLY_CAP) nLines -= 1;
  return checked(page(nLines, nLinks));
}

// search: lines whose text holds `text`, case-insensitive. The excerpt is the whole line. Up to 50 per page.
// Each hit names its source universe, as find does: a hybrid's hits come from its own files and its sources' canon.
export async function search(source, { universe: name, text, book, scope, offset } = {}) {
  const universe = universeArg(source.mapping, name);
  const needle = textArg(text, 'text').toLowerCase();
  if (!needle) invalid('text must not be empty');
  const { index, files } = await visible(source, universe, {
    scope: scopeArg(scope), book: book === undefined ? undefined : textArg(book, 'book'),
  });
  const hits = [];
  for (const file of files) {
    for (const [line, body] of file.lines) {
      if (!body.toLowerCase().includes(needle)) continue;
      const hit = { path: file.path, line, excerpt: body, scope: file.scope, universe: file.universe };
      // The excerpt is the whole line. A line too long to fit alone is marked, not cut and not refused.
      hits.push(bytes({ commit: index.commit, results: [hit], next: Number.MAX_SAFE_INTEGER }) > QUERY_REPLY_CAP
        ? { path: file.path, line, excerpt: null, excerpt_too_long: true, scope: file.scope, universe: file.universe } : hit);
    }
  }
  return fitPage(hits, {
    offset: offsetArg(offset),
    build: (picked, next) => ({ commit: index.commit, results: picked, next }),
  });
}

// manuscript: one chapter file of a book, by its file name, up to 400 lines per call from `from` to `to`.
// Without a chapter it returns the book's first chapter file in path order. Every reply carries
// `next_chapter`, the next chapter file's name in path order or null. A name that matches two visible
// files is refused. Files outside every book (book null) are never chapters of a book.
export async function manuscript(source, { universe: name, book, scope, chapter, from, to } = {}) {
  const universe = universeArg(source.mapping, name);
  const bookName = textArg(book, 'book');
  const chapterName = chapter === undefined ? undefined : textArg(chapter, 'chapter');
  const start = positionArg(from, 'from', 1);
  const end = positionArg(to, 'to', Infinity);
  if (end < start) invalid('to must not be before from');
  const { index, files } = await visible(source, universe, { scope: scopeArg(scope, MANUSCRIPT_SCOPES), book: bookName, type: 'manuscript' });
  let at = 0;
  if (chapterName !== undefined) {
    const matches = files.filter(file => file.stem === chapterName);
    if (matches.length > 1) raise('query-chapter-ambiguous', `${matches.length} visible files are named ${JSON.stringify(chapterName)}`);
    if (!matches.length) return null;
    at = files.indexOf(matches[0]);
  }
  const file = files[at];
  if (!file) return null;
  const nextChapter = files[at + 1]?.stem ?? null;
  return fitPage(file.lines.slice(start - 1, end), {
    limit: MANUSCRIPT_LINES,
    build: (picked, next) => ({
      path: file.path, commit: index.commit, scope: file.scope, lines: picked,
      next: next === null ? null : start + next, next_chapter: nextChapter,
    }),
  });
}

// universeInfo: a universe's kind, name, books, playground prefixes, type folders and manuscript folders.
// A hybrid adds its sources, constraints and notes. A draft has no books and no canon: its kind, name
// and prefixes are the whole answer beside the mapping's types. It reads only the checked mapping and the
// served commit.
export async function universeInfo(source, { universe: name } = {}) {
  const universe = universeArg(source.mapping, name);
  const entry = source.mapping.universes[universe];
  const hybrid = entry.kind === 'hybrid';
  return checked({
    commit: source.commit, universe, name: entry.name, kind: entry.kind ?? 'universe',
    books: entry.books, playground: entry.playground, types: source.mapping.types, manuscript: source.mapping.manuscript,
    ...(hybrid ? {
      sources: entry.sources, constraints: entry.constraints,
      ...(entry.notes === undefined ? {} : { notes: entry.notes }),
    } : {}),
  });
}