stack/packages/library/src/reader.mjs

// The commit reader of design 1.3. It reads a commit of her repo, never a working tree: it lists the
// tree with `git ls-tree -r -z` and reads blobs with `cat-file`, through the same bounded git()
// and gitBytes() as packages/project, so hooks, protocols and the environment are fixed the same
// way. Nothing in the repo is checked out or run (D7). Each R-rule is a problem, never a refusal
// of the commit; the affected file or link is left out of the index.
import { git, gitBytes } from '../../project/src/git.mjs';
import { classify } from './scope.mjs';
import { extractLinks, isAttachment, parseWikilink, resolver } from './links.mjs';
import { LibraryError, problem, raise, sortProblems } from './problems.mjs';

export const MAX_BLOB = 1024 * 1024;
const MAX_LISTING = 32 * 1024 * 1024;
const SHA = /^[0-9a-f]{40}$/;
const ENTRY = /^([0-7]{6}) (blob|tree|commit) ([0-9a-f]{40}) +(-|\d+)\t(.+)$/s;
export const LINK_FIELDS = Object.freeze(['appears_in', 'relationships']);
const READERS = 8;

async function local(promise, message) {
  try { return await promise; } catch (e) { raise('library-git', `${message} (${e.code ?? 'error'})`); }
}

export async function headCommit(repo) {
  return local(git(repo, ['rev-parse', '--verify', '--end-of-options', 'HEAD^{commit}']), 'repository has no readable HEAD commit');
}

// Every entry of the commit's tree: { mode, type, oid, size, path }. Sizes come from `-l`, so no
// blob is opened to list it.
export async function listTree(repo, commit) {
  if (typeof commit !== 'string' || !SHA.test(commit)) raise('library-commit', 'expected a full commit id');
  const kind = await local(git(repo, ['cat-file', '-t', commit]), 'commit is not in the repository');
  if (kind !== 'commit') raise('library-commit', 'not a commit');
  const bytes = await local(gitBytes(repo, ['ls-tree', '-r', '-z', '-l', '--full-tree', commit], MAX_LISTING), 'tree listing refused');
  let text;
  try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); }
  catch { raise('library-git', 'tree listing is not UTF-8'); }
  const entries = text.split('\0');
  if (entries.pop() !== '') raise('library-git', 'invalid tree listing');
  return entries.map(entry => {
    const m = ENTRY.exec(entry);
    if (!m) raise('library-git', 'invalid tree listing');
    return { mode: m[1], type: m[2], oid: m[3], size: m[4] === '-' ? null : Number(m[4]), path: m[5] };
  });
}

// R1: only a 100644 blob up to 1 MiB of UTF-8 text without NUL is read.
export const readable = entry => entry.mode === '100644' && entry.type === 'blob'
  && Number.isSafeInteger(entry.size) && entry.size <= MAX_BLOB;

// Text of each readable entry, by path. An entry whose bytes aren't UTF-8 text is missing from the
// map, and the caller reports it under R1.
export async function readTexts(repo, entries) {
  const texts = new Map(), queue = [...entries];
  const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false });
  const worker = async () => {
    for (let entry = queue.shift(); entry; entry = queue.shift()) {
      const bytes = await local(gitBytes(repo, ['cat-file', 'blob', entry.oid], MAX_BLOB), 'blob read refused');
      let value;
      try { value = decoder.decode(bytes); } catch { continue; }
      if (!value.includes('\0')) texts.set(entry.path, value);
    }
  };
  await Promise.all(Array.from({ length: READERS }, worker));
  return texts;
}

const QUOTED = /^"(?:[^"\\]|\\.)*"$|^'(?:[^']|'')*'$/;
const PLAIN_START = /^[^\s\[\]{}&*!|>%@`"'#,]/;
function scalar(raw) {
  const value = raw.trim();
  if (parseWikilink(value)) return { ok: true, value };
  if (QUOTED.test(value)) {
    if (value[0] === "'") return { ok: true, value: value.slice(1, -1).replace(/''/g, "'") };
    try { return { ok: true, value: JSON.parse(value) }; } catch { return { ok: false }; }
  }
  if (value === '' || value === '~' || value === 'null') return { ok: true, value: null };
  if (!PLAIN_START.test(value) || /: |\s#/.test(value) || value.endsWith(':')) return { ok: false };
  if (value === 'true' || value === 'false') return { ok: true, value: value === 'true' };
  if (/^-?(0|[1-9]\d*)(\.\d+)?$/.test(value)) return { ok: true, value: Number(value) };
  return { ok: true, value };
}
function flow(raw) {
  const inner = raw.trim().slice(1, -1), items = [];
  let current = '', quote = null;
  for (const ch of inner) {
    if (quote) { current += ch; if (ch === quote) quote = null; continue; }
    if (ch === '"' || ch === "'") { quote = ch; current += ch; continue; }
    if (ch === '[' || ch === ']' || ch === '{' || ch === '}') return { ok: false };
    if (ch === ',') { items.push(current); current = ''; continue; }
    current += ch;
  }
  if (quote) return { ok: false };
  if (current.trim() || items.length) items.push(current);
  const values = [];
  for (const item of items) { const s = scalar(item); if (!s.ok || !item.trim()) return { ok: false }; values.push(s.value); }
  return { ok: true, value: values };
}
function value(raw) {
  const text = raw.trim();
  if (text.startsWith('[') && !parseWikilink(text)) return text.endsWith(']') ? flow(text) : { ok: false };
  return scalar(text);
}

// Front matter of design 1.3 (R3): scalars, quoted strings, flow lists and block lists of scalars or
// wikilinks. Anything else, a nested map, a block scalar or an anchor for example, leaves the whole
// front matter as raw text with one R3 problem at the line where parsing stopped.
// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. Parsed front matter also
// returns `itemLines`, each block list key's item lines in order.
export function parseFrontMatter(lines) {
  if (lines[0]?.replace(/^/, '').trimEnd() !== '---') return { frontmatter: null, raw: null, bodyLine: 1, stop: null };
  const close = lines.findIndex((text, i) => i > 0 && /^(---|\.\.\.)\s*$/.test(text));
  if (close < 0) return { frontmatter: null, raw: null, bodyLine: 1, stop: 1 };
  const raw = lines.slice(1, close).join('\n'), data = {}, itemLines = {};
  let list = null;
  for (let i = 1; i < close; i++) {
    const text = lines[i], line = i + 1;
    if (!text.trim() || /^\s*#/.test(text)) continue;
    const item = /^(\s*)- (.*)$|^(\s*)-$/.exec(text);
    if (item && list) {
      const parsed = scalar(item[2] ?? '');
      if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
      data[list].push(parsed.value);
      itemLines[list].push(line);
      continue;
    }
    const pair = /^([^\s:#\-][^:]*?|"[^"]+"|'[^']+')\s*:(?:\s+(.*))?$/.exec(text);
    if (!pair) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
    const key = pair[1].replace(/^(["'])(.*)\1$/, '$2');
    if (Object.hasOwn(data, key) || key === '__proto__') return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
    if (pair[2] === undefined || !pair[2].trim()) {
      const next = lines[i + 1];
      if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; itemLines[key] = []; list = key; }
      else { data[key] = null; list = null; }
      continue;
    }
    list = null;
    const parsed = value(pair[2]);
    if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
    data[key] = parsed.value;
  }
  return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null, itemLines };
}

export const splitLines = text => text.replace(/^/, '').split('\n').map(l => l.endsWith('\r') ? l.slice(0, -1) : l);

// One parsed file: its lines (all of them, numbered from 1), front matter and the links it holds.
// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. A
// link in a block list carries its item's line, one in a scalar or flow value its key's line.
export function parseFile(text, { markdown = false } = {}) {
  const lines = splitLines(text);
  const fm = parseFrontMatter(lines);
  const numbered = lines.map((t, i) => [i + 1, t]);
  const links = extractLinks(numbered.slice(fm.bodyLine - 1), { markdown });
  if (fm.frontmatter) {
    for (const field of LINK_FIELDS) {
      const raw = fm.frontmatter[field];
      const values = Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw];
      const items = fm.itemLines[field];
      values.forEach((v, i) => {
        const link = typeof v === 'string' ? parseWikilink(v.trim()) : null;
        if (link) links.push({ ...link, field, line: items ? items[i] : frontMatterLine(lines, fm.bodyLine, field) });
      });
    }
  }
  return { lines: numbered, bodyLine: fm.bodyLine, frontmatter: fm.frontmatter, frontmatterRaw: fm.raw, stop: fm.stop, links };
}
function frontMatterLine(lines, bodyLine, field) {
  const i = lines.slice(0, bodyLine - 1).findIndex(t => new RegExp(`^["']?${field}["']?\\s*:`).test(t));
  return i < 0 ? undefined : i + 1;
}

// Read a commit under a checked mapping (mapping.mjs). Returns the indexed files and every
// R1-R7 problem: { commit, files: [...], problems: [...] }, frozen, sorted by path.
export async function readLibrary({ repo, commit, mapping }) {
  if (!mapping || mapping.kind !== 'library') throw new LibraryError('library-mapping-invalid', 'expected a checked library mapping');
  const entries = await listTree(repo, commit);
  const problems = [], twins = new Set(), byLower = new Map();
  for (const e of entries) byLower.set(e.path.toLowerCase(), [...(byLower.get(e.path.toLowerCase()) ?? []), e.path]);
  for (const group of byLower.values()) if (group.length > 1) for (const path of group) twins.add(path);

  const candidates = [];
  for (const entry of entries) {
    const where = classify(mapping, entry.path);
    if (where.scope === 'unmapped') { problems.push(problem('R2', entry.path)); continue; }
    if (twins.has(entry.path)) { problems.push(problem('R7', entry.path)); continue; }
    if (!readable(entry)) { problems.push(problem('R1', entry.path)); continue; }
    candidates.push({ entry, where });
  }
  const texts = await readTexts(repo, candidates.map(c => c.entry));
  const files = [];
  for (const { entry, where } of candidates) {
    if (!texts.has(entry.path)) { problems.push(problem('R1', entry.path)); continue; }
    const parsed = parseFile(texts.get(entry.path));
    if (parsed.stop !== null) problems.push(problem('R3', entry.path, parsed.stop));
    files.push({ path: entry.path, ...where, name: entry.path.split('/').pop().replace(/\.md$/, ''), ...parsed });
  }

  const resolve = resolver(files.map(f => f.path), { vaultPrefix: mapping.vaultPrefix });
  const byPath = new Map(files.map(f => [f.path, f]));
  for (const file of files) {
    const kept = [];
    for (const link of file.links) {
      if (!link.file) continue;
      const target = resolve(file.path, link.file);
      if (!target.path) { if (!isAttachment(link)) problems.push(problem('R4', file.path, link.line)); continue; }
      const to = byPath.get(target.path);
      if (file.scope === 'canon') {
        if (to.scope === 'playground') problems.push(problem('R5', file.path, link.line));
        const sources = mapping.universes[file.universe].sources ?? [];
        if (to.universe !== file.universe && !sources.includes(to.universe)) problems.push(problem('R6', file.path, link.line));
      }
      kept.push({ ...link, path: target.path, step: target.step });
    }
    file.links = kept;
    delete file.stop;
  }
  files.sort((a, b) => a.path < b.path ? -1 : 1);
  return deepFreeze({ commit, files, problems: sortProblems(problems) });
}

function deepFreeze(value) {
  if (value && typeof value === 'object' && !Object.isFrozen(value)) { Object.freeze(value); for (const v of Object.values(value)) deepFreeze(v); }
  return value;
}