jason.woltje
ddf20567ee
(cherry picked from commitec7fa046c1) (cherry picked from commit9ebeb00fbb)
271 lines
14 KiB
JavaScript
271 lines
14 KiB
JavaScript
// The commit reader of design 1.3. It reads a commit of her repo, never a working tree: it lists the
|
||
// tree with `git ls-tree -r -z` and reads blobs with `cat-file`, through the same bounded git()
|
||
// and gitBytes() as packages/project, so hooks, protocols and the environment are fixed the same
|
||
// way. Nothing in the repo is checked out or run (D7). Each R-rule is a problem, never a refusal
|
||
// of the commit; the affected file or link is left out of the index.
|
||
import { git, gitBytes } from '../../project/src/git.mjs';
|
||
import { classify, excluded } from './scope.mjs';
|
||
import { extractLinks, isAttachment, parseWikilink, resolver } from './links.mjs';
|
||
import { LibraryError, problem, raise, sortProblems } from './problems.mjs';
|
||
|
||
export const MAX_BLOB = 1024 * 1024;
|
||
const MAX_LISTING = 32 * 1024 * 1024;
|
||
const SHA = /^[0-9a-f]{40}$/;
|
||
const ENTRY = /^([0-7]{6}) (blob|tree|commit) ([0-9a-f]{40}) +(-|\d+)\t(.+)$/s;
|
||
export const LINK_FIELDS = Object.freeze(['appears_in', 'relationships']);
|
||
const READERS = 8;
|
||
|
||
async function local(promise, message) {
|
||
try { return await promise; } catch (e) { raise('library-git', `${message} (${e.code ?? 'error'})`); }
|
||
}
|
||
|
||
export async function headCommit(repo) {
|
||
return local(git(repo, ['rev-parse', '--verify', '--end-of-options', 'HEAD^{commit}']), 'repository has no readable HEAD commit');
|
||
}
|
||
|
||
// Every entry of the commit's tree: { mode, type, oid, size, path }. Sizes come from `-l`, so no
|
||
// blob is opened to list it.
|
||
export async function listTree(repo, commit) {
|
||
if (typeof commit !== 'string' || !SHA.test(commit)) raise('library-commit', 'expected a full commit id');
|
||
const kind = await local(git(repo, ['cat-file', '-t', commit]), 'commit is not in the repository');
|
||
if (kind !== 'commit') raise('library-commit', 'not a commit');
|
||
const bytes = await local(gitBytes(repo, ['ls-tree', '-r', '-z', '-l', '--full-tree', commit], MAX_LISTING), 'tree listing refused');
|
||
let text;
|
||
try { text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); }
|
||
catch { raise('library-git', 'tree listing is not UTF-8'); }
|
||
const entries = text.split('\0');
|
||
if (entries.pop() !== '') raise('library-git', 'invalid tree listing');
|
||
return entries.map(entry => {
|
||
const m = ENTRY.exec(entry);
|
||
if (!m) raise('library-git', 'invalid tree listing');
|
||
return { mode: m[1], type: m[2], oid: m[3], size: m[4] === '-' ? null : Number(m[4]), path: m[5] };
|
||
});
|
||
}
|
||
|
||
// R1: only a 100644 blob up to 1 MiB of UTF-8 text without NUL is read.
|
||
export const readable = entry => entry.mode === '100644' && entry.type === 'blob'
|
||
&& Number.isSafeInteger(entry.size) && entry.size <= MAX_BLOB;
|
||
|
||
// Text of each readable entry, by path. An entry whose bytes aren't UTF-8 text is missing from the
|
||
// map, and the caller reports it under R1.
|
||
export async function readTexts(repo, entries) {
|
||
const texts = new Map(), queue = [...entries];
|
||
const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false });
|
||
const worker = async () => {
|
||
for (let entry = queue.shift(); entry; entry = queue.shift()) {
|
||
const bytes = await local(gitBytes(repo, ['cat-file', 'blob', entry.oid], MAX_BLOB), 'blob read refused');
|
||
let value;
|
||
try { value = decoder.decode(bytes); } catch { continue; }
|
||
if (!value.includes('\0')) texts.set(entry.path, value);
|
||
}
|
||
};
|
||
await Promise.all(Array.from({ length: READERS }, worker));
|
||
return texts;
|
||
}
|
||
|
||
const QUOTED = /^"(?:[^"\\]|\\.)*"$|^'(?:[^']|'')*'$/;
|
||
const PLAIN_START = /^[^\s\[\]{}&*!|>%@`"'#,]/;
|
||
function scalar(raw) {
|
||
const value = raw.trim();
|
||
if (parseWikilink(value)) return { ok: true, value };
|
||
if (QUOTED.test(value)) {
|
||
if (value[0] === "'") return { ok: true, value: value.slice(1, -1).replace(/''/g, "'") };
|
||
try { return { ok: true, value: JSON.parse(value) }; } catch { return { ok: false }; }
|
||
}
|
||
if (value === '' || value === '~' || value === 'null') return { ok: true, value: null };
|
||
if (!PLAIN_START.test(value) || /: |\s#/.test(value) || value.endsWith(':')) return { ok: false };
|
||
if (value === 'true' || value === 'false') return { ok: true, value: value === 'true' };
|
||
if (/^-?(0|[1-9]\d*)(\.\d+)?$/.test(value)) return { ok: true, value: Number(value) };
|
||
return { ok: true, value };
|
||
}
|
||
function flow(raw) {
|
||
const inner = raw.trim().slice(1, -1), items = [];
|
||
let current = '', quote = null;
|
||
for (const ch of inner) {
|
||
if (quote) { current += ch; if (ch === quote) quote = null; continue; }
|
||
if (ch === '"' || ch === "'") { quote = ch; current += ch; continue; }
|
||
if (ch === '[' || ch === ']' || ch === '{' || ch === '}') return { ok: false };
|
||
if (ch === ',') { items.push(current); current = ''; continue; }
|
||
current += ch;
|
||
}
|
||
if (quote) return { ok: false };
|
||
if (current.trim() || items.length) items.push(current);
|
||
const values = [];
|
||
for (const item of items) { const s = scalar(item); if (!s.ok || !item.trim()) return { ok: false }; values.push(s.value); }
|
||
return { ok: true, value: values };
|
||
}
|
||
function value(raw) {
|
||
const text = raw.trim();
|
||
if (text.startsWith('[') && !parseWikilink(text)) return text.endsWith(']') ? flow(text) : { ok: false };
|
||
return scalar(text);
|
||
}
|
||
|
||
// Front matter of design 1.3 (R3): scalars, quoted strings, flow lists and block lists of scalars or
|
||
// wikilinks. Anything else, a nested map, a block scalar or an anchor for example, leaves the whole
|
||
// front matter as raw text with one R3 problem at the line where parsing stopped.
|
||
// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. Parsed front matter also
|
||
// returns `itemLines`, each block list key's item lines in order.
|
||
export function parseFrontMatter(lines) {
|
||
if (lines[0]?.replace(/^/, '').trimEnd() !== '---') return { frontmatter: null, raw: null, bodyLine: 1, stop: null };
|
||
const close = lines.findIndex((text, i) => i > 0 && /^(---|\.\.\.)\s*$/.test(text));
|
||
if (close < 0) return { frontmatter: null, raw: null, bodyLine: 1, stop: 1 };
|
||
const raw = lines.slice(1, close).join('\n'), data = {}, itemLines = {};
|
||
let list = null;
|
||
for (let i = 1; i < close; i++) {
|
||
const text = lines[i], line = i + 1;
|
||
if (!text.trim() || /^\s*#/.test(text)) continue;
|
||
const item = /^(\s*)- (.*)$|^(\s*)-$/.exec(text);
|
||
if (item && list) {
|
||
const parsed = scalar(item[2] ?? '');
|
||
if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
|
||
data[list].push(parsed.value);
|
||
itemLines[list].push(line);
|
||
continue;
|
||
}
|
||
const pair = /^([^\s:#\-][^:]*?|"[^"]+"|'[^']+')\s*:(?:\s+(.*))?$/.exec(text);
|
||
if (!pair) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
|
||
const key = pair[1].replace(/^(["'])(.*)\1$/, '$2');
|
||
if (Object.hasOwn(data, key) || key === '__proto__') return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
|
||
if (pair[2] === undefined || !pair[2].trim()) {
|
||
const next = lines[i + 1];
|
||
if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; itemLines[key] = []; list = key; }
|
||
else { data[key] = null; list = null; }
|
||
continue;
|
||
}
|
||
list = null;
|
||
const parsed = value(pair[2]);
|
||
if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
|
||
data[key] = parsed.value;
|
||
}
|
||
return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null, itemLines };
|
||
}
|
||
|
||
export const splitLines = text => text.replace(/^/, '').split('\n').map(l => l.endsWith('\r') ? l.slice(0, -1) : l);
|
||
|
||
// One parsed file: its lines (all of them, numbered from 1), front matter and the links it holds.
|
||
// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. A
|
||
// link in a block list carries its item's line, one in a scalar or flow value its key's line.
|
||
export function parseFile(text, { markdown = false } = {}) {
|
||
const lines = splitLines(text);
|
||
const fm = parseFrontMatter(lines);
|
||
const numbered = lines.map((t, i) => [i + 1, t]);
|
||
const links = extractLinks(numbered.slice(fm.bodyLine - 1), { markdown });
|
||
if (fm.frontmatter) {
|
||
for (const field of LINK_FIELDS) {
|
||
const raw = fm.frontmatter[field];
|
||
const values = Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw];
|
||
const items = fm.itemLines[field];
|
||
values.forEach((v, i) => {
|
||
const link = typeof v === 'string' ? parseWikilink(v.trim()) : null;
|
||
if (link) links.push({ ...link, field, line: items ? items[i] : frontMatterLine(lines, fm.bodyLine, field) });
|
||
});
|
||
}
|
||
}
|
||
return { lines: numbered, bodyLine: fm.bodyLine, frontmatter: fm.frontmatter, frontmatterRaw: fm.raw, stop: fm.stop, links };
|
||
}
|
||
function frontMatterLine(lines, bodyLine, field) {
|
||
const i = lines.slice(0, bodyLine - 1).findIndex(t => new RegExp(`^["']?${field}["']?\\s*:`).test(t));
|
||
return i < 0 ? undefined : i + 1;
|
||
}
|
||
|
||
// Read a commit under a checked mapping (mapping.mjs). Returns the indexed files and every
|
||
// R1-R8 problem: { commit, files: [...], problems: [...] }, frozen, sorted by path.
|
||
export async function readLibrary({ repo, commit, mapping }) {
|
||
if (!mapping || mapping.kind !== 'library') throw new LibraryError('library-mapping-invalid', 'expected a checked library mapping');
|
||
const entries = await listTree(repo, commit);
|
||
const problems = [], twins = new Set(), byLower = new Map();
|
||
for (const e of entries) byLower.set(e.path.toLowerCase(), [...(byLower.get(e.path.toLowerCase()) ?? []), e.path]);
|
||
for (const group of byLower.values()) if (group.length > 1) for (const path of group) twins.add(path);
|
||
|
||
const candidates = [];
|
||
for (const entry of entries) {
|
||
const where = classify(mapping, entry.path);
|
||
if (where.scope === 'unmapped') { problems.push(problem('R2', entry.path)); continue; }
|
||
if (twins.has(entry.path)) { problems.push(problem('R7', entry.path)); continue; }
|
||
if (!readable(entry)) { problems.push(problem('R1', entry.path)); continue; }
|
||
candidates.push({ entry, where });
|
||
}
|
||
const texts = await readTexts(repo, candidates.map(c => c.entry));
|
||
const files = [];
|
||
for (const { entry, where } of candidates) {
|
||
if (!texts.has(entry.path)) { problems.push(problem('R1', entry.path)); continue; }
|
||
const parsed = parseFile(texts.get(entry.path));
|
||
if (parsed.stop !== null) problems.push(problem('R3', entry.path, parsed.stop));
|
||
files.push({ path: entry.path, ...where, name: entry.path.split('/').pop().replace(/\.md$/, ''), ...parsed });
|
||
}
|
||
|
||
const resolve = resolver(files.map(f => f.path), { vaultPrefix: mapping.vaultPrefix });
|
||
const byPath = new Map(files.map(f => [f.path, f]));
|
||
for (const file of files) {
|
||
const kept = [];
|
||
for (const link of file.links) {
|
||
if (!link.file) continue;
|
||
const target = resolve(file.path, link.file);
|
||
if (!target.path) { if (!isAttachment(link)) problems.push(problem('R4', file.path, link.line)); continue; }
|
||
const to = byPath.get(target.path);
|
||
if (file.scope === 'canon') {
|
||
if (to.scope === 'playground') problems.push(problem('R5', file.path, link.line));
|
||
const sources = mapping.universes[file.universe].sources ?? [];
|
||
// A hybrid's link into a path its exclude covers is R6 too (C2-15). An excluded path is always in a
|
||
// source, so a link gets at most one R6.
|
||
if ((to.universe !== file.universe && !sources.includes(to.universe)) || excluded(mapping, file.universe, to.path))
|
||
problems.push(problem('R6', file.path, link.line));
|
||
}
|
||
kept.push({ ...link, path: target.path, step: target.step });
|
||
}
|
||
file.links = kept;
|
||
delete file.stop;
|
||
}
|
||
problems.push(...pinProblems(mapping, files, byPath));
|
||
files.sort((a, b) => a.path < b.path ? -1 : 1);
|
||
return deepFreeze({ commit, files, problems: sortProblems(problems) });
|
||
}
|
||
|
||
// R8 (C2-15, design 1.4): each pin of each hybrid. The pinned entity gets R8 when it isn't indexed (absent,
|
||
// R1, R2 or R7) or its front matter doesn't hold `field` as a scalar whose text equals String(value); a
|
||
// list, null, a missing key and R3 front matter all count as changed. A hybrid canon file is a version of
|
||
// the entity when classify gives it the entity's type and its name matches: the trimmed front matter
|
||
// `name` on both sides, or the file names when either has no text name. A version that sets `field` to
|
||
// another value gets R8 on that key's line. A link alone never makes a version.
|
||
function pinProblems(mapping, files, byPath) {
|
||
const out = [];
|
||
for (const [hybrid, entry] of Object.entries(mapping.universes)) {
|
||
if (entry.kind !== 'hybrid') continue;
|
||
for (const { pin } of entry.constraints.filter(c => c.pin)) {
|
||
const entity = byPath.get(pin.entity);
|
||
if (!entity) out.push(problem('R8', pin.entity));
|
||
else if (!pinned(entity, pin)) out.push(problem('R8', pin.entity, keyLine(entity, pin.field)));
|
||
const type = classify(mapping, pin.entity).type;
|
||
const stem = pin.entity.split('/').pop().replace(/\.md$/, '');
|
||
const name = entity ? textName(entity) : null;
|
||
for (const file of files) {
|
||
if (file.universe !== hybrid || file.scope !== 'canon' || file.type !== type) continue;
|
||
const own = textName(file);
|
||
if ((name !== null && own !== null ? own !== name : file.name !== stem)) continue;
|
||
if (file.frontmatter && Object.hasOwn(file.frontmatter, pin.field) && !pinned(file, pin))
|
||
out.push(problem('R8', file.path, keyLine(file, pin.field)));
|
||
}
|
||
}
|
||
}
|
||
// Two pins (or two hybrids) on one missing or changed entity give one entry per path and line (Filbert N1).
|
||
const seen = new Set();
|
||
return out.filter(p => { const key = `${p.path}\0${p.line ?? ''}`; return !seen.has(key) && seen.add(key); });
|
||
}
|
||
const textName = file => typeof file.frontmatter?.name === 'string' && file.frontmatter.name.trim() ? file.frontmatter.name.trim() : null;
|
||
function pinned(file, { field, value }) {
|
||
const held = file.frontmatter && Object.hasOwn(file.frontmatter, field) ? file.frontmatter[field] : undefined;
|
||
return ['string', 'number', 'boolean'].includes(typeof held) && String(held) === String(value);
|
||
}
|
||
// The front matter line of `field` (bare or quoted key), or undefined.
|
||
function keyLine(file, field) {
|
||
if (file.bodyLine < 3) return undefined;
|
||
const key = field.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||
const at = file.lines.slice(1, file.bodyLine - 2).find(([, t]) => new RegExp(`^["']?${key}["']?\\s*:`).test(t));
|
||
return at?.[0];
|
||
}
|
||
|
||
function deepFreeze(value) {
|
||
if (value && typeof value === 'object' && !Object.isFrozen(value)) { Object.freeze(value); for (const v of Object.values(value)) deepFreeze(v); }
|
||
return value;
|
||
}
|