From 777f7358a6afa63e7c6fdc2371a13ac3c58c4c09 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Tue, 20 Oct 2026 17:07:28 -0500 Subject: [PATCH] gosalyn: C2-01b survey escapes DEL, C1 and bidi controls in folder names, a front matter block list link carries its item line, a link or embed to a non-.md target is an attachment, not followed and never R4, counted in the survey (row 231, #1733, webby-0041 N1-N3, sage-1517) --- packages/library/README.md | 15 +++++++++++---- packages/library/src/index.mjs | 2 +- packages/library/src/links.mjs | 9 ++++++++- packages/library/src/reader.mjs | 24 ++++++++++++++---------- packages/library/src/survey.mjs | 23 ++++++++++++++--------- packages/library/tests/links.test.mjs | 11 ++++++++++- packages/library/tests/reader.test.mjs | 22 +++++++++++++++++++++- packages/library/tests/survey.test.mjs | 20 +++++++++++++++++++- 8 files changed, 98 insertions(+), 28 deletions(-) diff --git a/packages/library/README.md b/packages/library/README.md index 200011cfe..3daf9ea8a 100644 --- a/packages/library/README.md +++ b/packages/library/README.md @@ -57,7 +57,8 @@ environment and ceiling directory apply. A failed git call refuses `library-git` It returns `{ commit, files, problems }`, deep-frozen and sorted. Each file has its path, scope, universe, book, type, numbered lines, `bodyLine`, parsed `frontmatter` or `null`, `frontmatterRaw` -and `links`. Front matter `appears_in` and `relationships` wikilinks count as links. +and `links`. Front matter `appears_in` and `relationships` wikilinks count as links. A link in a block +list carries its list item's line. One in a scalar or flow value carries its key's line. Problems carry `{ code, path, line, message }`: @@ -84,8 +85,12 @@ then with `.md` added: 3. `relative`: relative to the linking file, never above the repo root. 4. `shortest`: a target with no `/` that matches the base name of exactly one indexed file. -It resolves only against indexed files. A link to a `.canvas` file or an attachment is R4. A link into -another repo resolves to nothing. `extractLinks` skips fenced and inline code. +It resolves only against indexed files. A link or `![[...]]` embed whose target ends in an extension +other than `.md`, in any case, is an attachment: an extension is a dot and up to 8 letters and digits, +at least one a letter. The reader neither resolves nor indexes an attachment, and it is never R4. A +note embed, with no extension or with `.md`, resolves like any link, so an unresolved one is R4. A +link to a `.canvas` file is an attachment too. A link into another repo resolves to nothing. +`extractLinks` skips fenced and inline code. ## renderItems @@ -110,7 +115,9 @@ mosaic library survey [--mapping ] ``` It reads the repo's HEAD commit and prints folder names, file counts by extension, front matter key -sets and link-form counts. It never prints file text, front matter values, file names or link targets. +sets, link-form counts and the count of body links to attachments, which still count under their +form. Folder names print as JSON strings, with DEL, C1 and bidi controls also as `\uXXXX`. It never +prints file text, front matter values, file names or link targets. A key that is not a short identifier prints as `(other key)`, since it could be her words. With `--mapping` it adds scope counts, resolver step counts and problem counts by rule. diff --git a/packages/library/src/index.mjs b/packages/library/src/index.mjs index a76a34554..f64d77860 100644 --- a/packages/library/src/index.mjs +++ b/packages/library/src/index.mjs @@ -2,7 +2,7 @@ export { LibraryError, RULES, problem, sortProblems } from './problems.mjs'; export { checkMapping, TYPES } from './mapping.mjs'; export { classify } from './scope.mjs'; -export { extractLinks, parseWikilink, resolver, formOf } from './links.mjs'; +export { extractLinks, parseWikilink, resolver, formOf, isAttachment } from './links.mjs'; export { readLibrary, listTree, readTexts, parseFile, parseFrontMatter, headCommit, MAX_BLOB, LINK_FIELDS } from './reader.mjs'; export { renderItems, KINDS, NOTICES, UNREADABLE } from './render.mjs'; export { survey, formatSurvey, surveyMain } from './survey.mjs'; diff --git a/packages/library/src/links.mjs b/packages/library/src/links.mjs index 4d8d69b3a..c63b75beb 100644 --- a/packages/library/src/links.mjs +++ b/packages/library/src/links.mjs @@ -1,6 +1,8 @@ // Wikilinks and the link resolver of design 1.3. Links are [[target]], [[target|label]] and // [[target#heading]], with ![[...]] embeds read the same way. Links inside fenced code blocks and -// inline code spans are not links, as in Obsidian. +// inline code spans are not links, as in Obsidian. A link or embed whose target ends in an extension +// other than `.md`, in any case, refers to an attachment (design 1.3): the reader neither resolves +// nor indexes it, and it raises no problem. import { posix } from 'node:path'; const WIKILINK = /(!?)\[\[([^\[\]\n]+?)\]\]/g; @@ -23,6 +25,11 @@ function parts(inner, embed) { embed, }; } +// An extension is a dot and up to 8 letters and digits, at least one a letter, at the end of the +// target's last segment, so `Chapter 1.5` and `Mr. Smith` are notes and `cover.png` is not. +export const isAttachment = link => !link.markdown + && /\.(?=[a-z0-9]*[a-z])[a-z0-9]{1,8}$/i.test(link.file.slice(link.file.lastIndexOf('/') + 1)) + && !/\.md$/i.test(link.file); // The form a link is counted under in the survey. export const formOf = link => link.markdown ? 'markdown' : link.embed ? 'embed' : !link.file ? 'self' : link.label !== null ? 'label' : link.heading !== null ? 'heading' : 'plain'; diff --git a/packages/library/src/reader.mjs b/packages/library/src/reader.mjs index 752856ce0..64a7cf07b 100644 --- a/packages/library/src/reader.mjs +++ b/packages/library/src/reader.mjs @@ -5,7 +5,7 @@ // of the commit; the affected file or link is left out of the index. import { git, gitBytes } from '../../project/src/git.mjs'; import { classify } from './scope.mjs'; -import { extractLinks, parseWikilink, resolver } from './links.mjs'; +import { extractLinks, isAttachment, parseWikilink, resolver } from './links.mjs'; import { LibraryError, problem, raise, sortProblems } from './problems.mjs'; export const MAX_BLOB = 1024 * 1024; @@ -103,12 +103,13 @@ function value(raw) { // Front matter of design 1.3 (R3): scalars, quoted strings, flow lists and block lists of scalars or // wikilinks. Anything else, a nested map, a block scalar or an anchor for example, leaves the whole // front matter as raw text with one R3 problem at the line where parsing stopped. -// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. +// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. Parsed front matter also +// returns `itemLines`, each block list key's item lines in order. export function parseFrontMatter(lines) { if (lines[0]?.replace(/^/, '').trimEnd() !== '---') return { frontmatter: null, raw: null, bodyLine: 1, stop: null }; const close = lines.findIndex((text, i) => i > 0 && /^(---|\.\.\.)\s*$/.test(text)); if (close < 0) return { frontmatter: null, raw: null, bodyLine: 1, stop: 1 }; - const raw = lines.slice(1, close).join('\n'), data = {}; + const raw = lines.slice(1, close).join('\n'), data = {}, itemLines = {}; let list = null; for (let i = 1; i < close; i++) { const text = lines[i], line = i + 1; @@ -118,6 +119,7 @@ export function parseFrontMatter(lines) { const parsed = scalar(item[2] ?? ''); if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; data[list].push(parsed.value); + itemLines[list].push(line); continue; } const pair = /^([^\s:#\-][^:]*?|"[^"]+"|'[^']+')\s*:(?:\s+(.*))?$/.exec(text); @@ -126,7 +128,7 @@ export function parseFrontMatter(lines) { if (Object.hasOwn(data, key) || key === '__proto__') return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; if (pair[2] === undefined || !pair[2].trim()) { const next = lines[i + 1]; - if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; list = key; } + if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; itemLines[key] = []; list = key; } else { data[key] = null; list = null; } continue; } @@ -135,13 +137,14 @@ export function parseFrontMatter(lines) { if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line }; data[key] = parsed.value; } - return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null }; + return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null, itemLines }; } export const splitLines = text => text.replace(/^/, '').split('\n').map(l => l.endsWith('\r') ? l.slice(0, -1) : l); // One parsed file: its lines (all of them, numbered from 1), front matter and the links it holds. -// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. +// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. A +// link in a block list carries its item's line, one in a scalar or flow value its key's line. export function parseFile(text, { markdown = false } = {}) { const lines = splitLines(text); const fm = parseFrontMatter(lines); @@ -151,10 +154,11 @@ export function parseFile(text, { markdown = false } = {}) { for (const field of LINK_FIELDS) { const raw = fm.frontmatter[field]; const values = Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw]; - for (const v of values) { + const items = fm.itemLines[field]; + values.forEach((v, i) => { const link = typeof v === 'string' ? parseWikilink(v.trim()) : null; - if (link) links.push({ ...link, field, line: frontMatterLine(lines, fm.bodyLine, field) }); - } + if (link) links.push({ ...link, field, line: items ? items[i] : frontMatterLine(lines, fm.bodyLine, field) }); + }); } } return { lines: numbered, bodyLine: fm.bodyLine, frontmatter: fm.frontmatter, frontmatterRaw: fm.raw, stop: fm.stop, links }; @@ -195,7 +199,7 @@ export async function readLibrary({ repo, commit, mapping }) { for (const file of files) { const kept = []; for (const link of file.links) { - if (!link.file) continue; + if (!link.file || isAttachment(link)) continue; const target = resolve(file.path, link.file); if (!target.path) { problems.push(problem('R4', file.path, link.line)); continue; } const to = byPath.get(target.path); diff --git a/packages/library/src/survey.mjs b/packages/library/src/survey.mjs index 3e72623fa..c35980c40 100644 --- a/packages/library/src/survey.mjs +++ b/packages/library/src/survey.mjs @@ -1,6 +1,7 @@ // `mosaic library survey [--mapping ]` (design 1.1, 10 step 7). It prints what a mapping -// needs: folder names, file counts by extension, front matter key sets and link-form counts. It -// never prints file text, front matter values, file names or link targets. With a mapping it adds +// needs: folder names, file counts by extension, front matter key sets, link-form counts and the +// count of body links to attachments. It never prints file text, front matter values, file names or +// link targets. With a mapping it adds // scope counts, resolver step counts and R-problem counts by code. It reads the repo's HEAD commit // through the commit reader, never the working tree. import { readFileSync, realpathSync, statSync } from 'node:fs'; @@ -8,7 +9,7 @@ import { resolve as resolvePath } from 'node:path'; import { pathToFileURL } from 'node:url'; import { checkMapping } from './mapping.mjs'; import { classify } from './scope.mjs'; -import { formOf } from './links.mjs'; +import { formOf, isAttachment } from './links.mjs'; import { headCommit, listTree, parseFile, readable, readLibrary, readTexts } from './reader.mjs'; import { LibraryError, RULES } from './problems.mjs'; @@ -37,7 +38,7 @@ export async function survey({ repo, commit, mapping }) { const texts = await readTexts(repo, markdown.filter(readable)); const keySets = new Map(), forms = Object.fromEntries(FORMS.map(f => [f, 0])); const fields = { appears_in: { wikilink: 0, name: 0 }, relationships: { wikilink: 0, name: 0 } }; - let noFrontMatter = 0, unparsed = 0; + let noFrontMatter = 0, unparsed = 0, attachments = 0; for (const text of texts.values()) { const file = parseFile(text, { markdown: true }); if (file.stop !== null) unparsed++; @@ -51,14 +52,14 @@ export async function survey({ repo, commit, mapping }) { fields[field][typeof v === 'string' && /^\s*!?\[\[[^\]]+\]\]\s*$/.test(v) ? 'wikilink' : 'name']++; } } - for (const link of file.links) if (!link.field) forms[formOf(link)]++; + for (const link of file.links) if (!link.field) { forms[formOf(link)]++; if (isAttachment(link)) attachments++; } } const result = { commit: head, entries: entries.length, folders: [...folders].sort(([a], [b]) => a < b ? -1 : 1).map(([name, counts]) => ({ name, ...counts })), markdown: { files: markdown.length, read: texts.size, noFrontMatter, unparsed }, keySets: [...keySets].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([keys, count]) => ({ keys, count })), - forms, fields, + forms, attachments, fields, }; if (mapping) { const scopes = { canon: 0, playground: 0, unmapped: 0 }; @@ -74,14 +75,18 @@ export async function survey({ repo, commit, mapping }) { return result; } -// Folder names are quoted as JSON strings, so no name can carry a control sequence to the terminal. +// Folder names are quoted as JSON strings, which escape C0 controls but leave DEL, C1 and bidi +// controls raw. quote() escapes those too, so no name can carry a control sequence to the terminal +// or reorder the line around it. +const RAW_CONTROLS = /[\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/g; +export const quote = name => JSON.stringify(name).replace(RAW_CONTROLS, c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`); export function formatSurvey(s) { const out = [`commit ${s.commit}`, `entries ${s.entries}`, 'folders (direct children: md canvas base other)']; - for (const f of s.folders) out.push(` ${f.name === '' ? '(root)' : JSON.stringify(f.name)} ${f.md} ${f.canvas} ${f.base} ${f.other}`); + for (const f of s.folders) out.push(` ${f.name === '' ? '(root)' : quote(f.name)} ${f.md} ${f.canvas} ${f.base} ${f.other}`); out.push(`markdown ${s.markdown.files} files, ${s.markdown.read} read, ${s.markdown.noFrontMatter} without front matter, ${s.markdown.unparsed} with front matter not fully parsed`); out.push('front matter key sets'); for (const k of s.keySets) out.push(` ${k.count} [${k.keys}]`); - out.push(`body links ${FORMS.map(f => `${f} ${s.forms[f]}`).join(', ')}`); + out.push(`body links ${FORMS.map(f => `${f} ${s.forms[f]}`).join(', ')}; attachment ${s.attachments}`); for (const [field, c] of Object.entries(s.fields)) out.push(`field ${field} wikilink ${c.wikilink}, name ${c.name}`); if (s.scopes) { out.push(`scopes canon ${s.scopes.canon}, playground ${s.scopes.playground}, unmapped ${s.scopes.unmapped}; indexed ${s.indexed}`); diff --git a/packages/library/tests/links.test.mjs b/packages/library/tests/links.test.mjs index 8cef702ed..746ca3bac 100644 --- a/packages/library/tests/links.test.mjs +++ b/packages/library/tests/links.test.mjs @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { extractLinks, parseWikilink, resolver } from '../src/index.mjs'; +import { extractLinks, isAttachment, parseWikilink, resolver } from '../src/index.mjs'; const PATHS = ['Book One/Characters/Mara Vell.md', 'Book One/Characters/Tomas.md', 'Book One/Places/Harbor Town.md', 'Book One/Chapters/Chapter 01.md', 'Book One/Places/Notes.md', 'Book One/Characters/Notes.md', 'Mara Vell.md']; @@ -48,3 +48,12 @@ test('wikilink forms, embeds, and links skipped inside code', () => { assert.deepEqual(extractLinks(lines).map(l => [l.file, l.line]), [['One', 1], ['Two', 1], ['Three', 1], ['Four', 1], ['', 1]]); assert.deepEqual(extractLinks(lines, { markdown: true }).filter(l => l.markdown).map(l => l.file), ['Five.md']); }); + +test('a target with an extension other than .md, in any case, is an attachment', () => { + const att = text => isAttachment(parseWikilink(text)); + for (const s of ['![[cover.png]]', '[[cover.PNG]]', '![[Art/map.pdf|Map]]', '[[Board.canvas]]', '![[clip.mp4#t=5]]']) + assert.equal(att(s), true, s); + for (const s of ['![[missing-note]]', '![[note.md]]', '[[Note.MD]]', '[[Chapter 1.5]]', '[[Mr. Smith]]', '[[A.b/Note]]', '[[#Self]]']) + assert.equal(att(s), false, s); + assert.equal(isAttachment({ file: 'pic.png', markdown: true }), false, 'markdown links are never followed anyway'); +}); diff --git a/packages/library/tests/reader.test.mjs b/packages/library/tests/reader.test.mjs index 1913e2974..22cfafffa 100644 --- a/packages/library/tests/reader.test.mjs +++ b/packages/library/tests/reader.test.mjs @@ -24,7 +24,7 @@ test('the fixture reads clean: every resolver step, appears_in and relationships assert.deepEqual(mara.frontmatter, { type: 'character', name: 'Mara Vell', appears_in: '[[Book One/Chapters/Chapter 01]]', relationships: ['[[Tomas]]', 'The Ferryman'] }); assert.deepEqual(mara.links.map(l => [l.field, l.path, l.line]), [ - ['appears_in', 'Book One/Chapters/Chapter 01.md', 4], ['relationships', 'Book One/Characters/Tomas.md', 5]]); + ['appears_in', 'Book One/Chapters/Chapter 01.md', 4], ['relationships', 'Book One/Characters/Tomas.md', 6]]); assert.deepEqual(file(result, 'Book One/Characters/Tomas.md').links.map(l => [l.field, l.path]), [['appears_in', 'Book One/Chapters/Chapter 01.md']]); assert.equal(mara.bodyLine, 9); @@ -97,6 +97,26 @@ test('R4: a link to no file, to more than one, or into another repo is reported assert.deepEqual(file(result, 'Book One/Chapters/Links.md').links.map(l => l.path), ['Book One/Characters/Tomas.md']); }); +test('a block list link carries its item line, a scalar or flow link its key line, and problems follow', async t => { + const r = repo(t, { ...BASE, 'Book One/Characters/Ana.md': ['---', 'appears_in: ["[[Nobody]]"]', 'relationships:', ' - "[[Tomas]]"', + ' - Plain Name', ' - "[[Ghost]]"', ' - "[[What If]]"', '---', ''].join('\n') }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.deepEqual(file(result, 'Book One/Characters/Ana.md').links.map(l => [l.field, l.path, l.line]), [ + ['relationships', 'Book One/Characters/Tomas.md', 4], ['relationships', 'Book One/Playground/What If.md', 7]]); + assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Characters/Ana.md').map(p => [p.code, p.line]), + [['R4', 2], ['R4', 6], ['R5', 7]]); +}); + +test('an attachment link or embed is not followed and gives no problem; a note embed resolves', async t => { + const r = repo(t, { ...BASE, 'Book One/Places/note.md': 'n\n', 'Book One/Places/Shore.md': ['---', + 'relationships:', ' - "![[portrait.JPG]]"', '---', '![[cover.png]] [[Board.canvas]] ![[map.pdf|Map]]', + '![[missing-note]] ![[note.md]] ![[Harbor Town]]', ''].join('\n') }); + const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); + assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Places/Shore.md').map(p => [p.code, p.line]), [['R4', 6]]); + assert.deepEqual(file(result, 'Book One/Places/Shore.md').links.map(l => [l.path, l.embed]), + [['Book One/Places/note.md', true], ['Book One/Places/Harbor Town.md', true]]); +}); + test('R5: a canon file that refers into the playground is reported, and the link still goes by path', async t => { const r = repo(t, { ...BASE, 'Book One/Places/Dock.md': 'See [[What If]].\n' }); const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() }); diff --git a/packages/library/tests/survey.test.mjs b/packages/library/tests/survey.test.mjs index 48a120813..1a809f8e9 100644 --- a/packages/library/tests/survey.test.mjs +++ b/packages/library/tests/survey.test.mjs @@ -39,13 +39,31 @@ test('the survey prints folder names, counts, key sets and link-form counts, and ' 1 [appears_in, name, type]', ' 1 [name]', ' 1 [type]', - 'body links plain 3, label 1, heading 1, embed 0, self 0, markdown 1', + 'body links plain 3, label 1, heading 1, embed 0, self 0, markdown 1; attachment 0', 'field appears_in wikilink 2, name 0', 'field relationships wikilink 1, name 1', ''].join('\n')); for (const s of NEVER) assert.ok(!out.stdout.includes(s), s); }); +test('folder names print DEL, C1 and bidi controls as escapes', async t => { + // One name per escaped range, with each range's first and last code point. + const names = ['A\u007fB', 'C\u0080D\u009fE', 'F\u200eG\u200fH', 'I\u202aJ\u202eK', 'L\u2066M\u2069N']; + const r = repo(t, Object.fromEntries(names.map(n => [`${n}/a.md`, `${MARKER}\n`]))); + const out = await run(['survey', r.repo]); + assert.equal(out.code, 0, out.stderr); + assert.deepEqual(out.stdout.split('\n').slice(3, 8), [ + ' "A\\u007fB" 1 0 0 0', ' "C\\u0080D\\u009fE" 1 0 0 0', ' "F\\u200eG\\u200fH" 1 0 0 0', + ' "I\\u202aJ\\u202eK" 1 0 0 0', ' "L\\u2066M\\u2069N" 1 0 0 0']); + assert.doesNotMatch(out.stdout, /[\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/); +}); + +test('an embed of an attachment counts as an embed and as an attachment', async t => { + const r = repo(t, { 'Book One/Chapters/A.md': ['![[cover.png]] ![[missing-note]] [[map.PDF]] [[Chapter 1.5]]', ''].join('\n') }); + const out = await run(['survey', r.repo]); + assert.match(out.stdout, /^body links plain 2, label 0, heading 0, embed 2, self 0, markdown 0; attachment 2$/m); +}); + test('with --mapping the survey adds scope, resolver step and problem counts', async t => { const r = repo(t); const out = await run(['survey', r.repo, '--mapping', join(FIXTURES, 'mapping.json')]);