gosalyn: C2-01b survey escapes DEL, C1 and bidi controls in folder names, a front matter block list link carries its item line, a link or embed to a non-.md target is an attachment, not followed and never R4, counted in the survey (row 231, #1733, webby-0041 N1-N3, sage-1517)

packages/library/README.md

@@ -57,7 +57,8 @@ environment and ceiling directory apply. A failed git call refuses `library-git`
 
 It returns `{ commit, files, problems }`, deep-frozen and sorted. Each file has its path, scope,
 universe, book, type, numbered lines, `bodyLine`, parsed `frontmatter` or `null`, `frontmatterRaw`
-and `links`. Front matter `appears_in` and `relationships` wikilinks count as links.
+and `links`. Front matter `appears_in` and `relationships` wikilinks count as links. A link in a block
+list carries its list item's line. One in a scalar or flow value carries its key's line.
 
 Problems carry `{ code, path, line, message }`:
 
@@ -84,8 +85,12 @@ then with `.md` added:
 3. `relative`: relative to the linking file, never above the repo root.
 4. `shortest`: a target with no `/` that matches the base name of exactly one indexed file.
 
-It resolves only against indexed files. A link to a `.canvas` file or an attachment is R4. A link into
+It resolves only against indexed files. A link or `![[...]]` embed whose target ends in an extension
-another repo resolves to nothing. `extractLinks` skips fenced and inline code.
+other than `.md`, in any case, is an attachment: an extension is a dot and up to 8 letters and digits,
+at least one a letter. The reader neither resolves nor indexes an attachment, and it is never R4. A
+note embed, with no extension or with `.md`, resolves like any link, so an unresolved one is R4. A
+link to a `.canvas` file is an attachment too. A link into another repo resolves to nothing.
+`extractLinks` skips fenced and inline code.
 
 ## renderItems
 
@@ -110,7 +115,9 @@ mosaic library survey <repo> [--mapping <file>]
 ```
 
 It reads the repo's HEAD commit and prints folder names, file counts by extension, front matter key
-sets and link-form counts. It never prints file text, front matter values, file names or link targets.
+sets, link-form counts and the count of body links to attachments, which still count under their
+form. Folder names print as JSON strings, with DEL, C1 and bidi controls also as `\uXXXX`. It never
+prints file text, front matter values, file names or link targets.
 A key that is not a short identifier prints as `(other key)`, since it could be her words. With
 `--mapping` it adds scope counts, resolver step counts and problem counts by rule.
 

packages/library/src/index.mjs

@@ -2,7 +2,7 @@
 export { LibraryError, RULES, problem, sortProblems } from './problems.mjs';
 export { checkMapping, TYPES } from './mapping.mjs';
 export { classify } from './scope.mjs';
-export { extractLinks, parseWikilink, resolver, formOf } from './links.mjs';
+export { extractLinks, parseWikilink, resolver, formOf, isAttachment } from './links.mjs';
 export { readLibrary, listTree, readTexts, parseFile, parseFrontMatter, headCommit, MAX_BLOB, LINK_FIELDS } from './reader.mjs';
 export { renderItems, KINDS, NOTICES, UNREADABLE } from './render.mjs';
 export { survey, formatSurvey, surveyMain } from './survey.mjs';

packages/library/src/links.mjs

@@ -1,6 +1,8 @@
 // Wikilinks and the link resolver of design 1.3. Links are [[target]], [[target|label]] and
 // [[target#heading]], with ![[...]] embeds read the same way. Links inside fenced code blocks and
-// inline code spans are not links, as in Obsidian.
+// inline code spans are not links, as in Obsidian. A link or embed whose target ends in an extension
+// other than `.md`, in any case, refers to an attachment (design 1.3): the reader neither resolves
+// nor indexes it, and it raises no problem.
 import { posix } from 'node:path';
 
 const WIKILINK = /(!?)\[\[([^\[\]\n]+?)\]\]/g;
@@ -23,6 +25,11 @@ function parts(inner, embed) {
     embed,
   };
 }
+// An extension is a dot and up to 8 letters and digits, at least one a letter, at the end of the
+// target's last segment, so `Chapter 1.5` and `Mr. Smith` are notes and `cover.png` is not.
+export const isAttachment = link => !link.markdown
+  && /\.(?=[a-z0-9]*[a-z])[a-z0-9]{1,8}$/i.test(link.file.slice(link.file.lastIndexOf('/') + 1))
+  && !/\.md$/i.test(link.file);
 // The form a link is counted under in the survey.
 export const formOf = link => link.markdown ? 'markdown' : link.embed ? 'embed' : !link.file ? 'self'
   : link.label !== null ? 'label' : link.heading !== null ? 'heading' : 'plain';

packages/library/src/reader.mjs

@@ -5,7 +5,7 @@
 // of the commit; the affected file or link is left out of the index.
 import { git, gitBytes } from '../../project/src/git.mjs';
 import { classify } from './scope.mjs';
-import { extractLinks, parseWikilink, resolver } from './links.mjs';
+import { extractLinks, isAttachment, parseWikilink, resolver } from './links.mjs';
 import { LibraryError, problem, raise, sortProblems } from './problems.mjs';
 
 export const MAX_BLOB = 1024 * 1024;
@@ -103,12 +103,13 @@ function value(raw) {
 // Front matter of design 1.3 (R3): scalars, quoted strings, flow lists and block lists of scalars or
 // wikilinks. Anything else, a nested map, a block scalar or an anchor for example, leaves the whole
 // front matter as raw text with one R3 problem at the line where parsing stopped.
-// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers.
+// Returns { frontmatter, raw, bodyLine, stop } with 1-based line numbers. Parsed front matter also
+// returns `itemLines`, each block list key's item lines in order.
 export function parseFrontMatter(lines) {
   if (lines[0]?.replace(/^/, '').trimEnd() !== '---') return { frontmatter: null, raw: null, bodyLine: 1, stop: null };
   const close = lines.findIndex((text, i) => i > 0 && /^(---|\.\.\.)\s*$/.test(text));
   if (close < 0) return { frontmatter: null, raw: null, bodyLine: 1, stop: 1 };
-  const raw = lines.slice(1, close).join('\n'), data = {};
+  const raw = lines.slice(1, close).join('\n'), data = {}, itemLines = {};
   let list = null;
   for (let i = 1; i < close; i++) {
     const text = lines[i], line = i + 1;
@@ -118,6 +119,7 @@ export function parseFrontMatter(lines) {
       const parsed = scalar(item[2] ?? '');
       if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
       data[list].push(parsed.value);
+      itemLines[list].push(line);
       continue;
     }
     const pair = /^([^\s:#\-][^:]*?|"[^"]+"|'[^']+')\s*:(?:\s+(.*))?$/.exec(text);
@@ -126,7 +128,7 @@ export function parseFrontMatter(lines) {
     if (Object.hasOwn(data, key) || key === '__proto__') return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
     if (pair[2] === undefined || !pair[2].trim()) {
       const next = lines[i + 1];
-      if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; list = key; }
+      if (i + 1 < close && /^\s*-( |$)/.test(next)) { data[key] = []; itemLines[key] = []; list = key; }
       else { data[key] = null; list = null; }
       continue;
     }
@@ -135,13 +137,14 @@ export function parseFrontMatter(lines) {
     if (!parsed.ok) return { frontmatter: null, raw, bodyLine: close + 2, stop: line };
     data[key] = parsed.value;
   }
-  return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null };
+  return { frontmatter: data, raw: null, bodyLine: close + 2, stop: null, itemLines };
 }
 
 export const splitLines = text => text.replace(/^/, '').split('\n').map(l => l.endsWith('\r') ? l.slice(0, -1) : l);
 
 // One parsed file: its lines (all of them, numbered from 1), front matter and the links it holds.
-// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names.
+// Front matter links come from LINK_FIELDS values that are wikilinks; other values are names. A
+// link in a block list carries its item's line, one in a scalar or flow value its key's line.
 export function parseFile(text, { markdown = false } = {}) {
   const lines = splitLines(text);
   const fm = parseFrontMatter(lines);
@@ -151,10 +154,11 @@ export function parseFile(text, { markdown = false } = {}) {
     for (const field of LINK_FIELDS) {
       const raw = fm.frontmatter[field];
       const values = Array.isArray(raw) ? raw : raw === undefined || raw === null ? [] : [raw];
-      for (const v of values) {
+      const items = fm.itemLines[field];
+      values.forEach((v, i) => {
         const link = typeof v === 'string' ? parseWikilink(v.trim()) : null;
-        if (link) links.push({ ...link, field, line: frontMatterLine(lines, fm.bodyLine, field) });
+        if (link) links.push({ ...link, field, line: items ? items[i] : frontMatterLine(lines, fm.bodyLine, field) });
-      }
+      });
     }
   }
   return { lines: numbered, bodyLine: fm.bodyLine, frontmatter: fm.frontmatter, frontmatterRaw: fm.raw, stop: fm.stop, links };
@@ -195,7 +199,7 @@ export async function readLibrary({ repo, commit, mapping }) {
   for (const file of files) {
     const kept = [];
     for (const link of file.links) {
-      if (!link.file) continue;
+      if (!link.file || isAttachment(link)) continue;
       const target = resolve(file.path, link.file);
       if (!target.path) { problems.push(problem('R4', file.path, link.line)); continue; }
       const to = byPath.get(target.path);

packages/library/src/survey.mjs

@@ -1,6 +1,7 @@
 // `mosaic library survey <repo> [--mapping <file>]` (design 1.1, 10 step 7). It prints what a mapping
-// needs: folder names, file counts by extension, front matter key sets and link-form counts. It
+// needs: folder names, file counts by extension, front matter key sets, link-form counts and the
-// never prints file text, front matter values, file names or link targets. With a mapping it adds
+// count of body links to attachments. It never prints file text, front matter values, file names or
+// link targets. With a mapping it adds
 // scope counts, resolver step counts and R-problem counts by code. It reads the repo's HEAD commit
 // through the commit reader, never the working tree.
 import { readFileSync, realpathSync, statSync } from 'node:fs';
@@ -8,7 +9,7 @@ import { resolve as resolvePath } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { checkMapping } from './mapping.mjs';
 import { classify } from './scope.mjs';
-import { formOf } from './links.mjs';
+import { formOf, isAttachment } from './links.mjs';
 import { headCommit, listTree, parseFile, readable, readLibrary, readTexts } from './reader.mjs';
 import { LibraryError, RULES } from './problems.mjs';
 
@@ -37,7 +38,7 @@ export async function survey({ repo, commit, mapping }) {
   const texts = await readTexts(repo, markdown.filter(readable));
   const keySets = new Map(), forms = Object.fromEntries(FORMS.map(f => [f, 0]));
   const fields = { appears_in: { wikilink: 0, name: 0 }, relationships: { wikilink: 0, name: 0 } };
-  let noFrontMatter = 0, unparsed = 0;
+  let noFrontMatter = 0, unparsed = 0, attachments = 0;
   for (const text of texts.values()) {
     const file = parseFile(text, { markdown: true });
     if (file.stop !== null) unparsed++;
@@ -51,14 +52,14 @@ export async function survey({ repo, commit, mapping }) {
           fields[field][typeof v === 'string' && /^\s*!?\[\[[^\]]+\]\]\s*$/.test(v) ? 'wikilink' : 'name']++;
       }
     }
-    for (const link of file.links) if (!link.field) forms[formOf(link)]++;
+    for (const link of file.links) if (!link.field) { forms[formOf(link)]++; if (isAttachment(link)) attachments++; }
   }
   const result = {
     commit: head, entries: entries.length,
     folders: [...folders].sort(([a], [b]) => a < b ? -1 : 1).map(([name, counts]) => ({ name, ...counts })),
     markdown: { files: markdown.length, read: texts.size, noFrontMatter, unparsed },
     keySets: [...keySets].sort(([a, x], [b, y]) => y - x || (a < b ? -1 : 1)).map(([keys, count]) => ({ keys, count })),
-    forms, fields,
+    forms, attachments, fields,
   };
   if (mapping) {
     const scopes = { canon: 0, playground: 0, unmapped: 0 };
@@ -74,14 +75,18 @@ export async function survey({ repo, commit, mapping }) {
   return result;
 }
 
-// Folder names are quoted as JSON strings, so no name can carry a control sequence to the terminal.
+// Folder names are quoted as JSON strings, which escape C0 controls but leave DEL, C1 and bidi
+// controls raw. quote() escapes those too, so no name can carry a control sequence to the terminal
+// or reorder the line around it.
+const RAW_CONTROLS = /[\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/g;
+export const quote = name => JSON.stringify(name).replace(RAW_CONTROLS, c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`);
 export function formatSurvey(s) {
   const out = [`commit ${s.commit}`, `entries ${s.entries}`, 'folders (direct children: md canvas base other)'];
-  for (const f of s.folders) out.push(`  ${f.name === '' ? '(root)' : JSON.stringify(f.name)} ${f.md} ${f.canvas} ${f.base} ${f.other}`);
+  for (const f of s.folders) out.push(`  ${f.name === '' ? '(root)' : quote(f.name)} ${f.md} ${f.canvas} ${f.base} ${f.other}`);
   out.push(`markdown ${s.markdown.files} files, ${s.markdown.read} read, ${s.markdown.noFrontMatter} without front matter, ${s.markdown.unparsed} with front matter not fully parsed`);
   out.push('front matter key sets');
   for (const k of s.keySets) out.push(`  ${k.count} [${k.keys}]`);
-  out.push(`body links ${FORMS.map(f => `${f} ${s.forms[f]}`).join(', ')}`);
+  out.push(`body links ${FORMS.map(f => `${f} ${s.forms[f]}`).join(', ')}; attachment ${s.attachments}`);
   for (const [field, c] of Object.entries(s.fields)) out.push(`field ${field} wikilink ${c.wikilink}, name ${c.name}`);
   if (s.scopes) {
     out.push(`scopes canon ${s.scopes.canon}, playground ${s.scopes.playground}, unmapped ${s.scopes.unmapped}; indexed ${s.indexed}`);

packages/library/tests/links.test.mjs

@@ -1,6 +1,6 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { extractLinks, parseWikilink, resolver } from '../src/index.mjs';
+import { extractLinks, isAttachment, parseWikilink, resolver } from '../src/index.mjs';
 
 const PATHS = ['Book One/Characters/Mara Vell.md', 'Book One/Characters/Tomas.md', 'Book One/Places/Harbor Town.md',
   'Book One/Chapters/Chapter 01.md', 'Book One/Places/Notes.md', 'Book One/Characters/Notes.md', 'Mara Vell.md'];
@@ -48,3 +48,12 @@ test('wikilink forms, embeds, and links skipped inside code', () => {
   assert.deepEqual(extractLinks(lines).map(l => [l.file, l.line]), [['One', 1], ['Two', 1], ['Three', 1], ['Four', 1], ['', 1]]);
   assert.deepEqual(extractLinks(lines, { markdown: true }).filter(l => l.markdown).map(l => l.file), ['Five.md']);
 });
+
+test('a target with an extension other than .md, in any case, is an attachment', () => {
+  const att = text => isAttachment(parseWikilink(text));
+  for (const s of ['![[cover.png]]', '[[cover.PNG]]', '![[Art/map.pdf|Map]]', '[[Board.canvas]]', '![[clip.mp4#t=5]]'])
+    assert.equal(att(s), true, s);
+  for (const s of ['![[missing-note]]', '![[note.md]]', '[[Note.MD]]', '[[Chapter 1.5]]', '[[Mr. Smith]]', '[[A.b/Note]]', '[[#Self]]'])
+    assert.equal(att(s), false, s);
+  assert.equal(isAttachment({ file: 'pic.png', markdown: true }), false, 'markdown links are never followed anyway');
+});

packages/library/tests/reader.test.mjs

@@ -24,7 +24,7 @@ test('the fixture reads clean: every resolver step, appears_in and relationships
   assert.deepEqual(mara.frontmatter, { type: 'character', name: 'Mara Vell', appears_in: '[[Book One/Chapters/Chapter 01]]',
     relationships: ['[[Tomas]]', 'The Ferryman'] });
   assert.deepEqual(mara.links.map(l => [l.field, l.path, l.line]), [
-    ['appears_in', 'Book One/Chapters/Chapter 01.md', 4], ['relationships', 'Book One/Characters/Tomas.md', 5]]);
+    ['appears_in', 'Book One/Chapters/Chapter 01.md', 4], ['relationships', 'Book One/Characters/Tomas.md', 6]]);
   assert.deepEqual(file(result, 'Book One/Characters/Tomas.md').links.map(l => [l.field, l.path]),
     [['appears_in', 'Book One/Chapters/Chapter 01.md']]);
   assert.equal(mara.bodyLine, 9);
@@ -97,6 +97,26 @@ test('R4: a link to no file, to more than one, or into another repo is reported
   assert.deepEqual(file(result, 'Book One/Chapters/Links.md').links.map(l => l.path), ['Book One/Characters/Tomas.md']);
 });
 
+test('a block list link carries its item line, a scalar or flow link its key line, and problems follow', async t => {
+  const r = repo(t, { ...BASE, 'Book One/Characters/Ana.md': ['---', 'appears_in: ["[[Nobody]]"]', 'relationships:', '  - "[[Tomas]]"',
+    '  - Plain Name', '  - "[[Ghost]]"', '  - "[[What If]]"', '---', ''].join('\n') });
+  const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() });
+  assert.deepEqual(file(result, 'Book One/Characters/Ana.md').links.map(l => [l.field, l.path, l.line]), [
+    ['relationships', 'Book One/Characters/Tomas.md', 4], ['relationships', 'Book One/Playground/What If.md', 7]]);
+  assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Characters/Ana.md').map(p => [p.code, p.line]),
+    [['R4', 2], ['R4', 6], ['R5', 7]]);
+});
+
+test('an attachment link or embed is not followed and gives no problem; a note embed resolves', async t => {
+  const r = repo(t, { ...BASE, 'Book One/Places/note.md': 'n\n', 'Book One/Places/Shore.md': ['---',
+    'relationships:', '  - "![[portrait.JPG]]"', '---', '![[cover.png]] [[Board.canvas]] ![[map.pdf|Map]]',
+    '![[missing-note]] ![[note.md]] ![[Harbor Town]]', ''].join('\n') });
+  const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() });
+  assert.deepEqual(result.problems.filter(p => p.path === 'Book One/Places/Shore.md').map(p => [p.code, p.line]), [['R4', 6]]);
+  assert.deepEqual(file(result, 'Book One/Places/Shore.md').links.map(l => [l.path, l.embed]),
+    [['Book One/Places/note.md', true], ['Book One/Places/Harbor Town.md', true]]);
+});
+
 test('R5: a canon file that refers into the playground is reported, and the link still goes by path', async t => {
   const r = repo(t, { ...BASE, 'Book One/Places/Dock.md': 'See [[What If]].\n' });
   const result = await readLibrary({ repo: r.repo, commit: r.commit, mapping: mapping() });

packages/library/tests/survey.test.mjs

@@ -39,13 +39,31 @@ test('the survey prints folder names, counts, key sets and link-form counts, and
     '  1 [appears_in, name, type]',
     '  1 [name]',
     '  1 [type]',
-    'body links plain 3, label 1, heading 1, embed 0, self 0, markdown 1',
+    'body links plain 3, label 1, heading 1, embed 0, self 0, markdown 1; attachment 0',
     'field appears_in wikilink 2, name 0',
     'field relationships wikilink 1, name 1',
     ''].join('\n'));
   for (const s of NEVER) assert.ok(!out.stdout.includes(s), s);
 });
 
+test('folder names print DEL, C1 and bidi controls as escapes', async t => {
+  // One name per escaped range, with each range's first and last code point.
+  const names = ['A\u007fB', 'C\u0080D\u009fE', 'F\u200eG\u200fH', 'I\u202aJ\u202eK', 'L\u2066M\u2069N'];
+  const r = repo(t, Object.fromEntries(names.map(n => [`${n}/a.md`, `${MARKER}\n`])));
+  const out = await run(['survey', r.repo]);
+  assert.equal(out.code, 0, out.stderr);
+  assert.deepEqual(out.stdout.split('\n').slice(3, 8), [
+    '  "A\\u007fB" 1 0 0 0', '  "C\\u0080D\\u009fE" 1 0 0 0', '  "F\\u200eG\\u200fH" 1 0 0 0',
+    '  "I\\u202aJ\\u202eK" 1 0 0 0', '  "L\\u2066M\\u2069N" 1 0 0 0']);
+  assert.doesNotMatch(out.stdout, /[\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/);
+});
+
+test('an embed of an attachment counts as an embed and as an attachment', async t => {
+  const r = repo(t, { 'Book One/Chapters/A.md': ['![[cover.png]] ![[missing-note]] [[map.PDF]] [[Chapter 1.5]]', ''].join('\n') });
+  const out = await run(['survey', r.repo]);
+  assert.match(out.stdout, /^body links plain 2, label 0, heading 0, embed 2, self 0, markdown 0; attachment 2$/m);
+});
+
 test('with --mapping the survey adds scope, resolver step and problem counts', async t => {
   const r = repo(t);
   const out = await run(['survey', r.repo, '--mapping', join(FIXTURES, 'mapping.json')]);