// Row 154 (#1648, Q168): the imported editor source and the build. third_party/editor/ is upstream source // copied byte for byte; this fails if one byte of it changes without a recorded local change, if a file is // added or removed, or if the built bundle is not the build of that source. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { spawnSync } from 'node:child_process'; import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; const webui = join(dirname(fileURLToPath(import.meta.url)), '..'); const imported = join(webui, 'third_party/editor'); const sha = buf => createHash('sha256').update(buf).digest('hex'); const gitBlob = buf => createHash('sha1').update(Buffer.concat([Buffer.from(`blob ${buf.length}\0`), buf])).digest('hex'); const provenance = JSON.parse(readFileSync(join(imported, 'PROVENANCE.json'), 'utf8')); const build = JSON.parse(readFileSync(join(webui, 'editor/BUILD.json'), 'utf8')); const filesOnDisk = () => readdirSync(imported, { withFileTypes: true, recursive: true }) .filter(e => e.isFile()).map(e => join(e.parentPath, e.name).slice(imported.length + 1)).filter(p => p !== 'PROVENANCE.json').sort(); const treeOf = entries => sha(entries.sort((a, b) => (a[0] < b[0] ? -1 : 1)).map(([path, h]) => `${h} ${path}\n`).join('')); test('the imported source is the tagged upstream source plus the recorded local changes, and nothing else', () => { assert.equal(provenance.schema, 1); const changes = new Map(provenance.localChanges.map(c => [c.path, c])); for (const c of changes.values()) assert.ok(c.path && /^[0-9a-f]{64}$/.test(c.sha256) && c.reason, `a local change names its path, its new sha256 and why: ${JSON.stringify(c)}`); // 1. The record names every file and the disk has exactly those. const recorded = new Map([['NOTICE', provenance.notice.sha256]]); for (const p of provenance.packages) { assert.match(p.commit, /^[0-9a-f]{40}$/, `${p.name} commit`); assert.match(p.upstreamTree, /^[0-9a-f]{40}$/, `${p.name} upstream tree`); assert.ok(p.files[p.license], `${p.name} ships its license file ${p.license}`); assert.ok(p.files['package.json'], `${p.name} ships its package.json`); for (const [path, f] of Object.entries(p.files)) recorded.set(`${p.name}/${path}`, f.sha256); } assert.deepEqual(filesOnDisk(), [...recorded.keys()].sort(), 'files on disk against files recorded'); // 2. Each file is the recorded bytes: the upstream bytes, or the recorded local change. const expected = []; for (const p of provenance.packages) { for (const [path, f] of Object.entries(p.files)) { const key = `${p.name}/${path}`, buf = readFileSync(join(imported, key)); const change = changes.get(key); if (change) assert.equal(sha(buf), change.sha256, `${key} differs from its recorded local change`); else { assert.equal(sha(buf), f.sha256, `${key} differs from upstream and has no recorded local change`); assert.equal(gitBlob(buf), f.blob, `${key} is not the git blob ${f.blob} of ${p.tag}`); assert.equal(buf.length, f.bytes, key); } expected.push([key, change ? change.sha256 : f.sha256]); } } for (const key of changes.keys()) assert.ok(recorded.has(key), `local change for ${key}, which is not an imported file`); const notice = readFileSync(join(imported, 'NOTICE')); assert.equal(sha(notice), provenance.notice.sha256, 'NOTICE'); expected.push(['NOTICE', provenance.notice.sha256]); // 3. The tree hash: from the record, and from the bytes on disk. assert.equal(treeOf(expected), provenance.tree.sha256, 'tree hash from the record'); const fromDisk = filesOnDisk().map(p => [p, sha(readFileSync(join(imported, p)))]); assert.equal(fromDisk.length, provenance.tree.files); assert.equal(treeOf(fromDisk), provenance.tree.sha256, 'tree hash from the files on disk'); }); test('every package names its origin and license, and NOTICE lists each', () => { const notice = readFileSync(join(imported, 'NOTICE'), 'utf8'); for (const p of provenance.packages) { assert.match(p.origin, /^https:\/\//, p.name); assert.ok(p.spdx && p.copyright.length, `${p.name} license id and copyright line`); assert.ok(notice.includes(`${p.name} ${p.version}`) && notice.includes(p.commit) && notice.includes(`${p.name}/${p.license}`), `NOTICE lists ${p.name}`); for (const line of p.copyright) assert.ok(notice.includes(line), `NOTICE carries "${line}"`); assert.ok(readFileSync(join(imported, p.name, p.license), 'utf8').includes(p.copyright[0]), `${p.name} license file carries its copyright`); } assert.equal(new Set(provenance.packages.map(p => p.name)).size, provenance.packages.length); }); test('the build reads only imported source and the editor\'s own files; the bundle is the recorded build', () => { const imp = new Set(provenance.packages.flatMap(p => Object.keys(p.files).map(f => `third_party/editor/${p.name}/${f}`))); for (const input of build.inputs) { if (input.startsWith('third_party/editor/')) assert.ok(imp.has(input), `${input} is read by the build but is not in PROVENANCE.json`); else assert.match(input, /^editor\/[a-z]+\.mjs$/, input); assert.ok(!/(^|\/)(dist|node_modules)\//.test(input), `${input}: no prebuilt output goes in`); } for (const input of build.inputs) readFileSync(join(webui, input)); // every input exists for (const [path, out] of Object.entries(build.outputs)) { const buf = readFileSync(join(webui, path)); assert.equal(sha(buf), out.sha256, `${path} is not the recorded build`); assert.equal(buf.length, out.bytes, path); } const bundle = readFileSync(join(webui, 'src/public/editor.js'), 'utf8'); assert.doesNotMatch(bundle, /sourceMappingURL/); assert.match(bundle, /^\(\(\) => \{/, 'one unminified IIFE'); assert.ok(bundle.split('\n').length > 5000, 'unminified'); }); // A fake install: node_modules/esbuild/package.json with a version, and the platform binary with the given bytes. const fakeInstall = (version, bytes) => { const dir = mkdtempSync(join(tmpdir(), 'editor-toolchain-')); mkdirSync(join(dir, 'node_modules/esbuild'), { recursive: true }); writeFileSync(join(dir, 'node_modules/esbuild/package.json'), JSON.stringify({ name: 'esbuild', version })); if (bytes !== undefined) { const bin = join(dir, `node_modules/@esbuild/${process.platform}-${process.arch}/bin`); mkdirSync(bin, { recursive: true }); writeFileSync(join(bin, 'esbuild'), bytes); writeFileSync(join(dir, `node_modules/@esbuild/${process.platform}-${process.arch}/package.json`), '{}'); } return { dir, from: pathToFileURL(join(dir, 'x.mjs')) }; }; test('MOSAIC_REQUIRE_EDITOR_BUILD: a missing or wrong esbuild skips without it and fails with it; a wrong binary always fails', async () => { const { ESBUILD, toolchain, required, verdict } = await import('../editor/toolchain.mjs'); const cases = []; try { for (const [label, install, kind] of [ ['not installed', fakeInstall('0.0.0'), null], ['another version', fakeInstall('0.28.1', 'x'), 'absent'], ['the pinned version, another binary', fakeInstall(ESBUILD, 'not the esbuild binary'), 'refused'], ['the pinned version, no binary', fakeInstall(ESBUILD), 'refused'], ]) { cases.push(install.dir); if (label === 'not installed') rmSync(join(install.dir, 'node_modules'), { recursive: true }); const tool = toolchain(install.from); assert.equal(tool.ok, false, label); assert.equal(tool.kind, kind ?? 'absent', label); assert.ok(tool.reason.length > 20, `${label}: the reason is printed`); assert.equal(verdict(tool, false), kind === 'refused' ? 'fail' : 'skip', `${label} without the switch`); assert.equal(verdict(tool, true), 'fail', `${label} with the switch`); } // The switch itself: "1" requires; unset, empty and "0" do not; anything else throws. assert.equal(required({}), false); assert.equal(required({ MOSAIC_REQUIRE_EDITOR_BUILD: '' }), false); assert.equal(required({ MOSAIC_REQUIRE_EDITOR_BUILD: '0' }), false); assert.equal(required({ MOSAIC_REQUIRE_EDITOR_BUILD: '1' }), true); for (const v of ['true', 'yes', ' 1', '2']) assert.throws(() => required({ MOSAIC_REQUIRE_EDITOR_BUILD: v }), /MOSAIC_REQUIRE_EDITOR_BUILD/); // ESBUILD_BINARY_PATH replaces the binary esbuild runs, so it is refused whatever the install holds. const fake = fakeInstall(ESBUILD, 'x'); cases.push(fake.dir); process.env.ESBUILD_BINARY_PATH = '/bin/true'; try { assert.equal(toolchain(fake.from).kind, 'refused'); } finally { delete process.env.ESBUILD_BINARY_PATH; } } finally { for (const dir of cases) rmSync(dir, { recursive: true, force: true }); } }); test('the toolchain check, run as a program, exits non-zero without esbuild when the build is required', () => { // A copy of the check in a directory with no esbuild above it, so it cannot find one. const dir = mkdtempSync(join(tmpdir(), 'editor-toolchain-cli-')); try { const copy = join(dir, 'toolchain.mjs'); copyFileSync(join(webui, 'editor/toolchain.mjs'), copy); const run = switchValue => { const env = { ...process.env }; delete env.MOSAIC_REQUIRE_EDITOR_BUILD; delete env.NODE_PATH; delete env.ESBUILD_BINARY_PATH; if (switchValue !== undefined) env.MOSAIC_REQUIRE_EDITOR_BUILD = switchValue; return spawnSync(process.execPath, [copy], { encoding: 'utf8', env, cwd: dir }); }; const routine = run(); assert.equal(routine.status, 0, routine.stderr); assert.match(routine.stdout, /^skip: esbuild /); assert.equal(run('0').status, 0); const required = run('1'); assert.equal(required.status, 3); assert.match(required.stderr, /^missing: esbuild /); assert.equal(required.stdout, ''); assert.notEqual(run('true').status, 0, 'a mistyped value is a failure, not a silent skip'); } finally { rmSync(dir, { recursive: true, force: true }); } }); test('rebuilding from the source gives the same bytes (needs the pinned esbuild; MOSAIC_REQUIRE_EDITOR_BUILD=1 makes its absence a failure)', async t => { const { toolchain, required, verdict } = await import('../editor/toolchain.mjs'); const tool = toolchain(), action = verdict(tool, required()); if (action === 'skip') { t.skip(`${tool.reason} (set MOSAIC_REQUIRE_EDITOR_BUILD=1 to fail instead of skip)`); return; } assert.equal(action, 'run', tool.reason); const { run } = await import('../editor/build.mjs'); const out = mkdtempSync(join(tmpdir(), 'editor-rebuild-')); try { const info = await run(out, join(out, 'BUILD.json')); assert.equal(info.esbuild, build.esbuild); assert.deepEqual(info.inputs, build.inputs); for (const path of Object.keys(build.outputs)) assert.equal(sha(readFileSync(join(out, path.split('/').pop()))), build.outputs[path].sha256, `${path} rebuilt`); assert.equal(readFileSync(join(out, 'BUILD.json'), 'utf8'), readFileSync(join(webui, 'editor/BUILD.json'), 'utf8')); } finally { rmSync(out, { recursive: true, force: true }); } });