diff --git a/packages/bus/tests/request-bootstrap.test.mjs b/packages/bus/tests/request-bootstrap.test.mjs index a61b30cf6..4376825e9 100644 --- a/packages/bus/tests/request-bootstrap.test.mjs +++ b/packages/bus/tests/request-bootstrap.test.mjs @@ -662,3 +662,163 @@ test('i2_a_completed_product_whose_marker_is_removed_is_not_ready_live_or_after_ const last = await f.boot(); assert.equal((await last.bus.admission({ business: 'demo', project: 'timer' })).ready, true); }); + +// ---- Fix round 1 for review r1 (#802): C1 retry confinement, C2 stale commits, C3 the serving fence ---------------- + +const listing = dir => readdirSync(dir, { recursive: true }).sort(); +const barePlain = (f) => { const repo = workingRepo(f.dataRoot, 'demo', 'timer'); mkdirSync(repo, { recursive: true }); git(repo, 'init', '-q', '--bare', '--template='); return repo; }; + +// C1. A working repository the first attempt found already there is not this plan's on any retry, whatever it holds. +// An empty ref list and an objects directory that is a link both used to pass the adoption check. +for (const [what, plant] of [ + ['an_empty_bare_repository_with_a_linked_objects_directory', (f, outside) => { + const repo = barePlain(f); rmSync(join(repo, 'objects'), { recursive: true }); symlinkSync(outside, join(repo, 'objects')); return repo; + }], + ['an_empty_bare_repository', f => barePlain(f)], +]) { + test(`r1c1_${what}_found_on_the_first_attempt_is_refused_on_every_retry_and_nothing_is_written_through_it`, async t => { + let outside, repo; + const f = await setup(t, { configureFactory: ({ root, f }) => { outside = external(root, 'external-objects'); repo = plant(f, outside); } }); + const before = listing(repo); + const first = await f.choose(); + assert.equal(first.phase, 'bootstrap'); assert.equal(first.wait.code, 'bootstrap-uncertain'); + for (let attempt = 0; attempt < 2; attempt++) { + const again = await f.first.bus.requestLifecycle.resume('demo', f.request); + assert.equal(again.phase, 'bootstrap', `retry ${attempt + 1} does not adopt it`); assert.equal(again.wait.code, 'bootstrap-uncertain'); + } + assert.deepEqual(readdirSync(outside), [], 'nothing was written behind the link'); + assert.deepEqual(listing(repo), before, 'the repository is kept as it was'); + assert.equal(git(repo, '--git-dir=' + repo, 'for-each-ref'), ''); + assert.equal(f.additions().length, 0); + assert.equal((await admission(f)).ready, false); + await f.first.bus.close(); + const next = await f.boot(); + const out = await stateOf(f, next.bus); + assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain', 'a restart does not adopt it either'); + assert.deepEqual(readdirSync(outside), []); + assert.equal((await next.bus.admission({ business: 'demo', project: 'timer' })).ready, false); + }); +} + +// C1. A repository this plan made is adopted on retry only while nothing inside it points elsewhere. +for (const [what, plant] of [ + ['an_alternates_file', (repo, outside) => writeFileSync(join(repo, 'objects/info/alternates'), outside + '\n')], + ['a_linked_objects_directory', (repo, outside) => { renameSync(join(repo, 'objects'), join(outside, 'objects')); symlinkSync(join(outside, 'objects'), join(repo, 'objects')); }], + ['a_link_inside_refs', (repo, outside) => symlinkSync(outside, join(repo, 'refs/heads/linked'))], + ['a_linked_head', (repo, outside) => { writeFileSync(join(outside, 'HEAD'), 'ref: refs/heads/main\n'); rmSync(join(repo, 'HEAD')); symlinkSync(join(outside, 'HEAD'), join(repo, 'HEAD')); }], +]) { + test(`r1c1_a_working_repository_this_plan_made_is_not_adopted_on_retry_once_it_holds_${what}`, async t => { + let armed = true, outside; + const f = await setup(t, { configureFactory: ({ root }) => { outside = external(root, 'external-objects'); }, barrier: step => { if (armed && step === 'working') { armed = false; throw new Error('killed'); } } }); + assert.equal((await f.choose()).wait.code, 'bootstrap-uncertain'); + await f.first.bus.close(); + const repo = workingRepo(f.f.dataRoot, 'demo', 'timer'); + plant(repo, outside); + const outsideBefore = listing(outside); + const next = await f.boot(); + const out = await stateOf(f, next.bus); + assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain'); + assert.deepEqual(listing(outside), outsideBefore, 'nothing was written behind it'); + assert.equal(f.additions().length, 0); + assert.equal((await next.bus.admission({ business: 'demo', project: 'timer' })).ready, false); + }); +} + +// C2. An organization write cannot land between the freshness check and the end of a protected commit: the lock is held +// through every git child, so the write is refused busy and the commit is made under the plan that was checked. +const attemptMove = (f, seen) => { try { moveOrganization(f); seen.landed = true; } catch (e) { seen.refused = e.code; } }; +const MOVED = 61; + +test('r1c2_an_organization_write_just_before_the_intent_commit_child_is_refused_and_no_stale_commit_is_made', async t => { + const f = await setup(t), seen = {}, base = f.writer.read('demo'); + const out = await withSpawnHook((binary, args) => { + if (!seen.at && binary === 'git' && args.includes(f.paths.records) && args.includes('commit') && args.includes('Edit spec/north-star.md')) { + seen.at = { commits: git(f.paths.records, 'rev-list', '--count', 'HEAD') }; + attemptMove(f, seen); + } + }, () => f.choose()); + assert.equal(seen.at.commits, '1', 'the write was attempted before the commit child'); + assert.equal(seen.refused, 'organization-busy'); assert.equal(seen.landed, undefined); + assert.notEqual(f.writer.read('demo').document.vars['tracker.pollSeconds'], MOVED); + assert.equal(f.writer.read('demo').revision, f.additions()[0].after, 'the organization is still the revision the plan produced'); + assert.equal(base.revision, f.additions()[0].before); + assert.equal(out.phase, 'discovery'); + assert.equal(git(f.paths.records, 'rev-list', '--count', 'HEAD'), '2'); + assert.equal(git(f.paths.records, 'log', '-1', '--format=%s'), 'Edit spec/north-star.md'); +}); + +test('r1c2_an_organization_write_while_the_records_init_child_runs_is_refused_and_no_stale_layout_commit_is_made', async t => { + const f = await setup(t), seen = {}; + const original = cp.spawn; + cp.spawn = function (binary, args, ...rest) { + const child = original.call(this, binary, args, ...rest); + if (!seen.at && binary === 'git' && args.includes(f.paths.records) && args.includes('init')) { seen.at = true; queueMicrotask(() => attemptMove(f, seen)); } + return child; + }; + syncBuiltinESMExports(); + let out; + try { out = await f.choose(); } finally { cp.spawn = original; syncBuiltinESMExports(); } + assert.equal(seen.at, true); + assert.equal(seen.refused, 'organization-busy'); assert.equal(seen.landed, undefined); + assert.equal(f.writer.read('demo').revision, f.additions()[0].after); + assert.equal(out.phase, 'discovery'); + assert.equal(git(f.paths.records, 'rev-list', '--count', 'HEAD'), '2'); + assert.equal(git(f.paths.records, 'log', '-1', '--format=%s', 'HEAD~1'), 'Initialize project layout v1'); +}); + +test('r1c2_the_lock_is_free_again_after_each_protected_effect_so_the_next_organization_write_lands', async t => { + const f = await setup(t); + assert.equal((await f.choose()).phase, 'discovery'); + moveOrganization(f); + assert.equal(f.writer.read('demo').document.vars['tracker.pollSeconds'], MOVED); +}); + +// C3. The broker serves a product only when the complete marker and the committed activate receipt both exist, at the +// moment it starts serving and afterwards, including when the receipt cannot be written. +test('r1c3_the_broker_is_reloaded_with_the_project_only_after_the_activate_receipt_is_committed', async t => { + const f = await setup(t), broker = f.first.bus.broker, original = broker.reloadBusiness, calls = []; + broker.reloadBusiness = function (...args) { + calls.push({ receipts: activations(f).length, marker: marked(f), admitted: Object.hasOwn(args[1].projects, 'timer') }); + const result = original.apply(this, args); + queueMicrotask(() => { calls.at(-1).after = { receipts: activations(f).length, made: Boolean(f.call('human', 'project.create', { name: 'Right after', repo: 'timer' }).id) }; }); + return result; + }; + let out; + try { out = await f.choose(); } finally { broker.reloadBusiness = original; } + assert.equal(out.phase, 'discovery'); + const admitting = calls.filter(c => c.admitted); + assert.equal(admitting.length, 1); + assert.equal(admitting[0].receipts, 1, 'the receipt was committed before the broker started serving the project'); + assert.equal(admitting[0].marker, 'complete'); + assert.deepEqual(admitting[0].after, { receipts: 1, made: true }); +}); + +test('r1c3_a_failed_activate_receipt_write_leaves_the_project_unserved_until_a_fault_free_retry_completes', async t => { + const f = await setup(t), original = DatabaseSync.prototype.prepare; + let faults = 0; + DatabaseSync.prototype.prepare = function (sql, ...rest) { + const statement = original.call(this, sql, ...rest); + if (sql.startsWith('INSERT INTO request_effects(')) { + const run = statement.run; + statement.run = function (...args) { + // at, version, business, request, op, kind, digest, stage, result, code + if (!faults && String(args[4]).endsWith(':activate') && args[7] === 'committed') { faults++; throw Object.assign(new Error('injected activation receipt I/O error'), { code: 'SQLITE_IOERR' }); } + return run.apply(this, args); + }; + } + return statement; + }; + let out; + try { out = await f.choose(); } finally { DatabaseSync.prototype.prepare = original; } + assert.equal(faults, 1, 'the fault fired on the activate receipt'); + assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain'); + assert.equal(marked(f), 'complete'); assert.deepEqual(activations(f), []); + const pending = await admission(f); + assert.equal(pending.ready, false); assert.equal(pending.code, 'project-pending'); + assert.throws(() => f.call('human', 'project.create', { name: 'After failed receipt', repo: 'timer' }), 'the live broker does not serve an unreceipted project'); + assert.throws(() => f.call('human', 'project.create', { name: 'Still after', repo: 'timer' })); + const again = await f.first.bus.requestLifecycle.resume('demo', f.request); + assert.equal(again.phase, 'discovery'); + assert.equal(activations(f).length, 1); assert.equal((await admission(f)).ready, true); + assert.ok(f.call('human', 'project.create', { name: 'After the retry', repo: 'timer' }).id); +}); diff --git a/packages/business/src/organization.mjs b/packages/business/src/organization.mjs index cf11c2857..fdf0b2b52 100644 --- a/packages/business/src/organization.mjs +++ b/packages/business/src/organization.mjs @@ -156,7 +156,9 @@ export function organizationWriter({ dir, rolesDir, barrier = () => {} }) { } } - function withLock(id, fn) { + // `through` lets `fn` return a promise: the lock is then released when that settles, so a protected effect that is + // still running (a git child) keeps every other organization writer out. A synchronous return releases at once. + function withLock(id, fn, through = false) { const parent = join(dir, "businesses"); directory(evidence); const home = join(evidence, id); @@ -187,12 +189,20 @@ export function organizationWriter({ dir, rolesDir, barrier = () => {} }) { } fail("organization-uncertain", "stale writer lock: confirm the old writer is gone before removing it"); } - try { - return fn({ home, parent }); - } finally { + const release = () => { rmSync(lock, { recursive: true }); syncDir(home); + }; + let out; + try { + out = fn({ home, parent }); + } catch (e) { + release(); + throw e; } + if (through && out !== null && typeof out === "object" && typeof out.then === "function") return out.finally(release); + release(); + return out; } function log(id, event) { @@ -353,6 +363,18 @@ export function organizationWriter({ dir, rolesDir, barrier = () => {} }) { }); } + // Like `hold`, but `fn` may return a promise, and the lock stays held until it settles. A protected effect + // (a commit made of several git children) keeps the organization still from its freshness check to its last child, + // and any other organization write meanwhile is refused `organization-busy`. The lock is released on a throw too. + function holdThrough(id, fn) { + requireId(id, "organization id"); + return withLock(id, () => { + recover(id); + const { document, revision } = read(id); + return fn({ document, revision }); + }, true); + } + // The committed addition of this request and plan, if there is one. A prepared receipt whose // `after` is the file's current hash counts: the rename landed and the commit line did not. function findAddition(id, { request, plan_digest, project }) { @@ -393,5 +415,5 @@ export function organizationWriter({ dir, rolesDir, barrier = () => {} }) { }); } - return { logFile, read, readRaw, write, addProject, hold, findAddition, record, recover: id => withLock(id, () => recover(id)) }; + return { logFile, read, readRaw, write, addProject, hold, holdThrough, findAddition, record, recover: id => withLock(id, () => recover(id)) }; } diff --git a/packages/seat/src/working.mjs b/packages/seat/src/working.mjs index 38961a18e..dc56cd1c2 100644 --- a/packages/seat/src/working.mjs +++ b/packages/seat/src/working.mjs @@ -81,7 +81,7 @@ // returned: it can carry the URL. import { randomBytes } from "node:crypto"; -import { closeSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { closeSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join, relative, sep } from "node:path"; import { gitEnv, GIT_LIMITS, runGit } from "./checkout.mjs"; import { withoutProcessing } from '../../processing-control/src/effects.mjs'; @@ -188,6 +188,35 @@ function ancestors(dataRoot, repo, make = false) { return true; } +// A bare repository this code made holds regular files and directories and nothing that points elsewhere. A link +// anywhere inside it, an alternates file or a commondir would send a git command out of the repository (an object +// read or write, a ref or a config lookup behind the link), so any of them is refused before a command runs. +// This is a read-only walk of lstat results; it never follows a link. It is for the host's own initializer, and +// is not in `ensure` because `push` shares that and must not walk a whole repository on every call. +export function refuseIndirection(repo) { + const bad = (path, why) => refuse("working-repo-invalid", `${path} ${why}`); + const top = look(repo); + if (top === null || top.isSymbolicLink() || !top.isDirectory()) bad(repo, "isn't a real directory"); + for (const name of ["objects/info/alternates", "objects/info/http-alternates", "commondir"]) { + if (look(join(repo, name)) !== null) bad(join(repo, name), "points the repository at another one"); + } + const walk = (dir) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (e) { + bad(dir, `can't be read: ${e.code}`); + } + for (const entry of entries) { + const path = join(dir, entry.name); + if (entry.isSymbolicLink()) bad(path, "is a link"); + if (entry.isDirectory()) walk(path); + else if (!entry.isFile()) bad(path, "isn't a regular file or directory"); + } + }; + walk(repo); +} + // The effect runs as it is unless the caller gives a wrapper. initializeWorking's caller (the bootstrap // executor) passes one that checks, under the organization lock, that its plan is still fresh and starts the // effect in the same synchronous run, so a write here never begins under a plan that went stale while this @@ -307,7 +336,8 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl // Host-only (lifecycle Task 2, #797): makes the project's working repository and seeds main from // the root's checked-out commit, under the same lock and through the same private seed a push // uses. It makes no work ref, takes no bundle and has no remote to push to. A second call is the -// same seed: it moves nothing the first call left. Resolves { repo, commit } with main's commit. +// same seed: it moves nothing the first call left. A repository holding a link, an alternates file or a +// commondir is refused (working-repo-invalid) before the seed runs a command in it. Resolves { repo, commit } with main's commit. // `around(effect)` wraps every write (the directories, the init and its rename, the seed's fetch and // ref update); it must run `effect` synchronously and return what it returns. export function initializeWorking({ dataRoot, business, project, root, env = process.env, limits = GIT_LIMITS, around = direct }) { @@ -316,6 +346,7 @@ export function initializeWorking({ dataRoot, business, project, root, env = pro const g = bound(env, limits); return locked(repo, async () => { await ensure(dataRoot, repo, g, around); + refuseIndirection(repo); await seed(repo, root, g, around); const commit = await tip(repo, "refs/heads/main", g); if (!commit) refuse("working-repo-invalid", "main has no commit after the seed");