mosaicstack/stack
25
0 Fork 1
Code Issues 404 Pull Requests 13 Packages Projects Releases 15 Wiki Activity

fix(bus): confine the working repo, serialize freshness with protected writes, fence readiness on publication (#802)

Browse Source
This commit is contained in:
jason.woltje committed 2026-11-04 10:38:54 -06:00
1 parent 1031686140
commit 7f5bbbf747
9 files changed
+414 -71
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/bus/src/request-bootstrap.mjs
+76 -32
View File
@@ -6,25 +6,28 @@
//
// reserve re-verify the plan against the world (#800), then reserve the product paths and a marker
// seed clone the source commit into a claimed temporary checkout, drop its remote, commit the one project-file change
// working seed the working repository from that commit
// working confine the working path, then seed the working repository from that commit
// mapping publish the one organization mapping through the private exact-add writer
// records initialize the records repository under that mapping
// intent save the original displayed intent as the owner's edit of spec/north-star.md
// chat associate the chat project
// activate admit the project to the running broker, then mark the product complete
// activate mark the product complete, then admit the project to the running broker
//
// Every inspection compares deterministic paths, the stored commits and exact bytes. What it cannot prove
// is kept as it is and reported as `bootstrap-uncertain`; nothing unrelated is removed and no commit is rewound.
// Nothing is ever deleted recursively, and every write goes through an exclusively created file and a rename,
// inside directories that are verified not to be links, so a link in a source checkout cannot redirect a write.
// Before every effect the organization revision and the chat association are checked again (#800).
// Before every effect the organization revision and the chat association are checked again (#800), and the check
// and the start of the effect are one synchronous run under the organization lock (`guarded`). A project is ready
// only with both a complete marker and a committed activate receipt, so the marker goes first and nothing is
// admitted until it is published.
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { closeSync, constants, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync, writeSync } from 'node:fs';
import { dirname, join, sep } from 'node:path';
import { dirname, join, relative, sep } from 'node:path';
import { digest, refuse, validatePlan } from './request-store.mjs';
import { gitEnv, runGit } from '../../seat/src/checkout.mjs';
import { initializeWorking } from '../../seat/src/working.mjs';
import { initializeWorking, workingRepo } from '../../seat/src/working.mjs';
import { initProject, writeHumanFile } from '../../project/src/index.mjs';
import { SAFE, identity } from '../../project/src/git.mjs';
import { loadProject } from '../../business/src/project.mjs';
@@ -139,16 +142,23 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
const publish = (file, temp, bytes) => stage(file, temp, bytes)();
// #800 before every effect, not only the first: the organization is still at the revision this plan expects (the
// confirmed one before the mapping, this plan's own post-add one after it), the mapping is the plan's, and the chat
// association is not another's. Synchronous and under the organization lock, so no write lands between the check
// and the effect after it. A mismatch keeps what is done and performs nothing further.
const fresh = () => writer.hold(business, ({ revision, document }) => {
// association is not another's. `fresh` runs under the organization lock, so a bare `fresh()` proves the world
// matched at that instant and nothing more: the lock is released when it returns. What keeps a write from landing
// under a stale plan is `guarded(effect)`: the caller has finished every asynchronous read the effect depends on,
// and the check and the start of the effect run in one synchronous run with the lock held, so no organization
// write lands between them. An effect that is a git child is started inside the lock; the child's own running
// time is not covered, and the next step's check catches a change made meanwhile. A mismatch keeps what is done
// and performs nothing further. `effect` must be synchronous and must not take the organization lock itself.
const fresh = effect => writer.hold(business, ({ revision, document }) => {
const stale = () => refuse('request-plan-stale');
if (document.id !== business || document.human !== owner) stale();
const added = writer.findAddition(business, { request, plan_digest: planDigest, project });
if (added) { if (revision !== added.after || canonicalJson(document.projects?.[project]) !== canonicalJson(entry)) stale(); }
else if (revision !== plan.organization_revision || Object.hasOwn(document.projects ?? {}, project)) stale();
if (services.chat.state({ business, project, name: plan.name, id: chatId }) === 'conflict') stale();
effect?.();
});
const guarded = effect => { let started; fresh(() => { started = effect(); }); return started; };
const git = (cwd, args, extra = {}, file = false) => runGit([...SAFE, ...(file ? ['-c', 'protocol.file.allow=always'] : []), ...(cwd ? ['-C', cwd] : []), ...args],
{ ...gitEnv(), ...(cwd ? { GIT_CEILING_DIRECTORIES: dirname(cwd) } : {}), ...extra });
@@ -160,6 +170,7 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
const op = base(name), last = journal.effects(business, request).filter(r => r.op === op).at(-1);
if (last?.stage === 'committed') return last.result;
if (last?.stage === 'refused') refuse(last.code ?? 'bootstrap-refused');
// An early refusal, before anything is recorded. The effects below each check again through `guarded`.
if (name !== 'reserve') fresh();
const header = { version: 1, business, request, op, kind: 'bootstrap', digest: planDigest };
journal.prepareEffect({ ...header, stage: 'prepared', result: null, code: null });
@@ -226,12 +237,11 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
// Cloned, or seeded but not yet committed. The only change allowed is the exact project file.
const onlyFile = status === '' || (/^( M|M ) \.mosaic\/project\.json$/.test(status) && readFileSync(seedFile, 'utf8') === seedBytes);
if (!onlyFile) uncertain();
fresh();
if (remotes === 'origin') await out(code, ['remote', 'remove', 'origin']);
if (status === '') publish(seedFile, seedTemp, seedBytes);
await out(code, ['add', '--', PROJECT_FILE]);
fresh();
await out(code, ['commit', '-q', '-m', `Bootstrap ${project} from ${plan.source.project}`], identity(human()));
const who = identity(human());
if (remotes === 'origin') await guarded(() => out(code, ['remote', 'remove', 'origin']));
if (status === '') guarded(() => publish(seedFile, seedTemp, seedBytes));
await guarded(() => out(code, ['add', '--', PROJECT_FILE]));
await guarded(() => out(code, ['commit', '-q', '-m', `Bootstrap ${project} from ${plan.source.project}`], who));
return { commit: await out(code, ['rev-parse', 'HEAD']), base: plan.source.commit };
}
const names = await tryOut(code, ['diff', '--name-status', `${head}~1`, head]);
@@ -256,12 +266,14 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
if (held && (!claim || held.isSymbolicLink() || !held.isDirectory())) uncertain();
if (!held) {
try {
if (!claim) writeFileSync(claimFile, claimBytes, { flag: 'wx', mode: 0o600 });
mkdirSync(tmp, { mode: 0o700 });
guarded(() => {
if (!claim) writeFileSync(claimFile, claimBytes, { flag: 'wx', mode: 0o600 });
mkdirSync(tmp, { mode: 0o700 });
});
} catch (e) { if (e.code === 'EEXIST') uncertain(); throw e; }
}
if (!held || readdirSync(tmp).length === 0) {
await out(null, ['clone', '-q', '--no-hardlinks', '--template=', '--', source.root, tmp], {}, true);
await guarded(() => out(null, ['clone', '-q', '--no-hardlinks', '--template=', '--', source.root, tmp], {}, true));
}
await services.barrier?.('clone');
// A clean clone of exactly the source commit, with at most the remote the clone itself added.
@@ -270,10 +282,11 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
if (head === null || remotes === null || status !== '' || !['', 'origin'].includes(remotes)) uncertain();
if (head !== plan.source.commit) refuse('request-plan-stale');
if (metadataProblem(tmp)) refuse('bootstrap-metadata-refused');
if (remotes === 'origin') await out(tmp, ['remote', 'remove', 'origin']);
fresh();
if (exists(here.root) || !confined(product)) uncertain();
renameSync(tmp, here.root);
if (remotes === 'origin') await guarded(() => out(tmp, ['remote', 'remove', 'origin']));
guarded(() => {
if (exists(here.root) || !confined(product)) uncertain();
renameSync(tmp, here.root);
});
await services.barrier?.('cloned');
}
const seed = async ({ recovering }) => {
@@ -282,13 +295,37 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
if (!code.isDirectory() || code.isSymbolicLink() || !markerOwns()) uncertain();
} else await clone(recovering);
const seededAt = await seeded();
dropClaim();
guarded(dropClaim);
return seededAt;
};
// 3. working. The working repository's main follows the checkout's HEAD, which is the seed commit.
const working = async () => {
const seedCommit = done('seed').commit, w = await initializeWorking({ dataRoot, business, project, root: here.root });
// 3. working. The working repository's main follows the checkout's HEAD, which is the seed commit. Nothing is made
// while a directory between the data root and the repository is a link or a file (N1: `working` linked to a
// sibling directory would take the repository and every file in it out of the data root). A repository that is
// already there is this plan's only on a retry, and only if it is a bare repository holding no ref but root and
// main, both at the seed commit; on a first attempt it is someone else's. Either way it is kept as it is.
const workingPath = workingRepo(dataRoot, business, project);
const workingExists = () => {
let at = dataRoot;
for (const name of relative(dataRoot, workingPath).split(sep)) {
at = join(at, name);
const st = exists(at);
if (!st) return false;
if (st.isSymbolicLink() || !st.isDirectory()) uncertain();
}
return true;
};
const working = async ({ recovering }) => {
const seedCommit = done('seed').commit;
if (workingExists()) {
if (!recovering) uncertain();
const at = [`--git-dir=${workingPath}`];
const refs = await tryOut(null, [...at, 'for-each-ref', '--format=%(refname) %(objectname)']);
const bare = await tryOut(null, [...at, 'rev-parse', '--is-bare-repository']);
const expected = new Set(['refs/heads/root', 'refs/heads/main'].map(ref => `${ref} ${seedCommit}`));
if (bare !== 'true' || refs === null || refs.split('\n').filter(Boolean).some(line => !expected.has(line))) uncertain();
}
const w = await initializeWorking({ dataRoot, business, project, root: here.root, around: guarded });
if (w.commit !== seedCommit) uncertain();
return { commit: w.commit };
};
@@ -312,7 +349,11 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
const records = async () => {
const repo = here.repo, st = exists(repo);
if (!st || !st.isDirectory() || st.isSymbolicLink()) uncertain();
if (readdirSync(repo).length === 0) { const made = await initProject(context(), { author: human() }); return { commit: made.commit }; }
if (readdirSync(repo).length === 0) {
const where = context(), author = human();
const made = await guarded(() => initProject(where, { author }));
return { commit: made.commit };
}
const count = await tryOut(repo, ['rev-list', '--count', 'HEAD']);
const head = await tryOut(repo, ['log', '-1', '--format=%H|%an|%s', 'HEAD']);
if (count !== '1' || head === null) uncertain();
@@ -326,7 +367,8 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
const repo = here.repo, init = done('records').commit;
const count = await tryOut(repo, ['rev-list', '--count', 'HEAD']), head = await tryOut(repo, ['rev-parse', 'HEAD']);
if (count === '1' && head === init) {
const saved = await writeHumanFile({ path: INTENT_PATH, content: plan.intent.text, base: init, ...context() });
// Every read above and inside the writer is finished before `around` checks and publishes the file.
const saved = await writeHumanFile({ path: INTENT_PATH, content: plan.intent.text, base: init, around: guarded, ...context() });
return { commit: saved.commit };
}
const line = await tryOut(repo, ['log', '-1', '--format=%P|%an|%s', 'HEAD']);
@@ -338,7 +380,8 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
return { commit: head };
};
// 7. chat. The association is one row; a row that is not this plan's is a conflict, never replaced.
// 7. chat. The association is one row; a row that is not this plan's is a conflict, never replaced. No await sits
// between the step's own check and the association.
const chat = () => {
const args = { business, project, name: plan.name, id: chatId }, state = services.chat.state(args);
if (state === 'conflict') uncertain();
@@ -346,15 +389,16 @@ export async function executeBootstrap({ binding, owner, journal, services }) {
return { chat_project: chatId };
};
// 8. activate. Synchronous from the activation to the receipt, so the broker and the marker agree.
// 8. activate. The complete marker is published first and the broker admits the project second, so a project is
// never admitted without it: a failed marker write ends the step before the broker is touched. A complete marker
// with no committed receipt admits nothing, since readiness at runtime and at startup needs both, and a revision
// that moved before the admission leaves the marker as it is and the project pending.
const activate = async () => {
await services.barrier?.('activate');
const mapped = done('mapping');
// The marker is staged before the broker admits anything, so a refusal leaves no half-admitted product.
const place = stage(marker, `${marker}.tmp`, markerBytes({ state: 'complete', request, plan_digest: planDigest, project }));
guarded(() => publish(marker, `${marker}.tmp`, markerBytes({ state: 'complete', request, plan_digest: planDigest, project })));
try { services.activate({ business, project, receipt, organization_revision: mapped.organization_revision }); }
catch (e) { if (e?.code === 'bootstrap-revision-stale') refuse('request-plan-stale'); throw e; }
place();
return { project, organization_revision: mapped.organization_revision };
};
packages/bus/src/runtime.mjs
+25 -10
View File
@@ -283,15 +283,20 @@ export async function startBroker({
let store, credentials, server, adapter, processing, syncTimer = null, syncPass = null, accessTimer = null;
try {
// Lifecycle Task 2 (#797): a product whose mapping is published but not yet activated is not part of
// this process. The request journal says which are activated; every other managed entry waits for the
// bootstrap executor's activateManagedProject, so the broker, the Credentials roots and the boot copy
// never see a half-built product. `businesses` becomes this process's own copy.
// this process. A managed product is part of it only with both a committed activate receipt in the
// request journal and a complete marker on disk (#802: a receipt alone, with the marker gone or never
// published, is not enough); every other managed entry waits for the bootstrap executor's
// activateManagedProject, so the broker, the Credentials roots and the boot copy never see a
// half-built product. `businesses` becomes this process's own copy.
store = new Store(dataRoot);
const requestJournal = createRequestStore(store);
const activatedProjects = business => new Set(requestJournal.committed(business, 'bootstrap')
.filter(r => r.op.endsWith(':activate') && typeof r.result?.project === 'string').map(r => r.result.project));
const pendingProject = (business, project, entry, activated) => entry?.kind !== 'library' && !activated.has(project)
&& managedState({ dataRoot, business, project, entry }) !== 'unmanaged';
const pendingProject = (business, project, entry, activated) => {
if (entry?.kind === 'library') return false;
const state = managedState({ dataRoot, business, project, entry });
return state !== 'unmanaged' && !(state === 'complete' && activated.has(project));
};
const withoutPending = (business, b, admit = []) => {
const activated = new Set([...activatedProjects(business), ...admit]);
const projects = Object.fromEntries(Object.entries(b.projects ?? {}).filter(([project, entry]) => !pendingProject(business, project, entry, activated)));
@@ -514,9 +519,11 @@ export async function startBroker({
}
}
// Lifecycle Task 2 (#797): the only way a project joins a running bus. The bootstrap executor calls it
// synchronously after every earlier step passed. It admits exactly the receipt-backed addition under the
// organization lock: the file must still be this business plus that entry, with the credentials, bots,
// libraries, roles and prior mappings the bus booted with. Anything else leaves the product pending.
// synchronously after every earlier step passed and the complete marker is published. It admits exactly the
// receipt-backed addition under the organization lock: the file must still be this business plus that entry,
// with the credentials, bots, libraries, roles and prior mappings the bus booted with. Anything else leaves
// the product pending. The executor commits the activate receipt right after this returns; admission() and a
// restart treat the product as ready only once that receipt exists.
const writer = organization ? organizationWriter({ dir: organization.dir, rolesDir: organization.rolesDir }) : null;
const known = new Map(Object.entries(businesses).map(([id, b]) => [id, canonicalJson(b.projects ?? {})]));
function activateManagedProject({ business: id, project, receipt, organization_revision }) {
@@ -536,6 +543,7 @@ export async function startBroker({
const here = managedPaths({ dataRoot, business: id, project });
if (!entry || entry.kind === 'library' || entry.root !== here.root || entry.repo !== here.repo
|| Object.keys(entry).some(k => !['root', 'repo', 'remote'].includes(k))) refuse('bootstrap-mapping-mismatch');
if (managedState({ dataRoot, business: id, project, entry }) !== 'complete') refuse('bootstrap-marker-incomplete');
const next = withoutPending(id, full, [project]);
const { [project]: _added, ...before } = next.projects;
if (canonicalJson(before) !== known.get(id)) refuse('bootstrap-mapping-mismatch');
@@ -549,7 +557,10 @@ export async function startBroker({
});
}
// Host IPC request.admission: may a project's code checkout be cloned and its tools used. A project the
// broker does not hold is never ready, and a declared one that is not yet held says it is pending.
// broker does not hold is never ready, and a declared one that is not yet held says it is pending. A managed
// product the broker holds is ready only while it has a complete marker and a committed activate receipt: the
// answer is read from both on every call, so a marker removed after the activation closes it again, and the
// broker's admission, which comes before the receipt commits, is not yet readiness (#802).
function admission({ business: id, project }) {
const no = code => ({ ready: false, revision: null, code });
if (typeof id !== 'string' || typeof project !== 'string' || !Object.hasOwn(businesses, id)) return no('project-unknown');
@@ -560,7 +571,11 @@ export async function startBroker({
revision = now.revision; declared = Object.hasOwn(now.document.projects ?? {}, project);
}
} catch { return no('organization-unreadable'); }
if (Object.hasOwn(businesses[id].projects ?? {}, project)) return { ready: true, revision, code: null };
if (Object.hasOwn(businesses[id].projects ?? {}, project)) {
try { if (pendingProject(id, project, businesses[id].projects[project], activatedProjects(id))) return no('project-pending'); }
catch { return no('project-pending'); }
return { ready: true, revision, code: null };
}
return no(declared ? 'project-pending' : 'project-unknown');
}
broker = new Broker({
packages/bus/tests/request-bootstrap.test.mjs
+191 -2
View File
@@ -3,8 +3,10 @@
// The test barrier stands in for a kill: it throws after a step's effect and before its receipt.
import test from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import cp, { execFileSync } from 'node:child_process';
import fs, { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import { syncBuiltinESMExports } from 'node:module';
import { DatabaseSync } from 'node:sqlite';
import { join } from 'node:path';
import { startBroker } from '../src/runtime.mjs';
import { prepareBootstrap } from '../src/request-bootstrap.mjs';
@@ -473,3 +475,190 @@ test('i3_host_admission_refuses_a_plan_whose_world_has_moved_with_a_named_code',
assert.equal(staffFor(now.document, 'cto'), null, 'the PM cannot also be the technical reviewer');
assert.equal(staffFor({ ...now.document, launch: { ...now.document.launch, instances: ['coder', 'reviewer', 'reviewer'] } }, 'pm'), null);
});
// ---- Fix round for review r2 (#802): N1, I1, I2 -------------------------------------------------------------------
// N1. The working repository is confined to the data root like every other bootstrap path.
for (const [what, plant, kind] of [
['the_working_directory_is_a_link', (f, outside) => { symlinkSync(outside, join(f.dataRoot, 'working')); return join(f.dataRoot, 'working'); }, 'link'],
['the_business_directory_under_working_is_a_link', (f, outside) => {
mkdirSync(join(f.dataRoot, 'working')); symlinkSync(outside, join(f.dataRoot, 'working', 'demo')); return join(f.dataRoot, 'working', 'demo');
}, 'link'],
['the_working_directory_is_a_file', f => { writeFileSync(join(f.dataRoot, 'working'), 'not a directory\n'); return join(f.dataRoot, 'working'); }, 'file'],
]) {
test(`n1_${what}_so_nothing_is_written_through_it_and_the_product_is_not_admitted`, async t => {
let outside, planted;
const f = await setup(t, { configureFactory: ({ root, f }) => { outside = external(root, 'external-working'); planted = plant(f, outside); } });
const out = await f.choose();
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain');
assert.deepEqual(readdirSync(outside), [], 'nothing was made behind the link');
assert.equal(kind === 'link' ? lstatSync(planted).isSymbolicLink() : lstatSync(planted).isFile(), true, 'what was planted is left as it was');
assert.equal(f.additions().length, 0);
assert.equal((await f.first.bus.admission({ business: 'demo', project: 'timer' })).ready, false);
});
}
test('n1_a_working_repository_already_there_on_the_first_attempt_is_kept_and_not_adopted', async t => {
const f = await setup(t, { configureFactory: ({ f }) => {
const repo = workingRepo(f.dataRoot, 'demo', 'timer'); mkdirSync(repo, { recursive: true }); writeFileSync(join(repo, 'keep.txt'), 'mine\n');
} });
const out = await f.choose();
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain');
const repo = workingRepo(f.f.dataRoot, 'demo', 'timer');
assert.deepEqual(readdirSync(repo), ['keep.txt']); assert.equal(readFileSync(join(repo, 'keep.txt'), 'utf8'), 'mine\n');
assert.equal(f.additions().length, 0);
assert.equal((await f.first.bus.admission({ business: 'demo', project: 'timer' })).ready, false);
});
// The plan's own repository, left by a kill after the step's effect, is adopted on retry and the project completes.
test('n1_a_working_repository_left_by_a_kill_is_adopted_on_retry_and_the_project_completes', async t => {
let armed = true;
const f = await setup(t, { barrier: step => { if (armed && step === 'working') { armed = false; throw new Error('killed'); } } });
const stopped = await f.choose();
assert.equal(stopped.wait.code, 'bootstrap-uncertain');
const repo = workingRepo(f.f.dataRoot, 'demo', 'timer');
assert.equal(git(repo, 'rev-parse', 'refs/heads/main'), git(f.paths.code, 'rev-parse', 'HEAD'));
await f.first.bus.close();
const next = await f.boot();
const done = await stateOf(f, next.bus);
assert.equal(done.phase, 'discovery'); assert.equal(f.additions().length, 1);
assert.equal(git(repo, 'rev-parse', 'refs/heads/main'), git(f.paths.code, 'rev-parse', 'HEAD'));
assert.equal((await next.bus.admission({ business: 'demo', project: 'timer' })).ready, true);
});
test('n1_a_working_repository_with_a_foreign_ref_is_kept_and_not_adopted_on_retry', async t => {
let armed = true;
const f = await setup(t, { barrier: step => { if (armed && step === 'working') { armed = false; throw new Error('killed'); } } });
await f.choose(); await f.first.bus.close();
const repo = workingRepo(f.f.dataRoot, 'demo', 'timer'), head = git(repo, 'rev-parse', 'refs/heads/main');
git(repo, 'update-ref', 'refs/heads/foreign', head);
const next = await f.boot();
const out = await stateOf(f, next.bus);
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain');
assert.equal(git(repo, 'rev-parse', 'refs/heads/foreign'), head, 'the foreign ref is kept');
assert.equal(f.additions().length, 0);
assert.equal((await next.bus.admission({ business: 'demo', project: 'timer' })).ready, false);
});
// I1. A protected write happens only while the plan is still fresh, with the check and the start of the write in one
// synchronous run. These interleavings land an ordinary organization write after the step's last asynchronous read and
// before its write, which is exactly where a bare check-then-await let a stale plan write.
const moveOrganization = f => {
const now = f.writer.read('demo'), next = structuredClone(now.document); next.vars['tracker.pollSeconds'] = 61;
f.writer.write('demo', next, { revision: now.revision, actor: 'jason' });
};
// `before(binary, args)` runs just ahead of every child the code spawns while `during` runs.
async function withSpawnHook(before, during) {
const original = cp.spawn;
cp.spawn = function (binary, args, ...rest) { before(binary, args); return original.call(this, binary, args, ...rest); };
syncBuiltinESMExports();
try { return await during(); } finally { cp.spawn = original; syncBuiltinESMExports(); }
}
test('i1_an_organization_write_during_the_intent_reads_leaves_no_intent_commit_under_the_stale_plan', async t => {
let ready = false, changed = false, f;
f = await setup(t, { barrier: step => { if (step === 'records') ready = true; } });
const out = await withSpawnHook((binary, args) => {
if (ready && !changed && binary === 'git' && args.includes(f.paths.records) && args.includes('rev-list') && args.includes('--count')) { changed = true; moveOrganization(f); }
}, () => f.choose());
assert.equal(changed, true, 'the organization write landed between the reads and the write');
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'request-plan-stale');
const star = join(f.paths.records, 'spec/north-star.md');
assert.notEqual(existsSync(star) ? readFileSync(star, 'utf8') : null, INTENT, 'the intent was not written');
assert.equal(git(f.paths.records, 'rev-list', '--count', 'HEAD'), '1');
assert.equal(git(f.paths.records, 'log', '-1', '--format=%s'), 'Initialize project layout v1');
assert.equal(git(f.paths.records, 'status', '--porcelain'), '');
assert.equal((await f.first.bus.admission({ business: 'demo', project: 'timer' })).ready, false);
});
test('i1_an_organization_write_during_the_seed_reads_leaves_the_checkout_unseeded', async t => {
let changed = false, f;
f = await setup(t);
const out = await withSpawnHook((binary, args) => {
if (!changed && binary === 'git' && args.includes(f.paths.code) && args.includes('status')) { changed = true; moveOrganization(f); }
}, () => f.choose());
assert.equal(changed, true);
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'request-plan-stale');
assert.equal(git(f.paths.code, 'rev-list', '--count', 'HEAD'), '1', 'no seed commit was made');
assert.equal(readFileSync(join(f.paths.code, '.mosaic/project.json'), 'utf8'), JSON.stringify(PROJECT_FILE, null, 2) + '\n');
assert.equal(git(f.paths.code, 'status', '--porcelain'), '');
assert.equal(f.additions().length, 0);
});
// I2. The complete marker is published before the broker admits the project, and admission needs both the marker and the
// committed activation receipt, live and after a restart.
const activations = f => {
const database = new DatabaseSync(join(f.f.dataRoot, 'bus/bus.sqlite'), { readOnly: true });
try { return database.prepare('SELECT op, stage FROM request_effects WHERE business = ? AND request = ?').all('demo', f.request).filter(r => r.op.endsWith(':activate') && r.stage === 'committed'); }
finally { database.close(); }
};
const marked = f => JSON.parse(readFileSync(join(f.paths.product, 'bootstrap.json'), 'utf8')).state;
const admission = f => f.first.bus.admission({ business: 'demo', project: 'timer' });
test('i2_a_failed_marker_publication_ends_the_step_before_the_broker_admits_the_project', async t => {
const f = await setup(t), marker = join(f.paths.product, 'bootstrap.json');
const original = fs.renameSync; let faults = 1;
fs.renameSync = function (from, to, ...rest) {
if (faults && from === `${marker}.tmp` && to === marker) { faults--; throw Object.assign(new Error('injected final publication I/O failure'), { code: 'EIO' }); }
return original.call(this, from, to, ...rest);
};
syncBuiltinESMExports();
let out;
try { out = await f.choose(); } finally { fs.renameSync = original; syncBuiltinESMExports(); }
assert.equal(faults, 0, 'the fault fired');
assert.equal(out.phase, 'bootstrap'); assert.equal(out.wait.code, 'bootstrap-uncertain');
assert.equal(marked(f), 'reserved');
const pending = await admission(f);
assert.equal(pending.ready, false); assert.equal(pending.code, 'project-pending');
assert.deepEqual(activations(f), [], 'no activation receipt was committed');
assert.throws(() => f.call('human', 'project.create', { name: 'Side', repo: 'timer' }), 'the broker does not serve the project');
// The fault is gone: a retry publishes the marker, admits the project and commits the receipt.
const again = await f.first.bus.requestLifecycle.resume('demo', f.request);
assert.equal(again.phase, 'discovery');
assert.equal(marked(f), 'complete'); assert.equal((await admission(f)).ready, true); assert.equal(activations(f).length, 1);
assert.ok(f.call('human', 'project.create', { name: 'Side', repo: 'timer' }).id);
});
// Defence in depth: even if the executor's marker write did nothing, the runtime refuses to admit a product whose marker
// is not complete.
test('i2_the_runtime_refuses_to_admit_a_product_whose_marker_is_not_complete', async t => {
const f = await setup(t), marker = join(f.paths.product, 'bootstrap.json');
const original = fs.renameSync; let skipped = 0;
fs.renameSync = function (from, to, ...rest) {
if (from === `${marker}.tmp` && to === marker) { skipped++; return undefined; }
return original.call(this, from, to, ...rest);
};
syncBuiltinESMExports();
let out;
try { out = await f.choose(); } finally { fs.renameSync = original; syncBuiltinESMExports(); }
assert.ok(skipped >= 1, 'the marker rename was skipped');
assert.equal(out.phase, 'bootstrap');
assert.equal(marked(f), 'reserved');
assert.equal((await admission(f)).ready, false); assert.deepEqual(activations(f), []);
assert.throws(() => f.call('human', 'project.create', { name: 'Side', repo: 'timer' }));
});
test('i2_a_completed_product_whose_marker_is_removed_is_not_ready_live_or_after_reopen', async t => {
const f = await setup(t), marker = join(f.paths.product, 'bootstrap.json');
assert.equal((await f.choose()).phase, 'discovery');
assert.equal((await admission(f)).ready, true);
const bytes = readFileSync(marker);
// Live: the broker is already serving the project, and admission still needs the marker.
rmSync(marker);
const live = await admission(f);
assert.equal(live.ready, false); assert.equal(live.code, 'project-pending');
writeFileSync(marker, bytes, { mode: 0o600 });
assert.equal((await admission(f)).ready, true, 'the marker back makes it ready again');
// After a restart: the committed receipt alone admits nothing, and the marker is not rewritten.
await f.first.bus.close();
rmSync(marker);
const next = await f.boot();
const reopened = await next.bus.admission({ business: 'demo', project: 'timer' });
assert.equal(reopened.ready, false); assert.equal(reopened.code, 'project-pending');
assert.equal(existsSync(marker), false, 'recovery never rewrites a marker it did not write');
assert.equal(next.warnings.length, 0);
await next.bus.close();
writeFileSync(marker, bytes, { mode: 0o600 });
const last = await f.boot();
assert.equal((await last.bus.admission({ business: 'demo', project: 'timer' })).ready, true);
});
packages/business/src/project-bootstrap.mjs
+4 -2
View File
@@ -39,8 +39,10 @@ export function additionProblem({ old, next, project, entry, receipt }) {
}
// The product marker, <dataRoot>/products/<business>/<project>/bootstrap.json. The bootstrap executor writes it
// `reserved` before the mapping exists and `complete` once the product is activated, so a process that cannot
// read the request journal (the launcher and the boot config) can tell a half-built product from a finished one.
// `reserved` before the mapping exists and `complete` once every step before the activation passed, just before the
// bus admits the project, so a process that cannot read the request journal (the launcher and the boot config) can
// tell a half-built product from a finished one. `complete` alone is not admission: the bus also needs the
// committed activate receipt (#802).
const MARKER_STATES = ["reserved", "complete"];
export const markerFile = ({ dataRoot, business, project }) => join(dataRoot, "products", business, project, "bootstrap.json");
export function readMarker(file) {
packages/project/src/index.mjs
+7 -4
View File
@@ -192,7 +192,9 @@ function publish(file, bytes) {
// One file, one local commit: the lock, the clean-tree and layout checks, publish, commit and the
// rollback. The agent writer and the human writer (row 148) differ only in who authors the commit
// and in `guard`, which runs under the lock with the file's current bytes (null when absent).
async function commitFile({ context, repo, path, bytes, name, subject, guard }) {
// `around(effect)` (the bootstrap executor's, #802) runs `effect`, the one write that starts the commit, after every
// read above it has finished; it checks its caller's precondition and starts the effect in one synchronous run.
async function commitFile({ context, repo, path, bytes, name, subject, guard, around }) {
const metadata = lstatSync(join(repo, '.git'));
if (!metadata.isDirectory() || metadata.isSymbolicLink())
throw new ProjectError('project-repository', 'expected a local .git directory');
@@ -214,7 +216,8 @@ async function commitFile({ context, repo, path, bytes, name, subject, guard })
if (guard) await guard({ before });
if (before?.equals(bytes)) throw new ProjectError('project-record-unchanged', 'record already has these bytes');
head = await git(repo, ['rev-parse', 'HEAD']);
publish(file, bytes); published = true;
if (around) around(() => publish(file, bytes)); else publish(file, bytes);
published = true;
await git(repo, ['add', '--', path]);
await git(repo, ['commit', '--only', '-m', subject, '--', path], identity(name));
const commit = await git(repo, ['rev-parse', 'HEAD']);
@@ -245,7 +248,7 @@ export async function writeRecord({ path, record, author, ...context }) {
// spec/north-star.md or docs/*.md and makes one commit authored by the business's human. `base` is
// the file's last commit as the editor opened it; a newer last commit refuses `stale-base` and
// writes nothing. Agents never reach this: only the human bus verb calls it.
export async function writeHumanFile({ path, content, base, ...context }) {
export async function writeHumanFile({ path, content, base, around, ...context }) {
const repo = projectRepo(context);
if (!documentPath(path)) throw new ProjectError('project-record-path', 'expected spec/north-star.md or a named Markdown doc');
if (typeof content !== 'string' || !content.length || !content.isWellFormed() || content.includes('\0') || Buffer.byteLength(content) > MAX_BYTES)
@@ -257,7 +260,7 @@ export async function writeHumanFile({ path, content, base, ...context }) {
if (await git(repo, ['log', '-1', '--format=%H', 'HEAD', '--', path]) !== base)
throw new ProjectError('stale-base', 'the file changed since the editor opened it');
};
return commitFile({ context, repo, path, bytes: Buffer.from(content, 'utf8'), name, subject: `Edit ${path}`, guard });
return commitFile({ context, repo, path, bytes: Buffer.from(content, 'utf8'), name, subject: `Edit ${path}`, guard, around });
}
export { readProjectFile, readSpec, listProjectFiles } from './read.mjs';
packages/seat/src/working.mjs
+63 -19
View File
@@ -82,7 +82,7 @@
import { randomBytes } from "node:crypto";
import { closeSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { dirname, join, relative, sep } from "node:path";
import { gitEnv, GIT_LIMITS, runGit } from "./checkout.mjs";
import { withoutProcessing } from '../../processing-control/src/effects.mjs';
@@ -153,22 +153,64 @@ function locked(repo, fn) {
return run;
}
// The bare repository, made on first use in a temp directory beside it and
// renamed into place. One that exists must be a real directory.
async function ensure(repo, g) {
const look = (path) => {
try {
if (!lstatSync(repo).isDirectory()) refuse("working-repo-invalid", `${repo} isn't a directory`);
return;
return lstatSync(path);
} catch (e) {
if (e instanceof WorkingError) throw e;
if (e.code !== "ENOENT") refuse("working-repo-invalid", `${repo} can't be read: ${e.code}`);
if (e.code === "ENOENT") return null;
refuse("working-repo-invalid", `${path} can't be read: ${e.code}`);
}
mkdirSync(dirname(repo), { recursive: true, mode: 0o700 });
};
// Every directory between the data root and the repository (`working`, then the business) must be a real
// directory, never a link or a file, so no later mkdir, init or rename can leave the data root. With `make`
// a missing one is created, one level at a time and not recursively, and checked again once it is there.
// The data root itself is the operator's; only a missing one is made. Returns whether all of them exist.
function ancestors(dataRoot, repo, make = false) {
const names = relative(dataRoot, repo).split(sep);
if (names.length < 2 || names[0] === "..") refuse("working-repo-invalid", `${repo} isn't under ${dataRoot}`);
if (make && look(dataRoot) === null) mkdirSync(dataRoot, { recursive: true, mode: 0o700 });
let at = dataRoot;
for (const name of names.slice(0, -1)) {
at = join(at, name);
let st = look(at);
if (st === null && make) {
try {
mkdirSync(at, { mode: 0o700 });
} catch (e) {
if (e.code !== "EEXIST") throw e;
}
st = look(at);
}
if (st === null) return false;
if (st.isSymbolicLink() || !st.isDirectory()) refuse("working-repo-invalid", `${at} isn't a real directory`);
}
return true;
}
// The effect runs as it is unless the caller gives a wrapper. initializeWorking's caller (the bootstrap
// executor) passes one that checks, under the organization lock, that its plan is still fresh and starts the
// effect in the same synchronous run, so a write here never begins under a plan that went stale while this
// code waited on the lock or on git.
const direct = (effect) => effect();
// The bare repository, made on first use in a temp directory beside it and
// renamed into place. One that exists must be a real directory, and so must every
// directory above it up to the data root.
async function ensure(dataRoot, repo, g, around = direct) {
if (ancestors(dataRoot, repo)) {
const st = look(repo);
if (st !== null) {
if (st.isSymbolicLink() || !st.isDirectory()) refuse("working-repo-invalid", `${repo} isn't a directory`);
return;
}
}
around(() => ancestors(dataRoot, repo, true));
const tmp = join(dirname(repo), `.init-${randomBytes(4).toString("hex")}`);
try {
const r = await g(["init", "-q", "--bare", "--template=", tmp]);
const r = await around(() => g(["init", "-q", "--bare", "--template=", tmp]));
if (!r.ok) refuse("working-repo-invalid", `git init failed: ${r.why}`);
renameSync(tmp, repo);
around(() => renameSync(tmp, repo));
} finally {
rmSync(tmp, { recursive: true, force: true });
}
@@ -178,9 +220,9 @@ async function ensure(repo, g) {
// owner's, not a session's, and may move any way. main starts at root and
// follows it only by fast-forward, since a landing (row 137) moves main past
// root and a reseed must not throw the landing away. main is never forced.
async function seed(repo, root, g) {
async function seed(repo, root, g, around = direct) {
const at = `--git-dir=${repo}`;
const r = await g([at, "fetch", "-q", "--no-tags", "--no-write-fetch-head", "--no-recurse-submodules", "--no-auto-maintenance", root, "+HEAD:refs/heads/root"]);
const r = await around(() => g([at, "fetch", "-q", "--no-tags", "--no-write-fetch-head", "--no-recurse-submodules", "--no-auto-maintenance", root, "+HEAD:refs/heads/root"]));
if (!r.ok) refuse("root-unreadable", `the root couldn't be fetched from ${root}: ${r.why}`);
const top = await tip(repo, "refs/heads/root", g);
if (!top) refuse("root-unreadable", `${root} has no commit checked out`);
@@ -189,7 +231,7 @@ async function seed(repo, root, g) {
if (main !== null && !(await g([at, "merge-base", "--is-ancestor", main, top])).ok) return;
// An empty old value creates main only if it doesn't exist; otherwise the
// swap moves it only from where it was read.
const moved = await g([at, "update-ref", "refs/heads/main", top, main ?? ""]);
const moved = await around(() => g([at, "update-ref", "refs/heads/main", top, main ?? ""]));
if (!moved.ok) refuse("working-repo-invalid", `main couldn't follow root to ${top}: ${moved.why}`);
}
@@ -220,7 +262,7 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl
const g = bound(env, limits);
return locked(repo, async () => {
if (!header(bundle)) refuse("invalid-bundle", "not a v2 git bundle");
await ensure(repo, g);
await ensure(dataRoot, repo, g);
await seed(repo, root, g);
const at = [`--git-dir=${repo}`];
// Not -q: with it, git 2.55 drops the list of missing prerequisites.
@@ -261,18 +303,20 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl
});
}
// await initializeWorking({ dataRoot, business, project, root, env, limits })
// await initializeWorking({ dataRoot, business, project, root, env, limits, around })
// Host-only (lifecycle Task 2, #797): makes the project's working repository and seeds main from
// the root's checked-out commit, under the same lock and through the same private seed a push
// uses. It makes no work ref, takes no bundle and has no remote to push to. A second call is the
// same seed: it moves nothing the first call left. Resolves { repo, commit } with main's commit.
export function initializeWorking({ dataRoot, business, project, root, env = process.env, limits = GIT_LIMITS }) {
// `around(effect)` wraps every write (the directories, the init and its rename, the seed's fetch and
// ref update); it must run `effect` synchronously and return what it returns.
export function initializeWorking({ dataRoot, business, project, root, env = process.env, limits = GIT_LIMITS, around = direct }) {
if (!PROJECT.test(project ?? "") || !PROJECT.test(business ?? "")) refuse("invalid-request", "not an id");
const repo = workingRepo(dataRoot, business, project);
const g = bound(env, limits);
return locked(repo, async () => {
await ensure(repo, g);
await seed(repo, root, g);
await ensure(dataRoot, repo, g, around);
await seed(repo, root, g, around);
const commit = await tip(repo, "refs/heads/main", g);
if (!commit) refuse("working-repo-invalid", "main has no commit after the seed");
return { repo, commit };
packages/seat/tests/working.test.mjs
+46 -2
View File
@@ -8,11 +8,11 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs";
import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { gitEnv } from "../src/checkout.mjs";
import { dropLand, finishLand, prepareLand, push, review, validBranch, workingRepo, workRef, WorkingError } from "../src/working.mjs";
import { dropLand, finishLand, initializeWorking, prepareLand, push, review, validBranch, workingRepo, workRef, WorkingError } from "../src/working.mjs";
import { rootHits } from "../src/landing.mjs";
const TASK = "vikunja:1/2";
@@ -124,6 +124,50 @@ async function lands(w, fn) {
return r;
}
// #802 N1: the helper makes its directories one at a time and never follows a link or a file among them.
for (const [what, plant] of [
["the working directory is a link", (w, outside) => symlinkSync(outside, join(w.dataRoot, "working"))],
["the business directory is a link", (w, outside) => { mkdirSync(join(w.dataRoot, "working")); symlinkSync(outside, join(w.dataRoot, "working", "acme")); }],
["the working directory is a file", (w) => writeFileSync(join(w.dataRoot, "working"), "not a directory\n")],
]) {
test(`push and initializeWorking refuse when ${what}, and nothing lands behind it`, async (t) => {
const w = world(t);
const outside = join(w.dir, "outside");
mkdirSync(outside);
plant(w, outside);
git(w.clone, "checkout", "-q", "-b", "feat");
commit(w.clone, "two");
const file = w.bundle();
await assert.rejects(w.go(file), (e) => e instanceof WorkingError && e.code === "working-repo-invalid");
await assert.rejects(initializeWorking({ dataRoot: w.dataRoot, business: "acme", project: "stack", root: w.root }), (e) => e instanceof WorkingError && e.code === "working-repo-invalid");
assert.deepEqual(readdirSync(outside), [], "nothing was made behind the link");
assert.equal(existsSync(join(w.dataRoot, "working", "acme", "stack.git")), false);
});
}
// #802 I1: every write initializeWorking makes goes through `around`, so a caller that checks there never has a
// write begin after its check went stale. A refusal from `around` stops that write and every later one.
test("initializeWorking runs each of its five writes through around, and a refusal stops it there", async (t) => {
const run = (w, around) => initializeWorking({ dataRoot: w.dataRoot, business: "acme", project: "stack", root: w.root, around });
let calls = 0;
const w = world(t);
const made = await run(w, (effect) => { calls++; return effect(); });
assert.equal(calls, 5, "the directories, the init, its rename, the root fetch and the main ref");
assert.equal(made.commit, w.first);
const at = [
{ repo: false, refs: "" }, { repo: false, refs: "" }, { repo: false, refs: "" },
{ repo: true, refs: "" }, { repo: true, refs: `refs/heads/root ${w.first}` },
];
for (let stop = 0; stop < 5; stop++) {
const v = world(t), refusal = new Error(`stale at ${stop}`);
let n = 0;
await assert.rejects(run(v, (effect) => { if (n++ === stop) throw refusal; return effect(); }), (e) => e === refusal);
assert.equal(existsSync(v.repo), at[stop].repo, `repository after a refusal at write ${stop}`);
assert.equal(v.refs(), at[stop].refs.replace(w.first, v.first), `refs after a refusal at write ${stop}`);
assert.deepEqual(existsSync(join(v.dataRoot, "working", "acme")) ? readdirSync(join(v.dataRoot, "working", "acme")).filter((x) => x.startsWith(".init-")) : [], [], "no init temp is left");
}
});
test("workRef and validBranch take only what a ref name can hold", () => {
assert.equal(workRef("vikunja:12/345"), "work/vikunja-12-345");
for (const bad of ["vikunja:0/1", "vikunja:1", "gitea:1/2", "vikunja:1/2\n", null]) assert.throws(() => workRef(bad), WorkingError);