Files
stack/packages/queue/tests/write.test.mjs
T
jason.woltjeandClaude Opus 5.5 34a72af912 feat(queue): queue as data A1, journal, lock, CLI and verify (#1508)
packages/queue, scripts/queue-commit.sh, scripts/git-hooks and
scripts/test-queue.sh, plus docs/plans/BRIEF-TEMPLATE.md. There is no
queue.json yet, so verify skips until the genesis commit after A2.

Darkwing built it, and Filbert reviewed R0 (6933b885, changes requested)
and r1 (e464be6c, approved). The 20 files match manifest 85a8a453. The
nine suites passed on an index export, including the new queue suite.
test-queue.sh joins the suite list in AGENTS.md. Lead decisions 20, 23
and 26.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-09-26 19:07:48 -05:00

374 lines
21 KiB
JavaScript

// The write path (8.5): injected faults, SIGKILL at each step, git
// interference, the witness, sync and accept-history, and unlocked reads
// racing a writer (8.4, F2).
import assert from "node:assert/strict";
import { spawn, spawnSync } from "node:child_process";
import { readFileSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { test } from "node:test";
import { cli, genesisCommitted, load, scratchRepo } from "./helpers.mjs";
const HERE = new URL(".", import.meta.url).pathname;
async function ready(t) {
const repo = scratchRepo(t);
genesisCommitted(repo);
return { repo, m: await load(repo) };
}
const note = (id, text, op, by = "darkwing") => ({ verb: "note", op, args: { id, text }, by });
const o = (repo, extra = {}) => ({ cwd: repo.root, env: repo.env, lockWaitMs: 300, lockStepMs: 50, ...extra });
const read = (p) => readFileSync(p, "utf8");
const revOf = (repo) => JSON.parse(read(repo.queuePath)).revision;
const witness = (repo) => JSON.parse(read(join(repo.gitDir, "mosaic-queue.head")));
const shownRev = (repo) => Number(/revision (\d+) by/.exec(read(repo.viewPath))[1]);
const tmps = (repo) => [...readdirSync(join(repo.root, "docs/plans")), ...readdirSync(repo.gitDir)].filter((n) => n.endsWith(".tmp"));
function throwsCode(fn, code, re) {
assert.throws(fn, (err) => {
assert.equal(err.code, code, err.message);
assert.match(err.message, re);
return true;
});
}
// realIo with one fault: `name` fails for paths that `match`.
function faultIo(m, name, match, code) {
const real = m.io.realIo;
const paths = new Map();
const fail = () => { throw Object.assign(new Error(code), { code }); };
const hit = (n, p) => n === name && p !== undefined && match(p);
return {
...real,
openExcl: (p, mode) => { const fd = real.openExcl(p, mode); paths.set(fd, p); return fd; },
openRead: (p) => { const fd = real.openRead(p); paths.set(fd, p); return fd; },
close: (fd) => { paths.delete(fd); real.close(fd); },
write: (fd, b, off, len) => (hit("write", paths.get(fd)) ? (code === "SHORT" ? 0 : fail()) : real.write(fd, b, off, len)),
fsync: (fd) => (hit("fsync", paths.get(fd)) ? fail() : real.fsync(fd)),
rename: (a, b) => (hit("rename", b) ? fail() : real.rename(a, b)),
fsyncDir: (d) => (hit("fsyncDir", d) ? fail() : real.fsyncDir(d)),
};
}
const queueTmp = (p) => /queue\.json\.[^/]*\.tmp$/.test(p);
test("a short write, ENOSPC or a file fsync failure: nothing visible, temp removed", async (t) => {
const { repo, m } = await ready(t);
const before = read(repo.queuePath);
for (const [name, code] of [["write", "SHORT"], ["write", "ENOSPC"], ["fsync", "EIO"]]) {
const io = faultIo(m, name, queueTmp, code);
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 1, /cannot write docs\/plans\/queue.json.note-9-000001.tmp \((ESHORT|ENOSPC|EIO)\); nothing changed/);
assert.equal(read(repo.queuePath), before);
assert.deepEqual(tmps(repo), []);
assert.equal(witness(repo).revision, 0);
}
assert.match(m.store.mutate(o(repo), note(9, "x", "note-9-000001")).out[0], /^ok note-9-000001 rev 1/);
});
test("a rename failure: nothing visible, temp removed", async (t) => {
const { repo, m } = await ready(t);
const before = read(repo.queuePath);
const io = faultIo(m, "rename", (p) => p === repo.queuePath, "EXDEV");
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 1, /cannot replace docs\/plans\/queue.json \(EXDEV\); nothing changed/);
assert.equal(read(repo.queuePath), before);
assert.deepEqual(tmps(repo), []);
});
test("a directory fsync failure: uncertain, exit 3, no receipt; the tail refuses new ops; a retry confirms", async (t) => {
const { repo, m } = await ready(t);
const view = read(repo.viewPath);
const io = faultIo(m, "fsyncDir", (d) => d === join(repo.root, "docs/plans"), "EIO");
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /^uncertain note-9-000001 rev 1: visible, durability not confirmed \(EIO\)$/);
assert.equal(revOf(repo), 1);
assert.equal(witness(repo).revision, 0);
assert.equal(read(repo.viewPath), view);
throwsCode(() => m.store.mutate(o(repo), note(9, "y", "note-9-000002")), 2, /unconfirmed tail: rev 1 \(op note-9-000001 by darkwing at .*\) visible but not confirmed durable; run `scripts\/mosaic queue sync` or retry that op/);
const unlocked = m.store.list(o(repo));
assert.match(unlocked.err.join("\n"), /rev 1 visible, not confirmed durable/);
const retry = m.store.mutate(o(repo), note(9, "x", "note-9-000001"));
assert.match(retry.err.join("\n"), /durable now, never acknowledged: note-9-000001 by darkwing/);
assert.match(retry.err.join("\n"), /warning: view stale/);
assert.match(retry.out[0], /\(already recorded at rev 1\)$/);
assert.equal(witness(repo).revision, 1);
});
test("a directory fsync failure, then sync names the op", async (t) => {
const { repo, m } = await ready(t);
const io = faultIo(m, "fsyncDir", (d) => d === join(repo.root, "docs/plans"), "EIO");
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /uncertain/);
const r = cli(repo, ["sync"]);
assert.equal(r.code, 0, r.err);
assert.match(r.out, /^durable now, never acknowledged: note-9-000001 by darkwing at /);
assert.match(r.err, /view stale/);
assert.equal(cli(repo, ["render"]).code, 0);
assert.equal(cli(repo, ["verify"]).code, 0);
});
test("a witness write failure: uncertain, durable, exit 3; the view is untouched", async (t) => {
const { repo, m } = await ready(t);
const io = faultIo(m, "rename", (p) => p.endsWith("mosaic-queue.head"), "EIO");
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /^uncertain note-9-000001 rev 1: durable, witness not updated \(EIO\)$/);
assert.equal(revOf(repo), 1);
assert.equal(witness(repo).revision, 0);
assert.equal(shownRev(repo), 0);
assert.deepEqual(tmps(repo), []);
assert.match(m.store.mutate(o(repo), note(9, "x", "note-9-000001")).out[0], /already recorded at rev 1/);
});
test("the .git fsync after the witness rename fails: uncertain, exit 3, the witness says so", async (t) => {
const { repo, m } = await ready(t);
const io = faultIo(m, "fsyncDir", (d) => d === repo.gitDir, "EIO");
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /^uncertain note-9-000001 rev 1: durable, witness written, its directory fsync failed \(EIO\)$/);
assert.equal(revOf(repo), 1);
assert.equal(witness(repo).revision, 1);
assert.equal(shownRev(repo), 0);
assert.match(m.store.mutate(o(repo), note(9, "x", "note-9-000001")).out[0], /already recorded at rev 1/);
});
test("confirming a tail fsyncs queue.json and docs/plans before the witness; either failure changes nothing", async (t) => {
const { repo, m } = await ready(t);
const docs = join(repo.root, "docs/plans");
throwsCode(() => m.store.mutate(o(repo, { io: faultIo(m, "fsyncDir", (d) => d === docs, "EIO") }), note(9, "x", "note-9-000001")), 3, /uncertain/);
for (const io of [faultIo(m, "fsync", (p) => p === repo.queuePath, "EIO"), faultIo(m, "fsyncDir", (d) => d === docs, "EIO")]) {
throwsCode(() => m.store.sync(o(repo, { io })), 1, /^cannot confirm rev 1 durable \(EIO\); nothing changed$/);
assert.equal(witness(repo).revision, 0);
}
assert.match(m.store.sync(o(repo)).out.join("\n"), /durable now, never acknowledged: note-9-000001/);
assert.equal(witness(repo).revision, 1);
});
test("the docs/plans fsync after the view rename fails: the op stands, the view is written, a warning says so", async (t) => {
const { repo, m } = await ready(t);
const docs = join(repo.root, "docs/plans");
let calls = 0;
const io = { ...m.io.realIo, fsyncDir: (d) => { if (d === docs && ++calls === 2) throw Object.assign(new Error("EIO"), { code: "EIO" }); m.io.realIo.fsyncDir(d); } };
const r = m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001"));
assert.equal(calls, 2);
assert.match(r.out[0], /^ok note-9-000001 rev 1/);
assert.match(r.err.join("\n"), /the view is written but not confirmed durable \(EIO\); the op stands/);
assert.equal(shownRev(repo), 1);
assert.deepEqual(tmps(repo), []);
});
test("a lock swapped while held is left in place and reported, on a receipt and on a refusal", async (t) => {
const { repo, m } = await ready(t);
const lock = join(repo.gitDir, "mosaic-queue.lock");
// Another inode with the same bytes, as a delayed unlock and relock would leave.
const swap = (name) => { if (name === "locked") { writeFileSync(`${lock}.copy`, readFileSync(lock)); renameSync(`${lock}.copy`, lock); } };
const done = m.store.mutate(o(repo, { hook: swap }), note(9, "x", "note-9-000001"));
assert.match(done.out[0], /^ok note-9-000001 rev 1/);
assert.match(done.err.join("\n"), /warning: lock .* is not the one this process took; left in place/);
unlinkSync(lock);
throwsCode(() => m.store.mutate(o(repo, { hook: swap }), note(9, "y", "note-9-000002", "rocko")), 2,
/may note row 9[^]*\nwarning: lock .* is not the one this process took; left in place$/);
unlinkSync(lock);
});
test("unlock prints a swapped gate's warning on stderr, the result on stdout", async (t) => {
const { repo, m } = await ready(t);
const gate = join(repo.gitDir, "mosaic-queue.unlock");
const swap = (name) => { if (name === "gate-held") { writeFileSync(`${gate}.copy`, readFileSync(gate)); renameSync(`${gate}.copy`, gate); } };
const r = m.store.unlock(o(repo, { hook: swap }));
assert.deepEqual(r.out, ["no queue lock present; nothing removed"]);
assert.match(r.err.join("\n"), /^warning: lock .*mosaic-queue\.unlock is not the one this process took; left in place$/);
unlinkSync(gate);
});
test("a view write that fails keeps the op and reports a stale view", async (t) => {
const { repo, m } = await ready(t);
const io = faultIo(m, "rename", (p) => p === repo.viewPath, "EIO");
const r = m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001"));
assert.match(r.out[0], /^ok note-9-000001 rev 1/);
assert.match(r.err.join("\n"), /the view write failed \(EIO\); the op stands and the view is stale/);
assert.equal(shownRev(repo), 0);
assert.deepEqual(tmps(repo), []);
});
function killAt(repo, m, step, req) {
const r = spawnSync(process.execPath, [join(HERE, "fixtures", "kill-at.mjs"), join(repo.root, "packages/queue/src"), step, JSON.stringify(req)], {
cwd: repo.root, env: repo.env, encoding: "utf8",
});
assert.equal(r.signal, "SIGKILL", `child did not die at ${step}: ${r.stdout}${r.stderr}`);
// The dead child's lock stays; nothing removes it by age.
// The 10 s wait is lock.test.mjs's; here the wait is short.
throwsCode(() => m.store.mutate(o(repo), note(9, "z", "note-9-blocked")), 2, /queue lock owner is dead: pid \d+, note .*; run `scripts\/mosaic queue unlock`/);
const u = cli(repo, ["unlock"]);
assert.equal(u.code, 0, u.err);
assert.match(u.out, /^removed queue lock \(dead/);
}
test("SIGKILL before the rename: nothing recorded; the retry removes the leftover temp", async (t) => {
const { repo, m } = await ready(t);
const before = read(repo.queuePath);
killAt(repo, m, "temp-written", note(9, "x", "note-9-000001"));
assert.equal(read(repo.queuePath), before);
assert.deepEqual(tmps(repo), ["queue.json.note-9-000001.tmp"]);
const r = cli(repo, ["note", "9", "x", "--op", "note-9-000001"], { by: "darkwing" });
assert.match(r.out, /^ok note-9-000001 rev 1 row 9 note$/m);
assert.deepEqual(tmps(repo), []);
});
test("SIGKILL after the rename, before the witness: the tail refuses new ops and sync names the op", async (t) => {
const { repo, m } = await ready(t);
killAt(repo, m, "renamed", note(9, "x", "note-9-000001"));
assert.equal(revOf(repo), 1);
assert.equal(witness(repo).revision, 0);
const r = cli(repo, ["note", "9", "y", "--op", "note-9-000002"], { by: "darkwing" });
assert.equal(r.code, 2);
assert.match(r.err, /unconfirmed tail: rev 1 \(op note-9-000001 by darkwing/);
assert.match(cli(repo, ["sync"]).out, /durable now, never acknowledged: note-9-000001 by darkwing/);
});
test("SIGKILL after the witness, before the view: the stale refusal names the op", async (t) => {
const { repo, m } = await ready(t);
killAt(repo, m, "witnessed", note(9, "x", "note-9-000001"));
assert.equal(witness(repo).revision, 1);
const r = cli(repo, ["note", "9", "y", "--op", "note-9-000002"], { by: "darkwing" });
assert.equal(r.code, 2);
assert.match(r.err, /view stale: QUEUE.md shows rev 0; rev 1 \(op note-9-000001 by darkwing at .*\) is recorded but the table shows rev 0 and may never have been acknowledged\. Tell darkwing/);
});
test("SIGKILL after the view, before the receipt: the retry returns the receipt", async (t) => {
const { repo, m } = await ready(t);
killAt(repo, m, "viewed", note(9, "x", "note-9-000001"));
assert.equal(shownRev(repo), 1);
const r = cli(repo, ["note", "9", "x", "--op", "note-9-000001"], { by: "darkwing" });
assert.equal(r.code, 0);
assert.equal(r.out, "ok note-9-000001 rev 1 row 9 note (already recorded at rev 1)\n");
assert.equal(r.err, "");
});
test("git checkout between steps 1 and 7: step 7 refuses and nothing is written", async (t) => {
const { repo, m } = await ready(t);
m.store.mutate(o(repo), note(9, "one", "note-9-000001"));
const hook = (n) => { if (n === "temp-written") repo.g("checkout", "--", "docs/plans/queue.json"); };
throwsCode(() => m.store.mutate(o(repo, { hook }), note(9, "two", "note-9-000002")), 2, /changed outside the queue lock \(git\?\) since it was read; nothing changed/);
assert.equal(revOf(repo), 0);
assert.deepEqual(tmps(repo), []);
throwsCode(() => m.store.list(o(repo)), 2, /history lost: the witness recorded rev 1/);
});
test("git stash restoring an older valid pair: history lost; accept-history needs privilege, a reason and --yes", async (t) => {
const { repo, m } = await ready(t);
m.store.mutate(o(repo), note(9, "one", "note-9-000001"));
repo.g("commit", "-q", "-am", "queue rev 1");
m.store.mutate(o(repo), note(9, "two", "note-9-000002"));
repo.g("stash", "-q");
assert.equal(revOf(repo), 1);
for (const args of [["list"], ["show", "9"], ["verify"], ["render"], ["sync"]]) {
const r = cli(repo, args);
assert.equal(r.code, 2, args.join(" "));
assert.match(r.err, /history lost: the witness recorded rev 2 \(logDigest [0-9a-f]{12}…, at .*\); this file holds revs 0\.\.1 and does not extend it; every verb refuses except `scripts\/mosaic queue accept-history`/);
}
assert.equal(cli(repo, ["note", "9", "x", "--op", "note-9-000003"], { by: "darkwing" }).code, 2);
const noYes = cli(repo, ["accept-history", "--reason", "stash by mistake", "--op", "accept-hist-01"], { by: "sage" });
assert.equal(noYes.code, 2);
assert.match(noYes.err, /history lost: .*revs 0\.\.1.*\. ops in that range are no longer deduplicated\. Re-run with --yes/);
const seat = cli(repo, ["accept-history", "--reason", "x", "--yes", "--op", "accept-hist-01"], { by: "darkwing" });
assert.equal(seat.code, 2);
assert.match(seat.err, /privileged/);
assert.equal(cli(repo, ["accept-history", "--yes", "--op", "accept-hist-01"], { by: "sage" }).code, 4);
const ok = cli(repo, ["accept-history", "--reason", "stash by mistake", "--yes", "--op", "accept-hist-01"], { by: "sage" });
assert.equal(ok.code, 0, ok.err);
assert.match(ok.err, /ops in that range are no longer deduplicated/);
assert.match(ok.out, /^ok accept-hist-01 rev 2 accept-history over witness rev 2/m);
const d = JSON.parse(read(repo.queuePath));
assert.equal(d.log[2].result.oldWitness.revision, 2);
assert.equal(witness(repo).revision, 2);
assert.equal(cli(repo, ["verify"]).code, 0);
// The lost op id is no longer deduplicated: it records again.
assert.match(cli(repo, ["note", "9", "two", "--op", "note-9-000002"], { by: "darkwing" }).out, /^ok note-9-000002 rev 3/m);
const again = cli(repo, ["accept-history", "--reason", "x", "--yes", "--op", "accept-hist-02"], { by: "sage" });
assert.equal(again.code, 2);
assert.match(again.err, /history is not lost/);
});
test("a deleted witness: refused after the locked recheck; accept-history records it absent", async (t) => {
const { repo, m } = await ready(t);
m.store.mutate(o(repo), note(9, "one", "note-9-000001"));
unlinkSync(join(repo.gitDir, "mosaic-queue.head"));
throwsCode(() => m.store.list(o(repo)), 2, /the witness \.git\/mosaic-queue.head is missing; this file holds revs 0\.\.1, and ops recorded after them may be lost; every verb refuses except/);
throwsCode(() => m.store.sync(o(repo)), 2, /is missing/);
writeFileSync(join(repo.gitDir, "mosaic-queue.head"), "not json\n");
throwsCode(() => m.store.list(o(repo)), 2, /the witness .* is invalid/);
const r = cli(repo, ["accept-history", "--reason", "witness deleted", "--yes", "--op", "accept-hist-01"], { by: "jason" });
assert.equal(r.code, 0, r.err);
assert.match(r.out, /accept-history over witness absent/);
assert.equal(JSON.parse(read(repo.queuePath)).log[2].result.oldWitness, null);
});
test("a header edit during a write: the op stands, the view write is skipped with a warning", async (t) => {
const { repo, m } = await ready(t);
const hook = (n) => { if (n === "witnessed") writeFileSync(repo.viewPath, read(repo.viewPath).replace("Header prose.", "Header prose, edited.")); };
const r = m.store.mutate(o(repo, { hook }), note(9, "one", "note-9-000001"));
assert.match(r.out[0], /^ok note-9-000001 rev 1/);
assert.match(r.err.join("\n"), /QUEUE.md changed since it was read; the view was not written and is stale/);
assert.equal(shownRev(repo), 0);
assert.match(m.store.renderView(o(repo)).out[0], /rendered rev 1 over rev 0/);
assert.match(read(repo.viewPath), /Header prose, edited\./);
});
test("a reader paused between the witness and the file while a writer finishes: no lost-history report", async (t) => {
const { repo, m } = await ready(t);
const hook = (n) => { if (n === "reader-between") m.store.mutate(o(repo), note(9, "one", "note-9-000001")); };
const r = m.store.list(o(repo, { hook }));
assert.equal(r.code, 0);
assert.deepEqual(r.err, ["rev 1 visible, not confirmed durable"]);
});
test("file-then-witness order forced by a hook: the locked recheck prevents a false report", async (t) => {
const { repo, m } = await ready(t);
const hook = (n) => { if (n === "reader-between") m.store.mutate(o(repo), note(9, "one", "note-9-000001")); };
const r = m.store.show(o(repo, { hook, readOrder: "file-first" }), 9);
assert.equal(r.code, 0);
assert.deepEqual(r.err, []);
assert.equal(JSON.parse(r.out[0]).note, "one", "the recheck reports the current revision");
});
test("a writer paused before and after the witness rename: readers see a tail, then a match", async (t) => {
const { repo, m } = await ready(t);
const seen = {};
const hook = (n) => {
if (n === "dir-synced" || n === "witnessed") seen[n] = m.store.list(o(repo)).err;
};
m.store.mutate(o(repo, { hook }), note(9, "one", "note-9-000001"));
assert.equal(seen["dir-synced"][0], "rev 1 visible, not confirmed durable");
assert.equal(seen.witnessed.some((l) => /lost|visible/.test(l)), false);
// Both readers also warn that the view (written after the witness) is behind.
for (const lines of Object.values(seen)) assert.match(lines.at(-1), /^warning: view stale: QUEUE.md shows rev 0/);
});
test("a true rollback is reported only after the locked recheck; a held lock names its holder instead", async (t) => {
const { repo, m } = await ready(t);
m.store.mutate(o(repo), note(9, "one", "note-9-000001"));
repo.g("checkout", "--", "docs/plans/queue.json", "docs/plans/QUEUE.md");
const child = spawn(process.execPath, [join(HERE, "fixtures", "lock-child.mjs"), repo.gitDir, "hold"], { stdio: ["ignore", "pipe", "ignore"] });
t.after(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } });
await new Promise((r) => child.stdout.once("data", r));
throwsCode(() => m.store.list(o(repo)), 2, /^queue lock held by move holder-op-1 since/);
child.kill("SIGKILL");
await new Promise((r) => child.on("exit", r));
m.store.unlock(o(repo));
throwsCode(() => m.store.list(o(repo)), 2, /history lost: the witness recorded rev 1/);
});
test("an accept-history in progress: an unlocked reader waits on the lock and never reports lost history", async (t) => {
const { repo, m } = await ready(t);
m.store.mutate(o(repo), note(9, "one", "note-9-000001"));
unlinkSync(join(repo.gitDir, "mosaic-queue.head"));
let during;
const hook = (n) => {
if (n !== "renamed") return;
try { during = m.store.list(o(repo)); } catch (err) { during = err; }
};
m.store.mutate(o(repo, { hook }), { verb: "accept-history", op: "accept-hist-01", args: { reason: "witness deleted" }, by: "sage", yes: true });
assert.match(during.message, /^queue lock held by accept-history accept-hist-01/);
assert.equal(m.store.list(o(repo)).err.length, 0);
});
test("the platform check refuses other filesystems", async (t) => {
const { repo, m } = await ready(t);
const io = { ...m.io.realIo, statfsType: () => 0x6969 };
throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "one", "note-9-000001")), 2, /unsupported filesystem .*type 0x6969/);
});