diff --git a/docs/plans/BRIEF-TEMPLATE.md b/docs/plans/BRIEF-TEMPLATE.md new file mode 100644 index 00000000..c5ece4ba --- /dev/null +++ b/docs/plans/BRIEF-TEMPLATE.md @@ -0,0 +1,48 @@ +# Brief template + +Copy the section below into the brief file for a queue row, one `##` +heading per row. The row records the file path and that heading as its +brief anchor (`--brief docs/plans/.md#`), plus the git blob +of the committed file. + +Rules the queue enforces (queue-as-data plan 8.13): +- the brief is committed before `queue add` or a brief change names it; a + staged or untracked brief is refused; +- the heading occurs exactly once in the file; +- a later edit to a committed brief makes `next` flag the row and `start` + refuse until the lead re-pins it. + +`queued` means the brief exists, not that it is accepted. The row moves to +`briefed` when its owner accepts it. + +--- + +## + +### Problem + +What is wrong or missing today, with evidence: file paths, issue numbers, +command output. One paragraph. + +### Owner and reviewer + +- Owner: the seat that does the work. +- Reviewer: the seat that reviews it, or "none" and why. + +### Files owned + +Every path this piece may create or change. Anything else is out of scope +unless the brief is amended. + +### What ships + +The concrete result: code, tests, docs. Name the test suites that must pass. + +### Out of scope + +What this piece deliberately leaves alone, and where that work goes. + +### Gate + +Who says yes, and on what evidence: the command to run, the output to +expect, or the comment to read. diff --git a/packages/queue/package.json b/packages/queue/package.json new file mode 100644 index 00000000..72752dec --- /dev/null +++ b/packages/queue/package.json @@ -0,0 +1,11 @@ +{ + "name": "@mosaic/queue", + "version": "0.1.0", + "private": true, + "description": "The work queue as data: docs/plans/queue.json with an append-only log, a generated QUEUE.md table, a lock, a durability witness and the lead's commit procedure.", + "license": "UNLICENSED", + "type": "module", + "engines": { "node": ">=24" }, + "exports": { ".": "./src/store.mjs" }, + "scripts": { "test": "node --test tests/" } +} diff --git a/packages/queue/README.md b/packages/queue/README.md new file mode 100644 index 00000000..74956816 --- /dev/null +++ b/packages/queue/README.md @@ -0,0 +1,233 @@ +# Queue + +The work queue as data (#1508). `docs/plans/queue.json` holds the rows and an +append-only log of every operation. The table in `docs/plans/QUEUE.md` +between the two `mosaic-queue` markers is generated from it. The +specification is section 8 of +`agents/filbert/work/queue-as-data-plan-2026-09-26.md`. This README states +what the code does and the limits it accepts. + +Until A2 adds `scripts/mosaic queue`, run the CLI directly from the canonical +checkout. Messages already name `scripts/mosaic queue`. + +```sh +node packages/queue/src/cli.mjs list +node packages/queue/src/cli.mjs next darkwing +node packages/queue/src/cli.mjs move 9 in-progress --op darkwing-9-start-1 --by darkwing +node packages/queue/src/cli.mjs verify --current +node --test packages/queue/tests/ +scripts/test-queue.sh +``` + +## Verbs + +| Verb | Lock | Writes | +|---|---|---| +| `list`, `show ID`, `next [SEAT]` | no (see "Unlocked reads") | nothing | +| `add`, `move`, `release`, `assign`, `note`, `set` | yes | one log entry, the witness, the table | +| `genesis`, `accept-history` | yes, privileged | one log entry, the witness, the table | +| `render` | yes | the table, only when it is stale | +| `render --check`, `verify [--current]` | yes | nothing | +| `verify --snapshot DIR` | no | nothing; reads only the files it is given | +| `sync [--op ID]` | yes | fsyncs and the witness; logs nothing | +| `snapshot --out DIR` | yes | copies of both files into an empty DIR outside the repository | +| `unlock [--check-gate]` | no; takes the unlock gate | removes a dead or mismatched lock | + +Every change takes `--op ID` (8 to 72 characters of `[a-z0-9._-]`, +starting with a letter or digit, not ending in `.outcome`) and an actor from `--by NAME` or +`$MOSAIC_AGENT_NAME`. Reads need no actor. `next` with neither a SEAT nor +`$MOSAIC_AGENT_NAME` refuses. The privileged actors are `jason` and `sage`. + +Exit codes: 0 ok; 1 the operation failed; 2 invalid data or refused; +3 uncertain (visible or durable, not acknowledged); 4 usage. + +Until piece D, `move ID in-review` needs `--candidate`: an existing file is +read as a manifest (one ` ` line per file), anything else as a +commit reachable from `refs/heads` or `refs/tags`. The candidate is frozen +for the round. The review's issue is the row's first issue. + +## Where the files live + +- `docs/plans/queue.json`: `{version, canonicalRoot, revision, rows, log}`, + serialized deterministically. A file that doesn't re-serialize byte for + byte, or doesn't replay from genesis to its `rows`, is refused by every + verb. +- `docs/plans/QUEUE.md`: hand-written header, generated body between the + markers. Each log entry records the SHA-256 of the body it rendered, so a + body is `current`, `stale` (an earlier render) or `unknown` (no render). +- `.git/mosaic-queue.head`: the witness, `{revision, logDigest, fileSha, + at}` for the last write whose directory fsync succeeded. It holds no rows. +- `.git/mosaic-queue.lock` and `.git/mosaic-queue.unlock`: the lock and the + unlock gate. + +## Canonical checkout only + +Every verb except `verify --snapshot` refuses unless the working directory's +toplevel is the `canonicalRoot` recorded at genesis, its git directory is +`/.git` (a linked worktree fails), HEAD names the genesis branch, and +the CLI's own file lies under the root. `GIT_DIR`, `GIT_WORK_TREE` and +`GIT_COMMON_DIR` refuse. A symlink to the root works. Only one host is +supported. + +## Trust boundary + +The queue is cooperative (J2). An actor is whoever `--by` or +`$MOSAIC_AGENT_NAME` claims. The CLI checks each claimed actor against the +transition matrix and records it, and replay rechecks every entry, but +nothing stops a seat from claiming another name or writing the file with +other tools. Detection is what the queue offers: replay catches a hand edit, +the witness catches a rollback, and piece E lists changes for review. +Enforcement needs a boundary outside #1508. + +## The git side of the protocol + +The lock only binds the queue CLI. Git doesn't honour it. So these rules +are protocol, not enforcement: + +- While the working `queue.json` has ops not yet committed, no one runs + `checkout`, `stash`, `restore`, `reset` or a branch switch that touches + `docs/plans/queue.json` or `docs/plans/QUEUE.md`. Only the lead restores, + and only by the manual recovery below. +- Seats commit with plain `git commit`, never `--no-verify`, so the queue + guard runs. +- Merge, rebase, cherry-pick, revert and `am` skip the guard. On the queue + branch they are the lead's. +- Nobody disables, replaces or overrides the guard: no `core.hooksPath` in + any scope, no `-c core.hooksPath=`, no `GIT_CONFIG_*` environment, no edit + to `.git/hooks/pre-commit` and no change to its mode. + +The witness detects lost history afterwards. It can't prevent a same-user +git write. A passing replay shows the file agrees with itself, not that no +history was lost. + +## Committing the queue + +The CLI never commits. The lead runs `scripts/queue-commit.sh -m MSG`. It +commits exactly the snapshot bytes of the two queue files on top of HEAD +through a temporary index and `commit-tree`, so nothing else that is staged +is swept in: + +1. Record H, then check the guard is active: `.git/hooks/pre-commit` is a + regular executable file owned by the user with HEAD's bytes, no + `core.hooksPath` is set in any scope, and a canary run of `git hook run + pre-commit` on a temporary index passes clean and refuses a changed + queue entry. Refuse if a queue path is staged. +2. `queue snapshot` under the lock. +3. The base is `H:docs/plans/queue.json` from the object database. +4. HEAD's archived code, unpacked outside any repository, runs its queue + tests and `verify --snapshot` against the base. +5. to 7. Blobs, a tree from a temporary index, `commit-tree`, the guard + checks again, then `update-ref` with H as the expected old value. If the + branch moved at any point after step 1, this fails and nothing is + published. +8. Reconcile the shared index's two queue entries with `git reset -q -- + docs/plans/queue.json docs/plans/QUEUE.md`, only if HEAD is the new + commit and those entries are still H's. Otherwise exit 3 and print the + command. Until it runs, the guard refuses ordinary commits. + +`scripts/git-hooks/pre-commit` is the guard. `scripts/queue-commit.sh +--install-hook` installs it (jason or sage). It refuses any `git commit` +whose index entries for the two queue files differ from HEAD's. + +Bootstrap: the implementation lands with no `queue.json`; the lead installs +the guard, runs `queue genesis` from the reviewed migration map, then +`scripts/queue-commit.sh --genesis -m MSG`. Every mutating verb refuses with +`genesis not committed` until that commit exists. After that, +`scripts/test-queue.sh` also runs `verify` on the live queue. + +Committing still needs its own authorization. The script never pushes. + +## Durability + +Supported: Linux with `docs/plans/` and `.git/` on local ext4, xfs or btrfs +(tmpfs for tests). The CLI checks the filesystem type and refuses others. +The host-crash claims assume that rename replaces its target atomically and +that fsync of a directory persists the renamed entry. The tests kill +processes with SIGKILL at each step. **Power loss is not tested.** + +A write has three separate points: + +| Point | When | Meaning | +|---|---|---| +| visible | the rename returns | readers see the new revision | +| durable | the directory fsync succeeds and the witness is written | it survives a host crash, within the assumptions above | +| acknowledged | the receipt `ok rev N ...` prints | the caller may treat the op as done | + +**The unacknowledged-op rule.** Only an op whose receipt printed is done. +If a call exits 3, is killed, or prints nothing, retry the same op id with +the same arguments. A recorded op answers with its original receipt and +changes nothing. A different verb or arguments under a used op id refuses. +`queue sync` confirms a visible tail and prints each op that became durable +without being acknowledged. Tell the seat named there. + +## Unlocked reads + +`list`, `show` and `next` read the witness, then `queue.json`, without the +lock. A file ahead of the witness prints `rev N visible, not confirmed +durable`. Any other disagreement takes the lock and rechecks before +reporting, so a write in progress is never reported as lost history. + +## Known windows and limits + +- **Check to rename.** A write checks that `queue.json` is unchanged since + it was read, then renames its temp file over it. A git write between the + check and the rename is not caught by the check. The witness catches the + result afterwards. The table write has the same window: a hand edit to + QUEUE.md between its check and its rename can be overwritten. +- **Same-seat claims.** A claim is the seat name only. Two sessions of the + same seat are one claimant; the queue can't tell them apart and doesn't + try. +- **Snapshot verify** checks that the given pair is valid, that the table + is its render, and that its log extends the base. It does not certify + witness continuity. Canonical `verify`, under the lock, does that. +- **Commit candidates** stay retrievable only while some ref keeps the + commit. The queue keeps the manifest text, not the source bytes. +- **Locks** are never removed for their age. `unknown` and `invalid` locks + wait for a person. A host rename makes old locks `unknown`. +- `queue unlock --check-gate` classifies a stale unlock gate. Remove the + gate by hand only after it says `dead` or `mismatch` and no queue command + is running. + +## Manual recovery + +For an invalid `queue.json`, lost history (the file doesn't extend the +witness) or a missing witness. No verb repairs anything; `accept-history` +is the only verb that runs in these states, and a file holding genesis +alone with no witness can also be confirmed by `sync`. + +1. Copy the current `queue.json` bytes to + `agents//work/queue-recovery-/` before touching anything. + Restoring HEAD would discard every op written since the last queue + commit. +2. Find the lost ops, the entries after the last committed revision, in the + preserved bytes, `git stash list`, the reflog, the receipts seats hold, + and issue comments carrying `mosaic-queue-op` markers. +3. Account for external effects. A lost op id is no longer deduplicated, + and reusing it could act again. +4. Put a valid file in place: the restored version with the lost ops + re-applied under new op ids, or a reviewed hand-built file. +5. Run `queue accept-history --op ID --reason TEXT --yes` as a privileged + actor. It records the old witness (or its absence), the new revision and + the reason, then rewrites the witness. The at-most-once guarantee does + not hold for the lost range, and the entry says so. + +A stale table needs a person to look, then `queue render`. An unknown table +needs the table restored with git, or the edit re-applied as queue ops, +then `queue render`. + +## Tests + +`node --test packages/queue/tests/` runs everything in scratch repositories +under the system temp directory; nothing touches this checkout's `.git`. +Faults reach the code only through options the tests pass in (`io`, `proc`, +`hook`, `now`, `readOrder`, `lockWaitMs`); the CLI passes none. + +- `data.test.mjs`: serialization, replay, the transition matrix, `next` + ordering, render. +- `lock.test.mjs`: every lock and unlock schedule in 8.4. +- `store.test.mjs`: genesis, canonical checks, op ids, claims, briefs, + views, snapshot. +- `write.test.mjs`: every write fault, SIGKILL at each step, git + interference, unlocked-read races. +- `commit.test.mjs`: `queue-commit.sh` and the guard, through PATH shims + that run an action at an exact point in the procedure. diff --git a/packages/queue/src/cli.mjs b/packages/queue/src/cli.mjs new file mode 100644 index 00000000..c5517747 --- /dev/null +++ b/packages/queue/src/cli.mjs @@ -0,0 +1,204 @@ +#!/usr/bin/env node +// Usage: queue [args] (A2 adds `scripts/mosaic queue` in front). +// +// Reads: list | show ID | next [SEAT] +// Changes: add --piece TEXT --gate TEXT --brief PATH#ANCHOR [--issue N]... [--note TEXT] +// [--owner SEAT] [--gate-owner SEAT] [--after ID[:settled]]... [--reviewer SEAT]... [--required] +// move ID STATE [--reason TEXT] [--candidate COMMIT|MANIFEST] [--evidence TEXT] +// release ID | assign ID SEAT | note ID TEXT | set ID FIELD VALUE [--reason TEXT] +// genesis --root PATH --branch NAME --map PATH +// accept-history --reason TEXT --yes +// Each needs --op ID and an actor (--by NAME or $MOSAIC_AGENT_NAME). +// No log: render [--check] | verify [--current] | verify --snapshot DIR (--base-file F | --base-absent) +// sync [--op ID] | snapshot --out DIR | unlock [--check-gate] +// +// Exit codes: 0 ok; 1 operation failed; 2 invalid data or refused; +// 3 uncertain (visible or durable, not acknowledged); 4 usage. +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { QueueError } from "./errors.mjs"; +import { SET_FIELDS } from "./queue.mjs"; +import { list, mutate, next, renderView, show, snapshot, sync, unlock, verify, verifySnapshot } from "./store.mjs"; + +const USAGE = [ + "usage: queue list | show ID | next [SEAT]", + " queue add --op ID --piece TEXT --gate TEXT --brief PATH#ANCHOR [--issue N]... [--note TEXT] [--owner SEAT] [--gate-owner SEAT] [--after ID[:settled]]... [--reviewer SEAT]... [--required]", + " queue move ID STATE --op ID [--reason TEXT] [--candidate COMMIT|MANIFEST] [--evidence TEXT]", + " queue release ID --op ID | assign ID SEAT --op ID | note ID TEXT --op ID", + ` queue set ID FIELD VALUE --op ID [--reason TEXT] (fields: ${SET_FIELDS.join(", ")})`, + " queue genesis --op ID --root PATH --branch NAME --map PATH", + " queue accept-history --op ID --reason TEXT --yes", + " queue render [--check] | verify [--current] | verify --snapshot DIR (--base-file F | --base-absent)", + " queue sync [--op ID] | snapshot --out DIR | unlock [--check-gate]", + " every change takes --by NAME, else $MOSAIC_AGENT_NAME", +].join("\n"); + +const VALUE_FLAGS = new Set([ + "--op", "--by", "--piece", "--gate", "--brief", "--issue", "--note", "--owner", "--gate-owner", "--after", "--reviewer", + "--reason", "--candidate", "--evidence", "--root", "--branch", "--map", "--snapshot", "--base-file", "--out", +]); +const REPEATED = new Set(["--issue", "--after", "--reviewer"]); +const BOOL_FLAGS = new Set(["--required", "--yes", "--check", "--current", "--base-absent", "--check-gate"]); + +function usage(message) { + return new QueueError(message, 4); +} + +export function parseArgs(argv) { + const pos = []; + const flags = new Map(); + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--") { pos.push(...argv.slice(i + 1)); break; } + if (VALUE_FLAGS.has(a)) { + if (i + 1 >= argv.length) throw usage(`missing value for ${a}`); + const v = argv[++i]; + if (REPEATED.has(a)) flags.set(a, [...(flags.get(a) ?? []), v]); + else if (flags.has(a)) throw usage(`${a} given twice`); + else flags.set(a, v); + } else if (BOOL_FLAGS.has(a)) { + flags.set(a, true); + } else if (a.startsWith("--")) { + throw usage(`unknown option ${a}`); + } else { + pos.push(a); + } + } + return { pos, flags }; +} + +function allow(flags, names) { + for (const k of flags.keys()) if (!names.includes(k)) throw usage(`${k} does not apply here`); +} + +function positional(pos, n, what) { + if (pos.length !== n) throw usage(`expected ${what}`); +} + +function intArg(v, what) { + if (!/^[1-9][0-9]{0,6}$/.test(v)) throw usage(`${what} must be a positive integer: ${JSON.stringify(v)}`); + return Number(v); +} + +function afterArg(v) { + const m = /^([1-9][0-9]{0,6})(?::(done|settled))?$/.exec(v); + if (!m) throw usage(`--after takes ID or ID:settled: ${JSON.stringify(v)}`); + return { id: Number(m[1]), when: m[2] ?? "done" }; +} + +function listArg(v) { + return v === "" || v === "none" ? [] : v.split(","); +} + +function setValue(field, v) { + switch (field) { + case "after": return listArg(v).map(afterArg); + case "reviewers": return listArg(v); + case "issues": case "closes": return listArg(v).map((n) => intArg(n.replace(/^#/, ""), field)); + case "required": + if (v !== "true" && v !== "false") throw usage("required takes true or false"); + return v === "true"; + default: return v; + } +} + +const CHANGE = ["--op", "--by"]; + +export function run(argv, opts = {}) { + const [verb, ...rest] = argv; + if (verb === undefined || verb === "--help" || verb === "-h" || verb === "help") return { out: [USAGE], err: [], code: verb === undefined ? 4 : 0 }; + const { pos, flags } = parseArgs(rest); + const f = (k) => flags.get(k) ?? null; + const change = (v, args, extra = {}) => mutate(opts, { verb: v, op: f("--op"), args, by: f("--by"), ...extra }); + switch (verb) { + case "list": + allow(flags, []); positional(pos, 0, "no arguments"); + return list(opts); + case "show": + allow(flags, []); positional(pos, 1, "show ID"); + return show(opts, intArg(pos[0], "ID")); + case "next": + allow(flags, []); + if (pos.length > 1) throw usage("expected next [SEAT]"); + return next(opts, pos[0] ?? null); + case "add": { + allow(flags, [...CHANGE, "--piece", "--gate", "--brief", "--issue", "--note", "--owner", "--gate-owner", "--after", "--reviewer", "--required"]); + positional(pos, 0, "only options"); + for (const k of ["--piece", "--gate", "--brief"]) if (f(k) === null) throw usage(`add needs ${k}`); + return change("add", { + piece: f("--piece"), gate: f("--gate"), brief: f("--brief"), + issues: (flags.get("--issue") ?? []).map((n) => intArg(n.replace(/^#/, ""), "--issue")), + note: f("--note"), owner: f("--owner"), gateOwner: f("--gate-owner"), + after: flags.has("--after") ? flags.get("--after").map(afterArg) : null, + reviewers: flags.get("--reviewer") ?? null, required: flags.has("--required") ? true : null, + }); + } + case "move": + allow(flags, [...CHANGE, "--reason", "--candidate", "--evidence"]); positional(pos, 2, "move ID STATE"); + return change("move", { id: intArg(pos[0], "ID"), to: pos[1], reason: f("--reason"), candidate: f("--candidate"), evidence: f("--evidence") }); + case "release": + allow(flags, CHANGE); positional(pos, 1, "release ID"); + return change("release", { id: intArg(pos[0], "ID") }); + case "assign": + allow(flags, CHANGE); positional(pos, 2, "assign ID SEAT"); + return change("assign", { id: intArg(pos[0], "ID"), seat: pos[1] }); + case "note": + allow(flags, CHANGE); positional(pos, 2, "note ID TEXT"); + return change("note", { id: intArg(pos[0], "ID"), text: pos[1] }); + case "set": + allow(flags, [...CHANGE, "--reason"]); positional(pos, 3, "set ID FIELD VALUE"); + if (!SET_FIELDS.includes(pos[1])) throw usage(`set fields: ${SET_FIELDS.join(", ")}`); + return change("set", { id: intArg(pos[0], "ID"), field: pos[1], value: setValue(pos[1], pos[2]), reason: f("--reason") }); + case "genesis": + allow(flags, [...CHANGE, "--root", "--branch", "--map"]); positional(pos, 0, "only options"); + for (const k of ["--root", "--branch", "--map"]) if (f(k) === null) throw usage(`genesis needs ${k}`); + return change("genesis", { root: f("--root"), branch: f("--branch"), map: f("--map") }); + case "accept-history": + allow(flags, [...CHANGE, "--reason", "--yes"]); positional(pos, 0, "only options"); + if (f("--reason") === null) throw usage("accept-history needs --reason"); + return change("accept-history", { reason: f("--reason") }, { yes: flags.has("--yes") }); + case "render": + allow(flags, ["--check"]); positional(pos, 0, "no arguments"); + return renderView(opts, { check: flags.has("--check") }); + case "verify": + if (flags.has("--snapshot")) { + allow(flags, ["--snapshot", "--base-file", "--base-absent"]); positional(pos, 0, "only options"); + return verifySnapshot(f("--snapshot"), { baseFile: f("--base-file"), baseAbsent: flags.has("--base-absent") }); + } + allow(flags, ["--current"]); positional(pos, 0, "no arguments"); + return verify(opts, { current: flags.has("--current") }); + case "sync": + allow(flags, ["--op"]); positional(pos, 0, "no arguments"); + return sync(opts, f("--op")); + case "snapshot": + allow(flags, ["--out"]); positional(pos, 0, "only options"); + if (f("--out") === null) throw usage("snapshot needs --out DIR"); + return snapshot(opts, f("--out")); + case "unlock": + allow(flags, ["--check-gate"]); positional(pos, 0, "no arguments"); + return unlock(opts, { checkGateOnly: flags.has("--check-gate") }); + default: + throw usage(`unknown verb ${JSON.stringify(verb)}\n${USAGE}`); + } +} + +export function main(argv) { + let res; + try { + res = run(argv); + } catch (err) { + if (err instanceof QueueError) { + process.stderr.write(`queue: ${err.message}\n`); + return err.code; + } + process.stderr.write(`queue: ${err?.stack ?? err}\n`); + return 1; + } + for (const line of res.err) process.stderr.write(`${line}\n`); + for (const line of res.out) process.stdout.write(`${line}\n`); + return res.code; +} + +if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) { + process.exitCode = main(process.argv.slice(2)); +} diff --git a/packages/queue/src/errors.mjs b/packages/queue/src/errors.mjs new file mode 100644 index 00000000..37637994 --- /dev/null +++ b/packages/queue/src/errors.mjs @@ -0,0 +1,9 @@ +// Exit codes: 0 ok; 1 operation failed; 2 invalid data or refused; +// 3 uncertain (visible or durable, not acknowledged); 4 usage. +export class QueueError extends Error { + constructor(message, code = 1) { + super(message); + this.name = "QueueError"; + this.code = code; + } +} diff --git a/packages/queue/src/io.mjs b/packages/queue/src/io.mjs new file mode 100644 index 00000000..6ebb63cb --- /dev/null +++ b/packages/queue/src/io.mjs @@ -0,0 +1,104 @@ +// The file layer every queue write goes through. `realIo` is the only layer +// the CLI uses. Tests pass their own layer through the API to inject one +// fault at a time (8.5); no flag or environment variable selects it. +import { + closeSync, constants, fsyncSync, linkSync, lstatSync, openSync, readFileSync, renameSync, statSync, + statfsSync, unlinkSync, writeSync, +} from "node:fs"; +import { QueueError } from "./errors.mjs"; + +export const realIo = Object.freeze({ + openExcl: (path, mode) => openSync(path, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, mode), + openRead: (path) => openSync(path, constants.O_RDONLY), + write: (fd, buf, off, len) => writeSync(fd, buf, off, len), + fsync: (fd) => fsyncSync(fd), + close: (fd) => closeSync(fd), + readFile: (path) => readFileSync(path), + rename: (from, to) => renameSync(from, to), + unlink: (path) => unlinkSync(path), + link: (from, to) => linkSync(from, to), + stat: (path) => statSync(path, { bigint: true }), + lstat: (path) => lstatSync(path, { bigint: true }), + fsyncDir: (dir) => { + const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY); + try { fsyncSync(fd); } finally { closeSync(fd); } + }, + statfsType: (path) => statfsSync(path).type, +}); + +// ext4 (ext2/ext3 share the magic), xfs, btrfs; tmpfs for tests. +const FS_TYPES = new Map([[0xef53, "ext4"], [0x58465342, "xfs"], [0x9123683e, "btrfs"], [0x01021994, "tmpfs"]]); + +export function checkPlatform(io, dirs) { + if (process.platform !== "linux") throw new QueueError(`unsupported platform ${process.platform}: the queue runs on Linux only`, 2); + for (const dir of dirs) { + const type = io.statfsType(dir); + if (!FS_TYPES.has(type)) throw new QueueError(`unsupported filesystem under ${dir} (type 0x${type.toString(16)}); ext4, xfs, btrfs or tmpfs only`, 2); + } +} + +export function sleepMs(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +// Loops on short writes and checks the total. +export function writeAll(io, fd, bytes) { + let off = 0; + while (off < bytes.length) { + const n = io.write(fd, bytes, off, bytes.length - off); + if (!Number.isInteger(n) || n <= 0) throw Object.assign(new Error(`short write: ${off} of ${bytes.length} bytes`), { code: "ESHORT" }); + off += n; + } + if (off !== bytes.length) throw Object.assign(new Error(`wrote ${off} of ${bytes.length} bytes`), { code: "ESHORT" }); +} + +// O_EXCL temp file, every byte written, fsync, close. On any failure the +// temp file is removed and the error rethrown. +export function writeTemp(io, path, bytes, mode = 0o644) { + const fd = io.openExcl(path, mode); + try { + writeAll(io, fd, bytes); + io.fsync(fd); + } catch (err) { + try { io.close(fd); } catch { /* already failing */ } + unlinkQuiet(io, path); + throw err; + } + try { + io.close(fd); + } catch (err) { + unlinkQuiet(io, path); + throw err; + } +} + +export function fsyncFile(io, path) { + const fd = io.openRead(path); + try { io.fsync(fd); } finally { io.close(fd); } +} + +export function unlinkQuiet(io, path) { + try { io.unlink(path); } catch { /* absent or not ours to report */ } +} + +export function readOrNull(io, path) { + try { + return io.readFile(path); + } catch (err) { + if (err.code === "ENOENT") return null; + throw err; + } +} + +export function lstatOrNull(io, path) { + try { + return io.lstat(path); + } catch (err) { + if (err.code === "ENOENT") return null; + throw err; + } +} + +export function errno(err) { + return err?.code ?? err?.message ?? String(err); +} diff --git a/packages/queue/src/lock.mjs b/packages/queue/src/lock.mjs new file mode 100644 index 00000000..56c43f0f --- /dev/null +++ b/packages/queue/src/lock.mjs @@ -0,0 +1,180 @@ +// The queue lock (8.4): /.git/mosaic-queue.lock, published by link() +// from an O_EXCL temp file, and the unlock gate /.git/mosaic-queue.unlock +// made the same way. Nothing is ever removed because of its age. Only +// `unlock` removes another process's lock, and only a dead or mismatched one. +import { randomBytes } from "node:crypto"; +import { hostname } from "node:os"; +import { join } from "node:path"; +import { bootId, pidAlive, processStart, validBoot, validStart } from "../../discord/src/journal.mjs"; +import { QueueError } from "./errors.mjs"; +import { errno, lstatOrNull, readOrNull, sleepMs, unlinkQuiet, writeAll } from "./io.mjs"; + +export const LOCK_NAME = "mosaic-queue.lock"; +export const GATE_NAME = "mosaic-queue.unlock"; + +export const realProc = Object.freeze({ + pid: process.pid, + host: () => hostname(), + processStart: (pid) => processStart(pid), + bootId: () => bootId(), + pidAlive: (pid) => pidAlive(pid), +}); + +const RECORD_KEYS = ["pid", "start", "boot", "host", "op", "verb", "at"]; + +function ownRecord(proc, op, verb) { + const start = proc.processStart(proc.pid); + const boot = proc.bootId(); + if (start === null || boot === null) throw new QueueError("cannot read this process's start time or boot id from /proc; refusing to take the lock", 2); + return { pid: proc.pid, start, boot, host: proc.host(), op: op ?? null, verb, at: new Date().toISOString() }; +} + +export function parseRecord(bytes) { + if (bytes === null || bytes.length === 0) return null; + let rec; + try { rec = JSON.parse(bytes.toString("utf8")); } catch { return null; } + if (rec === null || typeof rec !== "object" || Array.isArray(rec)) return null; + if (Object.keys(rec).join() !== RECORD_KEYS.join()) return null; + if (!Number.isInteger(rec.pid) || rec.pid <= 0) return null; + if (!validStart(rec.start) || !validBoot(rec.boot)) return null; + if (typeof rec.host !== "string" || rec.host.length === 0) return null; + if (rec.op !== null && typeof rec.op !== "string") return null; + if (typeof rec.verb !== "string" || typeof rec.at !== "string") return null; + return rec; +} + +// The 8.4 table, in order; the first test that applies decides. +export function classify(bytes, proc) { + const rec = parseRecord(bytes); + if (rec === null) return { state: "invalid", rec: null }; + const selfStart = proc.processStart(proc.pid); + const selfBoot = proc.bootId(); + if (selfStart === null || selfBoot === null) return { state: "unknown", rec, why: "this process cannot read its own /proc identity" }; + if (rec.host !== proc.host()) return { state: "unknown", rec, why: `recorded on host ${rec.host}` }; + if (rec.boot !== selfBoot) return { state: "mismatch", rec, why: "recorded in a previous boot" }; + if (!proc.pidAlive(rec.pid)) return { state: "dead", rec }; + const start = proc.processStart(rec.pid); + if (start === null) return { state: "unknown", rec, why: `pid ${rec.pid} is alive but its start time is unreadable` }; + if (start !== rec.start) return { state: "mismatch", rec, why: `pid ${rec.pid} was reused` }; + return { state: "live", rec }; +} + +function describe(c) { + if (c.rec === null) return c.state; + const r = c.rec; + return `${c.state}: pid ${r.pid}, ${r.verb}${r.op ? ` ${r.op}` : ""} since ${r.at}${c.why ? `; ${c.why}` : ""}`; +} + +// Temp file with the whole record, fsynced, read back, then link()ed to +// `target`. Returns {linked, dev, ino, bytes}; linked is false on EEXIST. +function publish(io, target, bytes, hook, waitMs, stepMs) { + const tmp = `${target}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; + let fd; + try { + fd = io.openExcl(tmp, 0o600); + } catch (err) { + throw new QueueError(`cannot create ${tmp}: ${errno(err)}`, 1); + } + try { + writeAll(io, fd, bytes); + io.fsync(fd); + io.close(fd); + fd = null; + const back = io.readFile(tmp); + if (!back.equals(bytes)) throw Object.assign(new Error("read-back differs from the record"), { code: "EREADBACK" }); + } catch (err) { + if (fd !== null) { try { io.close(fd); } catch { /* already failing */ } } + unlinkQuiet(io, tmp); + throw new QueueError(`cannot write the lock record ${tmp}: ${errno(err)}; no lock taken`, 1); + } + hook("lock-temp-written"); + const deadline = Date.now() + waitMs; + try { + for (;;) { + try { + io.link(tmp, target); + } catch (err) { + if (err.code !== "EEXIST") throw new QueueError(`cannot link ${target}: ${errno(err)}; no lock taken`, 1); + if (Date.now() >= deadline) return { linked: false }; + sleepMs(stepMs); + continue; + } + const st = io.stat(tmp); + return { linked: true, dev: st.dev, ino: st.ino, bytes }; + } + } finally { + unlinkQuiet(io, tmp); + } +} + +export function acquire({ gitDir, io, proc = realProc, op = null, verb, waitMs = 10000, stepMs = 100, hook = () => {} }) { + const path = join(gitDir, LOCK_NAME); + const rec = ownRecord(proc, op, verb); + const bytes = Buffer.from(JSON.stringify(rec) + "\n"); + const got = publish(io, path, bytes, hook, waitMs, stepMs); + if (!got.linked) { + const c = classify(readOrNull(io, path), proc); + if (c.state === "live") throw new QueueError(`queue lock held by ${c.rec.verb}${c.rec.op ? ` ${c.rec.op}` : ""} since ${c.rec.at}; retry the same op later`, 2); + if (c.state === "dead" || c.state === "mismatch") throw new QueueError(`queue lock owner is ${describe(c)}; run \`scripts/mosaic queue unlock\` once nothing is running`, 2); + if (c.state === "unknown") throw new QueueError(`queue lock owner is ${describe(c)}; unlock refuses this too; diagnose ${path} by hand`, 2); + throw new QueueError(`queue lock record is invalid; inspect ${path} by hand`, 2); + } + const handle = { path, dev: got.dev, ino: got.ino, bytes: got.bytes }; + hook("lock-linked"); + const gate = join(gitDir, GATE_NAME); + if (lstatOrNull(io, gate) !== null) { + const c = classify(readOrNull(io, gate), proc); + release(handle, io); + throw new QueueError(`unlock gate ${gate} is present (${describe(c)}); check it with \`scripts/mosaic queue unlock --check-gate\``, 2); + } + return handle; +} + +// Unlinks only the lock this process published: same inode, same record. +export function release(handle, io) { + const st = lstatOrNull(io, handle.path); + if (st === null) return `lock ${handle.path} was already gone`; + const bytes = readOrNull(io, handle.path); + if (st.dev !== handle.dev || st.ino !== handle.ino || bytes === null || !bytes.equals(handle.bytes)) { + return `lock ${handle.path} is not the one this process took; left in place`; + } + io.unlink(handle.path); + return null; +} + +export function unlock({ gitDir, io, proc = realProc, hook = () => {} }) { + const lockPath = join(gitDir, LOCK_NAME); + const gatePath = join(gitDir, GATE_NAME); + const rec = ownRecord(proc, null, "unlock"); + const bytes = Buffer.from(JSON.stringify(rec) + "\n"); + const got = publish(io, gatePath, bytes, () => {}, 0, 0); + if (!got.linked) { + const c = classify(readOrNull(io, gatePath), proc); + throw new QueueError(`unlock gate ${gatePath} is held (${describe(c)}); another unlock is running or a stale gate needs \`--check-gate\``, 2); + } + const gate = { path: gatePath, dev: got.dev, ino: got.ino, bytes }; + hook("gate-held"); + try { + const lockBytes = readOrNull(io, lockPath); + if (lockBytes === null) return "no queue lock present; nothing removed"; + const c = classify(lockBytes, proc); + if (c.state !== "dead" && c.state !== "mismatch") { + throw new QueueError(`queue lock owner is ${describe(c)}; unlock refuses`, 2); + } + io.unlink(lockPath); + return `removed queue lock (${describe(c)}): ${lockBytes.toString("utf8").trim()}`; + } finally { + release(gate, io); + } +} + +export function checkGate({ gitDir, io, proc = realProc }) { + const gatePath = join(gitDir, GATE_NAME); + const bytes = readOrNull(io, gatePath); + if (bytes === null) return { state: "absent", line: "no unlock gate present" }; + const c = classify(bytes, proc); + const advice = c.state === "dead" || c.state === "mismatch" + ? `; remove ${gatePath} by hand only once no queue command is running` + : "; leave it for diagnosis"; + return { state: c.state, line: `unlock gate ${describe(c)}${advice}` }; +} diff --git a/packages/queue/src/queue.mjs b/packages/queue/src/queue.mjs new file mode 100644 index 00000000..9999ba18 --- /dev/null +++ b/packages/queue/src/queue.mjs @@ -0,0 +1,918 @@ +// The queue as data (8.2, 8.6, 8.7, 8.8): schema, deterministic +// serialization, the one transition and permission matrix, replay from +// genesis, `next`, and the rendered table. Everything here is pure: no file, +// git or clock access. The write path lives in store.mjs. +import { createHash } from "node:crypto"; +import { QueueError } from "./errors.mjs"; + +export const VERSION = 1; +export const SEMANTICS = 1; +export const STATES = ["queued", "briefed", "in-progress", "in-review", "waiting-on-jason", "done", "blocked", "parked"]; +const NON_TERMINAL = new Set(["queued", "briefed", "in-progress", "in-review", "waiting-on-jason"]); +const CLAIM_KEPT = new Set(["in-progress", "in-review", "waiting-on-jason"]); +export const PRIVILEGED = new Set(["jason", "sage"]); +export const VERBS = ["genesis", "add", "move", "release", "assign", "note", "set", "accept-history"]; +export const SET_FIELDS = ["piece", "gate", "gate-owner", "after", "reviewers", "issues", "closes", "brief", "required"]; + +const NAME_RE = /^[a-z][a-z0-9-]{0,31}$/; +export const CALLER_OP_RE = /^[a-z0-9][a-z0-9._-]{7,71}$/; +export const LOG_OP_RE = /^[a-z0-9][a-z0-9._-]{7,79}$/; +const ISO_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; +const BLOB_RE = /^[0-9a-f]{40}$/; +const SHA256_RE = /^[0-9a-f]{64}$/; +const PATH_RE = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/; +const BRANCH_RE = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/; +// C0, DEL, C1, U+2028 and U+2029: none belongs in a one-line field. +const BAD_TEXT_RE = new RegExp(`[${String.fromCharCode(0)}-${String.fromCharCode(0x1f)}${String.fromCharCode(0x7f)}-${String.fromCharCode(0x9f)}${String.fromCharCode(0x2028, 0x2029)}]`); + +export const DOC_KEYS = ["version", "canonicalRoot", "revision", "rows", "log"]; +export const ROW_KEYS = [ + "id", "piece", "owner", "issues", "closes", "state", "previousState", "required", "requiredSince", "gate", "gateOwner", + "brief", "after", "reviewers", "review", "claim", "note", "blockedReason", "createdAt", "updatedAt", "updatedBy", +]; +export const ENTRY_KEYS = ["rev", "op", "verb", "args", "by", "at", "semantics", "result", "viewSha"]; +const MAP_ROW_KEYS = [ + "id", "piece", "owner", "issues", "closes", "state", "previousState", "required", "requiredSince", "gate", "gateOwner", + "brief", "after", "reviewers", "note", "blockedReason", "createdAt", +]; + +export const BEGIN_MARKER = ""; +export const END_MARKER = ""; + +function refuse(message) { + return new QueueError(message, 2); +} + +export function sha256(data) { + return createHash("sha256").update(data).digest("hex"); +} + +// Git's blob id for bytes, computed in-process with no git write (8.13). +export function gitBlobId(bytes) { + return createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex"); +} + +// --- field checks --- + +function isObj(v) { + return v !== null && typeof v === "object" && !Array.isArray(v); +} + +function keysExactly(obj, keys, what) { + if (!isObj(obj)) throw refuse(`${what} is not an object`); + const got = Object.keys(obj); + if (got.join("\u0001") !== keys.join("\u0001")) throw refuse(`${what} keys must be exactly ${keys.join(", ")} in that order; got ${got.join(", ")}`); +} + +export function checkText(v, what, { max = 500, empty = false } = {}) { + if (typeof v !== "string") throw refuse(`${what} must be text`); + if (!empty && v.length === 0) throw refuse(`${what} must not be empty`); + if (v.length > max) throw refuse(`${what} is longer than ${max} characters`); + if (BAD_TEXT_RE.test(v)) throw refuse(`${what} must be one line with no control characters`); + if (v !== v.trim()) throw refuse(`${what} must not start or end with whitespace`); + return v; +} + +export function checkName(v, what) { + if (typeof v !== "string" || !NAME_RE.test(v)) throw refuse(`${what} must be a seat name (lowercase letters, digits, hyphens): ${JSON.stringify(v)}`); + return v; +} + +function checkId(v, what = "row id") { + if (!Number.isInteger(v) || v < 1 || v > 1_000_000) throw refuse(`${what} must be a positive integer: ${JSON.stringify(v)}`); + return v; +} + +function checkIssues(v, what) { + if (!Array.isArray(v)) throw refuse(`${what} must be a list`); + for (const n of v) if (!Number.isInteger(n) || n < 1) throw refuse(`${what} must hold positive issue numbers`); + for (let i = 1; i < v.length; i++) if (v[i] <= v[i - 1]) throw refuse(`${what} must be ascending with no repeats`); + return v; +} + +function checkNames(v, what) { + if (!Array.isArray(v)) throw refuse(`${what} must be a list`); + v.forEach((n) => checkName(n, what)); + for (let i = 1; i < v.length; i++) if (v[i] <= v[i - 1]) throw refuse(`${what} must be sorted with no repeats`); + return v; +} + +function checkAfter(v, what = "after") { + if (!Array.isArray(v)) throw refuse(`${what} must be a list`); + for (const a of v) { + keysExactly(a, ["id", "when"], `${what} entry`); + checkId(a.id, `${what} id`); + if (a.when !== "done" && a.when !== "settled") throw refuse(`${what} when must be done or settled`); + } + for (let i = 1; i < v.length; i++) if (v[i].id <= v[i - 1].id) throw refuse(`${what} must be sorted by id with no repeats`); + return v; +} + +function checkTime(v, what, { unknown = false, date = false } = {}) { + if (unknown && v === "unknown") return v; + if (typeof v === "string" && (ISO_RE.test(v) || (date && DATE_RE.test(v)))) return v; + throw refuse(`${what} must be an ISO time${date ? " or date" : ""}${unknown ? " or \"unknown\"" : ""}: ${JSON.stringify(v)}`); +} + +export function checkRepoPath(v, what) { + if (typeof v !== "string" || !PATH_RE.test(v) || v.split("/").some((s) => s === "." || s === "..")) { + throw refuse(`${what} must be a repo-relative path: ${JSON.stringify(v)}`); + } + return v; +} + +export function parseBriefSpec(spec) { + if (typeof spec !== "string") throw refuse("brief must be PATH#ANCHOR"); + const i = spec.indexOf("#"); + if (i < 0) throw refuse(`brief must be PATH#ANCHOR, naming the heading: ${JSON.stringify(spec)}`); + const path = checkRepoPath(spec.slice(0, i), "brief path"); + const anchor = checkText(spec.slice(i + 1), "brief anchor", { max: 200 }); + return { path, anchor }; +} + +function checkBrief(v, what = "brief") { + keysExactly(v, ["path", "anchor", "blob"], what); + checkRepoPath(v.path, `${what} path`); + checkText(v.anchor, `${what} anchor`, { max: 200 }); + if (!BLOB_RE.test(v.blob)) throw refuse(`${what} blob must be a 40-hex git blob id`); + return v; +} + +function checkCandidate(v) { + keysExactly(v, ["kind", "digest", "text"], "candidate"); + if (v.kind === "commit") { + if (!BLOB_RE.test(v.digest) || v.text !== null) throw refuse("a commit candidate is a 40-hex commit id with no text"); + } else if (v.kind === "manifest") { + if (!SHA256_RE.test(v.digest) || typeof v.text !== "string" || sha256(v.text) !== v.digest) throw refuse("a manifest candidate's digest must be the SHA-256 of its text"); + parseManifest(v.text); + } else { + throw refuse("candidate kind must be commit or manifest"); + } + return v; +} + +// A manifest in the form of CANDIDATE.sha256: " " per line. +export function parseManifest(text) { + if (typeof text !== "string" || text.length === 0 || !text.endsWith("\n")) throw refuse("a candidate manifest must be non-empty and end with a newline"); + const seen = new Set(); + const lines = text.slice(0, -1).split("\n"); + for (const line of lines) { + const m = /^([0-9a-f]{64}) {2}(.+)$/.exec(line); + if (!m) throw refuse(`candidate manifest line is not " ": ${JSON.stringify(line)}`); + checkRepoPath(m[2], "candidate manifest path"); + if (seen.has(m[2])) throw refuse(`candidate manifest names ${m[2]} twice`); + seen.add(m[2]); + } + return lines.length; +} + +function checkRound(v, n) { + keysExactly(v, ["n", "op", "by", "at", "candidate", "request"], "review round"); + if (v.n !== n) throw refuse(`review rounds must be numbered from 1; expected ${n}`); + if (!LOG_OP_RE.test(v.op)) throw refuse("review round op is not an op id"); + checkName(v.by, "review round by"); + checkTime(v.at, "review round at"); + checkCandidate(v.candidate); + if (v.request !== "none") throw refuse("review round request must be none before Piece D"); +} + +export function validateRow(row) { + keysExactly(row, ROW_KEYS, `row ${row?.id}`); + const w = `row ${row.id}`; + checkId(row.id); + checkText(row.piece, `${w} piece`, { max: 300 }); + checkName(row.owner, `${w} owner`); + checkIssues(row.issues, `${w} issues`); + checkIssues(row.closes, `${w} closes`); + for (const n of row.closes) if (!row.issues.includes(n)) throw refuse(`${w} closes #${n}, which is not in its issues`); + if (!STATES.includes(row.state)) throw refuse(`${w} state ${JSON.stringify(row.state)} is not a state`); + if (row.state === "blocked") { + if (!NON_TERMINAL.has(row.previousState)) throw refuse(`${w} is blocked and needs the non-terminal state it returns to`); + checkText(row.blockedReason, `${w} blockedReason`); + } else if (row.previousState !== null || row.blockedReason !== null) { + throw refuse(`${w} carries previousState or blockedReason but is not blocked`); + } + if (typeof row.required !== "boolean") throw refuse(`${w} required must be true or false`); + if (row.required) checkTime(row.requiredSince, `${w} requiredSince`, { unknown: true, date: true }); + else if (row.requiredSince !== null) throw refuse(`${w} has requiredSince but is not required`); + if (row.required && row.state === "parked") throw refuse(`${w} is parked and required`); + checkText(row.gate, `${w} gate`, { max: 300 }); + checkName(row.gateOwner, `${w} gateOwner`); + if (row.brief === null) { + if (row.state !== "done") throw refuse(`${w} has no brief; only a done row may lack one`); + } else { + checkBrief(row.brief, `${w} brief`); + } + checkAfter(row.after, `${w} after`); + checkNames(row.reviewers, `${w} reviewers`); + if (row.review !== null) { + keysExactly(row.review, ["issue", "rounds"], `${w} review`); + if (row.review.issue !== null) checkId(row.review.issue, `${w} review issue`); + if (!Array.isArray(row.review.rounds) || row.review.rounds.length === 0) throw refuse(`${w} review needs at least one round`); + row.review.rounds.forEach((r, i) => checkRound(r, i + 1)); + } + if (row.claim !== null) { + keysExactly(row.claim, ["seat", "op"], `${w} claim`); + if (row.claim.seat !== row.owner) throw refuse(`${w} claim seat ${row.claim.seat} is not the owner ${row.owner}`); + if (!LOG_OP_RE.test(row.claim.op)) throw refuse(`${w} claim op is not an op id`); + const s = row.state === "blocked" ? row.previousState : row.state; + if (!CLAIM_KEPT.has(s)) throw refuse(`${w} is claimed but ${row.state}`); + } + if (row.note !== null) checkText(row.note, `${w} note`); + checkTime(row.createdAt, `${w} createdAt`, { unknown: true, date: true }); + checkTime(row.updatedAt, `${w} updatedAt`); + checkName(row.updatedBy, `${w} updatedBy`); + return row; +} + +// Whole-table rules: unique ids, `after` names existing other rows, no cycle. +export function validateRows(rows) { + const byId = new Map(); + for (const r of rows) { + validateRow(r); + if (byId.has(r.id)) throw refuse(`row id ${r.id} appears twice`); + byId.set(r.id, r); + } + for (const r of rows) { + for (const a of r.after) { + if (a.id === r.id) throw refuse(`row ${r.id} lists itself in after`); + if (!byId.has(a.id)) throw refuse(`row ${r.id} lists missing row ${a.id} in after`); + } + } + const state = new Map(); + const visit = (id, path) => { + if (state.get(id) === 2) return; + if (state.get(id) === 1) throw refuse(`after has a cycle: ${[...path, id].join(" -> ")}`); + state.set(id, 1); + for (const a of byId.get(id).after) visit(a.id, [...path, id]); + state.set(id, 2); + }; + for (const r of rows) visit(r.id, []); + return byId; +} + +// --- canonical arguments (8.6: identity is the op id, same verb and args) --- + +function ordered(obj, keys) { + const out = {}; + for (const k of keys) out[k] = obj[k]; + return out; +} + +function nullable(v, fn) { + return v === null || v === undefined ? null : fn(v); +} + +function sortedUnique(list, cmp) { + const out = [...list].sort(cmp); + for (let i = 1; i < out.length; i++) if (cmp(out[i], out[i - 1]) === 0) throw refuse(`repeated value ${JSON.stringify(out[i])}`); + return out; +} + +export function canonIssues(v) { + if (!Array.isArray(v)) throw refuse("issues must be a list"); + return checkIssues(sortedUnique(v.map((n) => checkId(n, "issue")), (a, b) => a - b), "issues"); +} + +export function canonNames(v, what) { + if (!Array.isArray(v)) throw refuse(`${what} must be a list`); + return sortedUnique(v.map((n) => checkName(n, what)), (a, b) => (a < b ? -1 : a > b ? 1 : 0)); +} + +export function canonAfter(v) { + if (!Array.isArray(v)) throw refuse("after must be a list"); + const list = v.map((a) => { + if (!isObj(a)) throw refuse("after entries are {id, when}"); + return { id: checkId(a.id, "after id"), when: a.when }; + }); + return checkAfter(sortedUnique(list, (a, b) => a.id - b.id)); +} + +function canonSetValue(field, value) { + switch (field) { + case "piece": return checkText(value, "piece", { max: 300 }); + case "gate": return checkText(value, "gate", { max: 300 }); + case "gate-owner": return checkName(value, "gate owner"); + case "after": return canonAfter(value); + case "reviewers": return canonNames(value, "reviewers"); + case "issues": return canonIssues(value); + case "closes": return canonIssues(value); + case "brief": parseBriefSpec(value); return value; + case "required": + if (typeof value !== "boolean") throw refuse("required must be true or false"); + return value; + default: throw refuse(`set cannot change ${JSON.stringify(field)}; fields: ${SET_FIELDS.join(", ")}`); + } +} + +export function canonArgs(verb, a) { + if (!isObj(a)) throw refuse("args must be an object"); + switch (verb) { + case "genesis": + if (typeof a.root !== "string" || !a.root.startsWith("/")) throw refuse("genesis root must be an absolute path"); + if (typeof a.branch !== "string" || !BRANCH_RE.test(a.branch)) throw refuse("genesis branch is not a branch name"); + checkRepoPath(a.map, "genesis map"); + return ordered(a, ["root", "branch", "map"]); + case "add": { + const brief = a.brief; + parseBriefSpec(brief); + return { + piece: checkText(a.piece, "piece", { max: 300 }), + gate: checkText(a.gate, "gate", { max: 300 }), + brief, + issues: canonIssues(a.issues ?? []), + note: nullable(a.note, (v) => checkText(v, "note")), + owner: nullable(a.owner, (v) => checkName(v, "owner")), + gateOwner: nullable(a.gateOwner, (v) => checkName(v, "gate owner")), + after: nullable(a.after, canonAfter), + reviewers: nullable(a.reviewers, (v) => canonNames(v, "reviewers")), + required: nullable(a.required, (v) => { if (typeof v !== "boolean") throw refuse("required must be true or false"); return v; }), + }; + } + case "move": + if (!STATES.includes(a.to)) throw refuse(`unknown state ${JSON.stringify(a.to)}; states: ${STATES.join(", ")}`); + return { + id: checkId(a.id), + to: a.to, + reason: nullable(a.reason, (v) => checkText(v, "reason")), + candidate: nullable(a.candidate, (v) => checkText(v, "candidate", { max: 300 })), + evidence: nullable(a.evidence, (v) => checkText(v, "evidence")), + }; + case "release": + return { id: checkId(a.id) }; + case "assign": + return { id: checkId(a.id), seat: checkName(a.seat, "seat") }; + case "note": + return { id: checkId(a.id), text: checkText(a.text, "note", { empty: true }) }; + case "set": { + if (!SET_FIELDS.includes(a.field)) throw refuse(`set cannot change ${JSON.stringify(a.field)}; fields: ${SET_FIELDS.join(", ")}`); + return { + id: checkId(a.id), + field: a.field, + value: canonSetValue(a.field, a.value), + reason: nullable(a.reason, (v) => checkText(v, "reason")), + }; + } + case "accept-history": + return { reason: checkText(a.reason, "reason") }; + default: + throw refuse(`unknown verb ${JSON.stringify(verb)}`); + } +} + +export function sameJson(a, b) { + return JSON.stringify(a) === JSON.stringify(b); +} + +// --- the matrix (8.7) --- + +const isPriv = (by) => PRIVILEGED.has(by); + +function ownerOrPriv(row, by, doing) { + if (by === row.owner || isPriv(by)) return; + if (row.claim !== null) throw refuse(`row ${row.id} is claimed by ${row.claim.seat}; ${by} cannot ${doing}`); + throw refuse(`only the owner (${row.owner}) or a privileged actor may ${doing} row ${row.id}`); +} + +function requirePriv(by, doing) { + if (!isPriv(by)) throw refuse(`only a privileged actor (jason or sage) may ${doing}`); +} + +function requireJason(by, doing) { + if (by !== "jason") throw refuse(`only jason may ${doing}`); +} + +function afterSatisfied(rows, row) { + const missing = []; + for (const a of row.after) { + const t = rows.get(a.id); + const ok = t && (t.state === "done" || (a.when === "settled" && t.state === "blocked")); + if (!ok) missing.push(`${a.id} (${a.when}; now ${t ? t.state : "missing"})`); + } + return missing; +} + +// `comment=,candidate=`: the J5 evidence before Piece D. +export function parseReviewEvidence(text) { + const m = /^comment=([1-9][0-9]{0,19}),candidate=([0-9a-f]{40}|[0-9a-f]{64})$/.exec(text ?? ""); + if (!m) throw refuse("in-review to done needs --evidence comment=,candidate= for the current round"); + return { comment: m[1], candidate: m[2] }; +} + +function touch(row, entry) { + return { ...row, updatedAt: entry.at, updatedBy: entry.by }; +} + +function fmt(v) { + if (v === null) return "none"; + if (Array.isArray(v)) return v.length ? v.map(fmt).join(", ") : "none"; + if (typeof v === "number") return `#${v}`; + if (isObj(v) && "when" in v) return `${v.id} ${v.when}`; + if (isObj(v) && "blob" in v) return `${v.path} § ${v.anchor} @${v.blob.slice(0, 12)}`; + return String(v); +} + +function receipt(entry, rev, rest) { + return `ok ${entry.op} rev ${rev} ${rest}`; +} + +function getRow(rows, id) { + const row = rows.get(id); + if (!row) throw refuse(`no row ${id}`); + return row; +} + +function applyMove(rows, row, entry, resolved) { + const { to, reason, candidate, evidence } = entry.args; + const by = entry.by; + const from = row.state; + const illegal = () => refuse(`row ${row.id}: ${from}→${to} is not a transition`); + if (from === "done") throw refuse(`row ${row.id} is done; done rows never change`); + if (reason !== null && to !== "blocked") throw refuse("--reason applies only to a move to blocked"); + if (candidate !== null && !(from === "in-progress" && to === "in-review")) throw refuse("--candidate applies only to in-progress→in-review"); + if (evidence !== null && to !== "done") throw refuse("--evidence applies only to a move to done"); + let next = { ...row, state: to }; + let round = null; + let cand = null; + if (to === "blocked") { + if (from === "blocked") throw refuse(`row ${row.id} is already blocked; update the reason with note`); + if (!NON_TERMINAL.has(from)) throw illegal(); + ownerOrPriv(row, by, "block"); + if (reason === null) throw refuse("a move to blocked needs --reason"); + next = { ...next, previousState: from, blockedReason: reason }; + } else if (from === "blocked") { + if (to !== row.previousState) throw refuse(`row ${row.id} is blocked from ${row.previousState} and returns only there`); + ownerOrPriv(row, by, "unblock"); + next = { ...next, previousState: null, blockedReason: null }; + } else if (from === "queued" && to === "briefed") { + requirePriv(by, "accept a brief (queued→briefed)"); + } else if (from === "briefed" && to === "in-progress") { + if (by !== row.owner) throw refuse(`only the owner (${row.owner}) may start row ${row.id}`); + const missing = afterSatisfied(rows, row); + if (missing.length) throw refuse(`row ${row.id} waits on ${missing.join(", ")}`); + next.claim = { seat: row.owner, op: entry.op }; + } else if (from === "in-progress" && to === "briefed") { + throw refuse(`use \`queue release ${row.id}\` to give up a claim`); + } else if (from === "in-progress" && to === "in-review") { + if (row.claim === null || by !== row.claim.seat) throw refuse(`only the claimant (${row.claim?.seat ?? "nobody"}) may request review of row ${row.id}`); + if (candidate === null) throw refuse("in-progress→in-review needs --candidate "); + cand = checkCandidate(resolved.candidate); + const rounds = row.review ? row.review.rounds : []; + round = rounds.length + 1; + next.review = { + issue: row.review ? row.review.issue : (row.issues[0] ?? null), + rounds: [...rounds, { n: round, op: entry.op, by, at: entry.at, candidate: cand, request: "none" }], + }; + } else if (from === "in-review" && (to === "in-progress" || to === "waiting-on-jason")) { + ownerOrPriv(row, by, `move to ${to}`); + } else if (from === "waiting-on-jason" && to === "done") { + if (by === "sage") { + if (evidence === null) throw refuse("sage closes a waiting-on-jason row only with --evidence citing Jason's approval"); + } else { + requireJason(by, "close a waiting-on-jason row without cited evidence"); + } + next.claim = null; + } else if (from === "in-review" && to === "done") { + if (row.gateOwner === "jason") throw refuse(`row ${row.id}'s gate is Jason's; it goes through waiting-on-jason`); + if (by !== row.gateOwner && !isPriv(by)) throw refuse(`only the gate owner (${row.gateOwner}) or a privileged actor may close row ${row.id}`); + const ev = parseReviewEvidence(evidence); + const cur = row.review?.rounds.at(-1); + if (!cur) throw refuse(`row ${row.id} has no review round to cite`); + if (ev.candidate !== cur.candidate.digest) throw refuse(`evidence candidate ${ev.candidate} is not round ${cur.n}'s candidate ${cur.candidate.digest}`); + round = cur.n; + next.claim = null; + } else if ((from === "queued" || from === "briefed") && to === "parked") { + requireJason(by, "park a row"); + if (row.required) throw refuse(`row ${row.id} is required and cannot be parked`); + } else if (from === "parked" && to === "queued") { + requireJason(by, "unpark a row"); + } else { + throw illegal(); + } + next = touch(next, entry); + return { row: next, result: { row: row.id, from, to, round, candidate: cand } }; +} + +function applySet(rows, row, entry, resolved) { + const { field, value, reason } = entry.args; + const by = entry.by; + if (row.state === "done") throw refuse(`row ${row.id} is done; done rows never change`); + if (reason !== null && field !== "closes") throw refuse("--reason applies only to closes"); + let key = field; + let next = { ...row }; + switch (field) { + case "piece": case "gate": case "reviewers": case "issues": + requirePriv(by, `change ${field}`); + next[field] = value; + if (field === "issues") next.closes = value; + break; + case "gate-owner": + requirePriv(by, "change the gate owner"); + key = "gateOwner"; + next.gateOwner = value; + break; + case "after": + if (row.required) requireJason(by, "change after on a required row"); + else requirePriv(by, "change after"); + next.after = value; + break; + case "closes": + requirePriv(by, "narrow closes"); + if (reason === null) throw refuse("narrowing closes needs --reason, which is logged"); + for (const n of value) if (!row.issues.includes(n)) throw refuse(`closes #${n} is not one of row ${row.id}'s issues`); + next.closes = value; + break; + case "brief": + requirePriv(by, "re-pin a brief"); + next.brief = checkBrief(resolved.brief); + if (next.brief.path !== parseBriefSpec(value).path || next.brief.anchor !== parseBriefSpec(value).anchor) throw refuse("resolved brief does not match the brief argument"); + if (row.brief !== null && next.brief.blob === row.brief.blob && sameJson(next.brief, row.brief)) throw refuse(`row ${row.id} brief is already pinned to ${row.brief.blob}`); + break; + case "required": + if (value) { + requirePriv(by, "set required"); + if (row.state === "parked") throw refuse(`row ${row.id} is parked; unpark it before making it required`); + next.requiredSince = entry.at; + } else { + requireJason(by, "clear required"); + next.requiredSince = null; + } + next.required = value; + break; + default: + throw refuse(`set cannot change ${field}`); + } + if (sameJson(next[key], row[key]) && (field !== "issues" || sameJson(next.closes, row.closes))) { + throw refuse(`row ${row.id} ${key} is already ${fmt(row[key])}`); + } + return { row: touch(next, entry), result: { row: row.id, field: key, from: row[key], to: next[key] } }; +} + +// One log entry against the state before it. `resolved` carries what the +// verb took from outside (a pinned brief, a candidate, the old witness): at +// write time from the checks in store.mjs, in replay from the entry's own +// result. Returns the new state and the result to record. +export function applyEntry(state, entry, resolved) { + const rows = new Map(state.rows); + let highWater = state.highWater; + const rev = state.revision + 1; + const by = entry.by; + const a = entry.args; + let result; + switch (entry.verb) { + case "add": { + const priv = isPriv(by); + if (!priv) { + for (const k of ["owner", "gateOwner", "after", "reviewers", "required"]) { + if (a[k] !== null && !(k === "owner" && a[k] === by)) throw refuse(`only a privileged actor may set ${k} on add`); + } + } + const brief = checkBrief(resolved.brief); + if (brief.path !== parseBriefSpec(a.brief).path || brief.anchor !== parseBriefSpec(a.brief).anchor) throw refuse("resolved brief does not match the brief argument"); + const id = highWater + 1; + highWater = id; + const required = a.required ?? false; + const row = { + id, piece: a.piece, owner: a.owner ?? by, issues: a.issues, closes: a.issues, state: "queued", previousState: null, + required, requiredSince: required ? entry.at : null, gate: a.gate, gateOwner: a.gateOwner ?? "jason", brief, + after: a.after ?? [], reviewers: a.reviewers ?? [], review: null, claim: null, note: a.note, blockedReason: null, + createdAt: entry.at, updatedAt: entry.at, updatedBy: by, + }; + rows.set(id, row); + result = { row: id, from: null, to: "queued", brief, receipt: receipt(entry, rev, `row ${id} none→queued`) }; + break; + } + case "move": { + const row = getRow(rows, a.id); + const out = applyMove(rows, row, entry, resolved); + rows.set(row.id, out.row); + const r = out.result; + result = { ...r, receipt: receipt(entry, rev, `row ${row.id} ${r.from}→${r.to}${r.round ? ` round ${r.round}` : ""}`) }; + break; + } + case "release": { + const row = getRow(rows, a.id); + if (row.state !== "in-progress") throw refuse(`row ${row.id} is ${row.state}; release applies to in-progress rows`); + if (!(row.claim && by === row.claim.seat) && !isPriv(by)) throw refuse(`only the claimant (${row.claim?.seat}) or a privileged actor may release row ${row.id}`); + rows.set(row.id, touch({ ...row, state: "briefed", claim: null }, entry)); + result = { row: row.id, from: "in-progress", to: "briefed", receipt: receipt(entry, rev, `row ${row.id} in-progress→briefed`) }; + break; + } + case "assign": { + const row = getRow(rows, a.id); + requirePriv(by, "assign a row"); + if (row.state === "done") throw refuse(`row ${row.id} is done; done rows never change`); + if (row.owner === a.seat) throw refuse(`row ${row.id} is already owned by ${a.seat}`); + const claim = row.claim ? { seat: a.seat, op: entry.op } : null; + rows.set(row.id, touch({ ...row, owner: a.seat, claim }, entry)); + result = { row: row.id, field: "owner", from: row.owner, to: a.seat, receipt: receipt(entry, rev, `row ${row.id} owner: ${row.owner}→${a.seat}`) }; + break; + } + case "note": { + const row = getRow(rows, a.id); + if (row.state === "done" || row.state === "parked") throw refuse(`row ${row.id} is ${row.state}; notes are closed`); + if (by !== row.owner && !row.reviewers.includes(by) && !isPriv(by)) throw refuse(`only the owner (${row.owner}), a listed reviewer or a privileged actor may note row ${row.id}`); + // On a blocked row the note is the reason (8.7: "update the reason + // with note"); elsewhere it is the row's note, and "" clears it. + const field = row.state === "blocked" ? "blockedReason" : "note"; + if (field === "blockedReason" && a.text === "") throw refuse(`row ${row.id} is blocked and its reason cannot be empty`); + const text = a.text === "" ? null : a.text; + if (row[field] === text) throw refuse(`row ${row.id} already has that ${field}`); + rows.set(row.id, touch({ ...row, [field]: text }, entry)); + result = { row: row.id, field, receipt: receipt(entry, rev, `row ${row.id} ${field}`) }; + break; + } + case "set": { + const row = getRow(rows, a.id); + const out = applySet(rows, row, entry, resolved); + rows.set(row.id, out.row); + const r = out.result; + result = { ...r, receipt: receipt(entry, rev, `row ${row.id} ${r.field}: ${fmt(r.from)}→${fmt(r.to)}`) }; + break; + } + case "accept-history": { + requirePriv(by, "accept history"); + const old = resolved.oldWitness; + if (old !== null) { + keysExactly(old, ["revision", "logDigest"], "old witness"); + if (!Number.isInteger(old.revision) || old.revision < 0 || !SHA256_RE.test(old.logDigest)) throw refuse("old witness is malformed"); + } + result = { + oldWitness: old, + warning: "ops in the lost range are no longer deduplicated", + receipt: receipt(entry, rev, `accept-history over witness ${old ? `rev ${old.revision}` : "absent"}`), + }; + break; + } + case "genesis": + throw refuse("genesis is only log[0]"); + default: + throw refuse(`unknown verb ${entry.verb}`); + } + validateRows([...rows.values()]); + return { state: { rows, highWater, retired: state.retired, revision: rev }, result }; +} + +// What a verb took from outside, read back from its recorded result. +export function resolvedFromResult(verb, args, result) { + if (!isObj(result)) throw refuse("entry result must be an object"); + if (verb === "add") return { brief: result.brief }; + if (verb === "move") return { candidate: result.candidate }; + if (verb === "set" && args.field === "brief") return { brief: result.to }; + if (verb === "accept-history") return { oldWitness: result.oldWitness }; + return {}; +} + +// --- genesis --- + +export function parseMigrationMap(text) { + const blocks = [...text.matchAll(/^```json queue-map\n([\s\S]*?)^```$/gm)]; + if (blocks.length !== 1) throw refuse("the migration map must hold exactly one ```json queue-map block"); + let map; + try { map = JSON.parse(blocks[0][1]); } catch (err) { throw refuse(`the migration map's queue-map block is not JSON: ${err.message}`); } + keysExactly(map, ["rows", "retired", "highWater"], "migration map"); + if (!Array.isArray(map.rows)) throw refuse("migration map rows must be a list"); + map.rows.forEach((r) => { + keysExactly(r, MAP_ROW_KEYS, `migration map row ${r?.id}`); + if (r.brief !== null) keysExactly(r.brief, ["path", "anchor"], `migration map row ${r.id} brief`); + }); + if (!Array.isArray(map.retired)) throw refuse("migration map retired must be a list"); + map.retired.forEach((id) => checkId(id, "retired id")); + checkId(map.highWater, "highWater"); + return map; +} + +// Rows at genesis: the map's rows, briefs pinned to HEAD blobs, a claim for +// any row already under way so its owner can move it on. +export function genesisRows(map, blobs, op, at, by) { + return map.rows.map((r) => { + const s = r.state === "blocked" ? r.previousState : r.state; + return { + id: r.id, piece: r.piece, owner: r.owner, issues: r.issues, closes: r.closes, state: r.state, + previousState: r.previousState, required: r.required, requiredSince: r.requiredSince, gate: r.gate, + gateOwner: r.gateOwner, brief: r.brief === null ? null : { path: r.brief.path, anchor: r.brief.anchor, blob: blobs.get(r.id) }, + after: r.after, reviewers: r.reviewers, review: null, claim: CLAIM_KEPT.has(s) ? { seat: r.owner, op } : null, + note: r.note, blockedReason: r.blockedReason, createdAt: r.createdAt, updatedAt: at, updatedBy: by, + }; + }).sort((x, y) => x.id - y.id); +} + +function checkGenesis(entry) { + if (entry.verb !== "genesis" || entry.rev !== 0) throw refuse("log[0] must be the genesis entry at rev 0"); + if (!PRIVILEGED.has(entry.by)) throw refuse("genesis must be by a privileged actor"); + const r = entry.result; + keysExactly(r, ["mapBlob", "highWater", "retired", "rows", "legacyView", "receipt"], "genesis result"); + if (!BLOB_RE.test(r.mapBlob)) throw refuse("genesis mapBlob must be a git blob id"); + if (!Array.isArray(r.retired)) throw refuse("genesis retired must be a list"); + const retired = new Set(r.retired.map((id) => checkId(id, "retired id"))); + if (typeof r.legacyView !== "string") throw refuse("genesis legacyView must be text"); + const byId = validateRows(r.rows); + for (let i = 1; i < r.rows.length; i++) if (r.rows[i].id <= r.rows[i - 1].id) throw refuse("genesis rows must be sorted by id"); + for (const id of retired) if (byId.has(id)) throw refuse(`retired id ${id} is also a row`); + const top = Math.max(0, ...byId.keys(), ...retired); + checkId(r.highWater, "highWater"); + if (r.highWater < top) throw refuse(`genesis highWater ${r.highWater} is below the highest id ${top}`); + for (const row of r.rows) { + if (row.updatedAt !== entry.at || row.updatedBy !== entry.by) throw refuse(`genesis row ${row.id} must carry the genesis time and actor`); + if (row.review !== null) throw refuse(`genesis row ${row.id} cannot carry a review`); + const s = row.state === "blocked" ? row.previousState : row.state; + const want = CLAIM_KEPT.has(s) ? { seat: row.owner, op: entry.op } : null; + if (!sameJson(row.claim, want)) throw refuse(`genesis row ${row.id} claim must be ${fmt(want)}`); + } + if (r.receipt !== genesisReceipt(entry.op, r.rows.length)) throw refuse("genesis receipt does not match"); + return { rows: new Map(r.rows.map((x) => [x.id, x])), highWater: r.highWater, retired, revision: 0 }; +} + +export function genesisReceipt(op, count) { + return `ok ${op} rev 0 genesis ${count} rows`; +} + +// --- document --- + +export function serialize(doc) { + return JSON.stringify(doc, null, 2) + "\n"; +} + +export function logDigest(log, revision) { + return sha256(JSON.stringify(log.slice(0, revision + 1))); +} + +export function rowsArray(state) { + return [...state.rows.values()].sort((a, b) => a.id - b.id); +} + +function checkEntryShape(e, i) { + keysExactly(e, ENTRY_KEYS, `log entry ${i}`); + if (e.rev !== i) throw refuse(`log entry ${i} has rev ${e.rev}`); + if (typeof e.op !== "string" || !LOG_OP_RE.test(e.op)) throw refuse(`log entry ${i} op ${JSON.stringify(e.op)} is not an op id`); + if (!VERBS.includes(e.verb)) throw refuse(`log entry ${i} verb ${JSON.stringify(e.verb)} is unknown`); + checkName(e.by, `log entry ${i} by`); + checkTime(e.at, `log entry ${i} at`); + if (e.semantics !== SEMANTICS) throw refuse(`log entry ${i} semantics ${e.semantics} is not ${SEMANTICS}`); + if (typeof e.viewSha !== "string" || !SHA256_RE.test(e.viewSha)) throw refuse(`log entry ${i} viewSha is not a SHA-256`); +} + +// Parses, checks the schema and byte-for-byte serialization, and replays the +// log from genesis. Returns {doc, state, branch}. Any failure is a refusal. +export function loadDoc(bytes) { + let text; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw refuse("queue.json is not valid UTF-8"); + } + let doc; + try { doc = JSON.parse(text); } catch (err) { throw refuse(`queue.json is not JSON: ${err.message}`); } + keysExactly(doc, DOC_KEYS, "queue.json"); + if (serialize(doc) !== text) throw refuse("queue.json does not re-serialize byte for byte (a hand edit?)"); + if (doc.version !== VERSION) throw refuse(`queue.json version ${doc.version} is not ${VERSION}`); + if (typeof doc.canonicalRoot !== "string" || !doc.canonicalRoot.startsWith("/")) throw refuse("canonicalRoot must be an absolute path"); + if (!Array.isArray(doc.rows) || !Array.isArray(doc.log) || doc.log.length === 0) throw refuse("queue.json needs rows and a log starting at genesis"); + doc.log.forEach(checkEntryShape); + const ops = new Set(); + for (const e of doc.log) { + if (ops.has(e.op)) throw refuse(`op id ${e.op} appears twice in the log`); + ops.add(e.op); + } + if (doc.revision !== doc.log.length - 1) throw refuse(`revision ${doc.revision} is not the last rev ${doc.log.length - 1}`); + const state = replay(doc.log); + if (!sameJson(rowsArray(state), doc.rows)) throw refuse("rows do not equal the replay of the log (a hand edit?)"); + if (doc.canonicalRoot !== doc.log[0].args.root) throw refuse("canonicalRoot differs from the genesis root"); + return { doc, state, branch: doc.log[0].args.branch }; +} + +export function replay(log) { + const g = log[0]; + if (!sameJson(canonArgs("genesis", g.args), g.args)) throw refuse("genesis args are not canonical"); + let state = checkGenesis(g); + if (g.viewSha !== sha256(render(rowsArray(state), 0))) throw refuse("log entry 0 viewSha is not the render of its rows"); + for (let i = 1; i < log.length; i++) { + const e = log[i]; + if (e.verb === "genesis") throw refuse(`log entry ${i} is a second genesis`); + if (!sameJson(canonArgs(e.verb, e.args), e.args)) throw refuse(`log entry ${i} args are not canonical`); + if (e.verb !== "accept-history" && e.op.endsWith(".outcome")) throw refuse(`log entry ${i} uses the reserved .outcome suffix`); + let out; + try { + out = applyEntry(state, e, resolvedFromResult(e.verb, e.args, e.result)); + } catch (err) { + if (err instanceof QueueError) throw refuse(`log entry ${i} (${e.op}) does not replay: ${err.message}`); + throw err; + } + if (!sameJson(out.result, e.result)) throw refuse(`log entry ${i} (${e.op}) result differs from its replay`); + state = out.state; + if (e.viewSha !== sha256(render(rowsArray(state), i))) throw refuse(`log entry ${i} viewSha is not the render of its rows`); + } + return state; +} + +export function buildDoc(canonicalRoot, state, log) { + return { version: VERSION, canonicalRoot, revision: log.length - 1, rows: rowsArray(state), log }; +} + +// --- next (8.8) --- + +// `briefMatches(row)` says whether the working brief still has the pinned blob. +export function nextFor(rows, seat, briefMatches) { + const list = [...rows.values()].sort((a, b) => a.id - b.id); + const byId = new Map(list.map((r) => [r.id, r])); + const resume = list.find((r) => r.state === "in-progress" && r.claim?.seat === seat); + if (resume) return { action: "resume", row: resume }; + const review = list.find((r) => r.state === "in-review" && r.reviewers.includes(seat) && r.owner !== seat); + if (review) return { action: "review", row: review }; + const start = list.find((r) => r.state === "briefed" && r.owner === seat && afterSatisfied(byId, r).length === 0); + if (start) return { action: "start", row: start, briefDiffers: !briefMatches(start) }; + const wait = list.find((r) => r.state === "in-review" && r.owner === seat); + if (wait) return { action: "wait", row: wait }; + return { action: "nothing", row: null }; +} + +// --- render --- + +function cell(text) { + return String(text).replaceAll("|", "\\|"); +} + +function stateCell(r) { + let s = r.state; + if (r.state === "blocked") s = `blocked (from ${r.previousState}): ${r.blockedReason}`; + else if (r.state === "in-review" && r.review) s = `in-review, round ${r.review.rounds.length}`; + return r.required ? `required; ${s}` : s; +} + +function issuesCell(r) { + const list = (ns) => (ns.length ? ns.map((n) => `#${n}`).join(", ") : "none"); + let s = r.issues.length ? list(r.issues) : "—"; + if (!sameJson(r.closes, r.issues)) s += `; closes ${list(r.closes)}`; + return s; +} + +// The table body for one revision. Its SHA-256 is each entry's viewSha. +export function render(rows, revision) { + const lines = [ + "", + `Generated from \`docs/plans/queue.json\` revision ${revision} by \`queue render\`. Do not edit between the markers; change the queue with \`scripts/mosaic queue\`.`, + "", + "| # | Piece | Owner | Issues | State | After | Gate | Brief | Note |", + "|---|---|---|---|---|---|---|---|---|", + ]; + for (const r of [...rows].sort((a, b) => a.id - b.id)) { + const owner = r.reviewers.length ? `${r.owner}; reviewers ${r.reviewers.join(", ")}` : r.owner; + const after = r.after.length ? r.after.map((a) => `${a.id} ${a.when}`).join(", ") : "—"; + const brief = r.brief ? `\`${r.brief.path}\` § ${r.brief.anchor}` : "—"; + lines.push(`| ${[r.id, r.piece, owner, issuesCell(r), stateCell(r), after, `${r.gate} (${r.gateOwner})`, brief, r.note ?? "—"].map(cell).join(" | ")} |`); + } + lines.push(""); + return lines.join("\n") + "\n"; +} + +// Splits QUEUE.md at the markers. Null when the markers are missing, +// repeated or out of order: the view is unknown. +export function splitView(text) { + const b = `${BEGIN_MARKER}\n`; + const e = `${END_MARKER}\n`; + const bi = text.indexOf(b); + const ei = text.indexOf(e); + if (bi < 0 || ei < 0 || text.indexOf(b, bi + 1) >= 0 || text.indexOf(e, ei + 1) >= 0) return null; + if (bi !== 0 && text[bi - 1] !== "\n") return null; + if (ei !== 0 && text[ei - 1] !== "\n") return null; + const bodyStart = bi + b.length; + if (ei < bodyStart) return null; + return { head: text.slice(0, bodyStart), body: text.slice(bodyStart, ei), tail: text.slice(ei) }; +} + +// current: the body is this revision's render; stale: an earlier logged +// render (the entries after it are unshown); unknown: anything else. +export function classifyView(text, log) { + const parts = text === null ? null : splitView(text); + if (parts === null) return { state: "unknown", why: "markers missing, repeated or out of order" }; + const sha = sha256(parts.body); + const last = log.length - 1; + if (log[last].viewSha === sha) return { state: "current", parts }; + for (let i = last - 1; i >= 0; i--) { + if (log[i].viewSha === sha) return { state: "stale", parts, shown: i, unshown: log.slice(i + 1) }; + } + return { state: "unknown", parts, why: "the table body matches no logged render" }; +} + +export function describeUnshown(unshown) { + return unshown.map((e) => `rev ${e.rev} (op ${e.op} by ${e.by} at ${e.at})`).join(", "); +} + +// Headings outside fenced code blocks whose text is exactly `anchor`. +export function countHeading(text, anchor) { + let fence = null; + let n = 0; + for (const line of text.split("\n")) { + const f = /^(```+|~~~+)/.exec(line); + if (f) { + if (fence === null) fence = f[1][0]; + else if (f[1][0] === fence) fence = null; + continue; + } + if (fence !== null) continue; + const m = /^#{1,6}[ \t]+(.*?)[ \t]*$/.exec(line); + if (m && m[1] === anchor) n++; + } + return n; +} diff --git a/packages/queue/src/store.mjs b/packages/queue/src/store.mjs new file mode 100644 index 00000000..97f8784c --- /dev/null +++ b/packages/queue/src/store.mjs @@ -0,0 +1,767 @@ +// The queue on disk: canonical checks (8.3), brief checks (8.13), the write +// path (8.5) and every verb. The CLI passes only its parsed arguments. Tests +// may also pass `io`, `proc`, `hook`, `now` and `readOrder` through this API +// to inject faults and pause at named steps; no flag or environment variable +// reaches them. +import { spawnSync } from "node:child_process"; +import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { isAbsolute, join, resolve as resolvePath } from "node:path"; +import { fileURLToPath } from "node:url"; +import { QueueError } from "./errors.mjs"; +import { checkPlatform, errno, fsyncFile, lstatOrNull, readOrNull, realIo, unlinkQuiet, writeTemp } from "./io.mjs"; +import { acquire, checkGate, realProc, release, unlock as unlockLock } from "./lock.mjs"; +import { + CALLER_OP_RE, PRIVILEGED, SEMANTICS, VERSION, applyEntry, buildDoc, canonArgs, checkName, classifyView, countHeading, + describeUnshown, genesisReceipt, genesisRows, gitBlobId, loadDoc, logDigest, nextFor, parseBriefSpec, parseManifest, + parseMigrationMap, render, rowsArray, sameJson, serialize, sha256, splitView, +} from "./queue.mjs"; + +export const QUEUE_REL = "docs/plans/queue.json"; +export const VIEW_REL = "docs/plans/QUEUE.md"; +export const WITNESS_NAME = "mosaic-queue.head"; +const CODE_FILE = fileURLToPath(import.meta.url); +const FIX = "scripts/mosaic queue"; + +function refuse(message) { + return new QueueError(message, 2); +} + +function makeCtx(opts = {}) { + return { + cwd: opts.cwd ?? process.cwd(), + env: opts.env ?? process.env, + io: opts.io ?? realIo, + proc: opts.proc ?? realProc, + hook: opts.hook ?? (() => {}), + now: opts.now ?? (() => new Date().toISOString()), + readOrder: opts.readOrder ?? "witness-first", + lockWaitMs: opts.lockWaitMs ?? 10000, + lockStepMs: opts.lockStepMs ?? 100, + }; +} + +// --- git and the canonical checks (8.3) --- + +function git(ctx, cwd, args, { allowFail = false } = {}) { + const r = spawnSync("git", ["-C", cwd, ...args], { env: ctx.env, maxBuffer: 64 << 20 }); + if (r.error) throw new QueueError(`cannot run git: ${errno(r.error)}`, 1); + if (r.status !== 0) { + if (allowFail) return null; + throw new QueueError(`git ${args.join(" ")} failed: ${r.stderr.toString().trim()}`, 1); + } + return r.stdout; +} + +function locate(ctx) { + for (const k of ["GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR"]) { + if (ctx.env[k] !== undefined) throw refuse(`${k} is set; the queue locates its repository only from the working directory`); + } + const raw = git(ctx, ctx.cwd, ["rev-parse", "--show-toplevel"], { allowFail: true }); + if (raw === null) throw refuse(`${ctx.cwd} is not inside a git checkout`); + const top = realpathSync(raw.toString().trim()); + const dirs = git(ctx, top, ["rev-parse", "--path-format=absolute", "--git-dir", "--git-common-dir"]).toString().split("\n"); + const gitDir = realpathSync(dirs[0]); + if (gitDir !== realpathSync(dirs[1])) throw refuse(`${top} is a linked worktree; the queue runs only in the canonical checkout`); + if (gitDir !== join(top, ".git")) throw refuse(`${top}'s git directory is ${gitDir}, not ${top}/.git`); + const ref = git(ctx, top, ["symbolic-ref", "-q", "HEAD"], { allowFail: true }); + return { + top, gitDir, ref: ref === null ? null : ref.toString().trim(), + queuePath: join(top, QUEUE_REL), viewPath: join(top, VIEW_REL), docsDir: join(top, "docs/plans"), witnessPath: join(gitDir, WITNESS_NAME), + }; +} + +function checkBranch(loc, branch) { + if (loc.ref === null) throw refuse(`HEAD is detached; the queue runs only on branch ${branch}`); + if (loc.ref !== `refs/heads/${branch}`) throw refuse(`HEAD is ${loc.ref}; the queue runs only on branch ${branch}`); +} + +function checkCode(root) { + if (!realpathSync(CODE_FILE).startsWith(`${root}/`)) throw refuse(`this queue code (${CODE_FILE}) is not under the canonical root ${root}`); +} + +function checkCanonical(loc, root, branch) { + if (loc.top !== root) throw refuse(`this checkout is ${loc.top}; the queue's canonical root is ${root}`); + checkBranch(loc, branch); + checkCode(root); +} + +function headHas(ctx, loc, path) { + return git(ctx, loc.top, ["cat-file", "-e", `HEAD:${path}`], { allowFail: true }) !== null; +} + +function actorOf(ctx, by) { + const name = by ?? ctx.env.MOSAIC_AGENT_NAME; + if (name === undefined || name === "") throw refuse("no actor: pass --by NAME or set MOSAIC_AGENT_NAME"); + return checkName(name, "actor"); +} + +function isSeatDir(top, name) { + try { + return lstatSync(join(top, "agents", name)).isDirectory(); + } catch { + return false; + } +} + +function checkActorSeat(top, name) { + if (name !== "jason" && !isSeatDir(top, name)) throw refuse(`actor ${name} is not jason or a seat under agents/`); +} + +function checkSeat(top, name, what, owner) { + if (name === "jason" || isSeatDir(top, name)) return; + if (owner && (name === "coordinator" || name === "unassigned")) return; + throw refuse(`${what} ${name} is not jason${owner ? ", coordinator, unassigned" : ""} or a seat under agents/`); +} + +// Seat names checked when a write sets them; replay never looks at agents/. +function checkNewSeats(top, before, after) { + for (const row of after.values()) { + const old = before.get(row.id); + if (!old || old.owner !== row.owner) checkSeat(top, row.owner, `row ${row.id} owner`, true); + if (!old || old.gateOwner !== row.gateOwner) checkSeat(top, row.gateOwner, `row ${row.id} gate owner`, true); + if (!old || !sameJson(old.reviewers, row.reviewers)) row.reviewers.forEach((r) => checkSeat(top, r, `row ${row.id} reviewer`, false)); + } +} + +// --- brief and candidate checks (8.13, 8.9) --- + +function headEntry(ctx, loc, path) { + const out = git(ctx, loc.top, ["ls-tree", "-z", "--full-tree", "HEAD", "--", path], { allowFail: true }); + if (out === null) return null; + const entries = out.toString().split("\0").filter(Boolean).map((e) => { + const m = /^(\d{6}) (\w+) ([0-9a-f]{40})\t(.*)$/s.exec(e); + return m ? { mode: m[1], type: m[2], oid: m[3], path: m[4] } : null; + }); + return entries.find((e) => e && e.path === path) ?? null; +} + +export function briefCheck(ctx, loc, path, anchor) { + const abs = join(loc.top, path); + let st; + try { + st = lstatSync(abs); + } catch { + throw refuse(`brief ${path} does not exist in the working tree`); + } + if (st.isSymbolicLink() || !st.isFile()) throw refuse(`brief ${path} must be a regular file, not a symlink`); + if (!realpathSync(abs).startsWith(`${loc.top}/`)) throw refuse(`brief ${path} resolves outside ${loc.top}`); + const entry = headEntry(ctx, loc, path); + if (entry === null) throw refuse(`brief ${path} is not committed in HEAD; commit it first (a staged brief is refused)`); + if (entry.type !== "blob" || entry.mode === "120000") throw refuse(`brief ${path} is a ${entry.mode === "120000" ? "symlink" : entry.type} in HEAD, not a file`); + const text = git(ctx, loc.top, ["cat-file", "blob", entry.oid]).toString("utf8"); + const n = countHeading(text, anchor); + if (n !== 1) throw refuse(`brief ${path} has ${n} headings "${anchor}" in HEAD; exactly one is required`); + return { path, anchor, blob: entry.oid }; +} + +// The working copy a seat reads (8.13): its git blob id, computed in-process. +function workingBriefMatches(loc, brief) { + try { + const abs = join(loc.top, brief.path); + if (!lstatSync(abs).isFile()) return false; + return gitBlobId(readFileSync(abs)) === brief.blob; + } catch { + return false; + } +} + +function resolveCandidate(ctx, loc, spec) { + const path = isAbsolute(spec) ? spec : resolvePath(ctx.cwd, spec); + let st = null; + try { st = lstatSync(path); } catch { /* not a file: try it as a commit */ } + if (st !== null) { + if (!st.isFile()) throw refuse(`candidate ${spec} is not a regular file`); + let text; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(readFileSync(path)); + } catch { + throw refuse(`candidate manifest ${spec} is not valid UTF-8`); + } + parseManifest(text); + return { kind: "manifest", digest: sha256(text), text }; + } + const out = git(ctx, loc.top, ["rev-parse", "--verify", "--quiet", "--end-of-options", `${spec}^{commit}`], { allowFail: true }); + if (out === null) throw refuse(`candidate ${spec} is neither a manifest file nor a commit`); + const sha = out.toString().trim(); + const refs = git(ctx, loc.top, ["for-each-ref", "--contains", sha, "--format=%(refname)", "refs/heads", "refs/tags"]).toString().trim(); + if (refs === "") throw refuse(`candidate commit ${sha} is not reachable from any local branch or tag`); + return { kind: "commit", digest: sha, text: null }; +} + +// --- files: queue.json, the witness, the view --- + +function statKey(st) { + return { dev: st.dev, ino: st.ino, size: st.size, mtimeNs: st.mtimeNs }; +} + +function sameStat(a, b) { + return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs; +} + +// 8.5 step 1: bytes, stat, schema, byte-for-byte form, replay, canonical checks. +function readLocked(ctx, loc) { + const st = lstatOrNull(ctx.io, loc.queuePath); + if (st === null) throw refuse(`no ${QUEUE_REL}; genesis has not run`); + const bytes = ctx.io.readFile(loc.queuePath); + const d = loadDoc(bytes); + checkCanonical(loc, d.doc.canonicalRoot, d.branch); + return { bytes, stat: statKey(st), sha: sha256(bytes), doc: d.doc, state: d.state, branch: d.branch }; +} + +function parseWitness(bytes) { + if (bytes === null) return null; + let w; + try { w = JSON.parse(bytes.toString("utf8")); } catch { return { invalid: true }; } + const ok = w !== null && typeof w === "object" && Object.keys(w).join() === "revision,logDigest,fileSha,at" + && Number.isInteger(w.revision) && w.revision >= 0 && /^[0-9a-f]{64}$/.test(w.logDigest) && /^[0-9a-f]{64}$/.test(w.fileSha) + && typeof w.at === "string"; + return ok ? w : { invalid: true }; +} + +function readWitness(ctx, loc) { + return parseWitness(readOrNull(ctx.io, loc.witnessPath)); +} + +// match, tail (the file extends the witness), lost, or absent. An invalid +// witness counts as absent. +export function compareWitness(doc, fileSha, w) { + if (w === null || w.invalid) return { state: "absent", invalid: Boolean(w?.invalid) }; + if (w.revision > doc.revision || logDigest(doc.log, w.revision) !== w.logDigest) return { state: "lost", w }; + if (w.revision === doc.revision) return w.fileSha === fileSha ? { state: "match", w } : { state: "lost", w }; + return { state: "tail", w, from: w.revision + 1 }; +} + +function writeWitness(ctx, loc, doc, bytes) { + const rec = { revision: doc.revision, logDigest: logDigest(doc.log, doc.revision), fileSha: sha256(bytes), at: ctx.now() }; + const tmp = `${loc.witnessPath}.tmp`; + unlinkQuiet(ctx.io, tmp); + writeTemp(ctx.io, tmp, Buffer.from(JSON.stringify(rec) + "\n")); + try { + ctx.io.rename(tmp, loc.witnessPath); + } catch (err) { + unlinkQuiet(ctx.io, tmp); + throw err; + } + ctx.io.fsyncDir(loc.gitDir); +} + +function confirmTail(ctx, loc, cur) { + try { + fsyncFile(ctx.io, loc.queuePath); + ctx.io.fsyncDir(loc.docsDir); + writeWitness(ctx, loc, cur.doc, cur.bytes); + } catch (err) { + throw new QueueError(`cannot confirm rev ${cur.doc.revision} durable (${errno(err)}); nothing changed`, 1); + } +} + +// The refusal for lost history or a missing witness. A file holding genesis +// alone has one more way out: `sync` confirms it (8.5 step 2). +function lostRefusal(cmp, doc) { + const way = cmp.state === "absent" && doc.log.length === 1 + ? `run \`${FIX} sync\` (the file holds genesis alone)` + : `every verb refuses except \`${FIX} accept-history\` (see the README's manual recovery)`; + return refuse(`${lostMessage(cmp, doc)}; ${way}`); +} + +function lostMessage(cmp, doc) { + if (cmp.state === "absent") { + return `the witness .git/${WITNESS_NAME} is ${cmp.invalid ? "invalid" : "missing"}; this file holds revs 0..${doc.revision}, and ops recorded after them may be lost`; + } + return `history lost: the witness recorded rev ${cmp.w.revision} (logDigest ${cmp.w.logDigest.slice(0, 12)}…, at ${cmp.w.at}); this file holds revs 0..${doc.revision} and does not extend it`; +} + +// 8.5 step 2 for a locked caller. Returns the comparison, with `confirmed` +// listing the entries this call made durable. +function witnessStep(ctx, loc, cur, { verb, op = null, args = null }) { + const cmp = compareWitness(cur.doc, cur.sha, readWitness(ctx, loc)); + const log = cur.doc.log; + const isRetry = (e) => e.op === op && e.verb === verb && sameJson(e.args, args); + if (cmp.state === "match") return { ...cmp, confirmed: [] }; + if (cmp.state === "tail") { + const tail = log.slice(cmp.from); + if (verb === "sync" || tail.some(isRetry)) { + confirmTail(ctx, loc, cur); + return { ...cmp, confirmed: tail }; + } + throw refuse(`unconfirmed tail: ${describeUnshown(tail)} visible but not confirmed durable; run \`${FIX} sync\` or retry that op`); + } + if (cmp.state === "absent" && log.length === 1 && (verb === "sync" || isRetry(log[0]))) { + confirmTail(ctx, loc, cur); + return { ...cmp, confirmed: log }; + } + if (verb === "accept-history") return { ...cmp, confirmed: [] }; + throw lostRefusal(cmp, cur.doc); +} + +function readView(ctx, loc) { + const bytes = readOrNull(ctx.io, loc.viewPath); + return { bytes, text: bytes === null ? null : bytes.toString("utf8") }; +} + +function staleMessage(view, log) { + const last = log.length - 1; + const lines = view.unshown.map((e) => `rev ${e.rev} (op ${e.op} by ${e.by} at ${e.at})`); + return `view stale: QUEUE.md shows rev ${view.shown}; ${lines.join(", ")} ${view.unshown.length === 1 ? "is" : "are"} recorded but the table shows rev ${view.shown}` + + ` and may never have been acknowledged. Tell ${[...new Set(view.unshown.map((e) => e.by))].join(", ")}, then run \`${FIX} render\` (the file is at rev ${last})`; +} + +function unknownMessage(view) { + return `view unknown: ${view.why}; restore the table with git or re-apply the edit as queue ops, then run \`${FIX} render\``; +} + +// 8.5 step 11. Returns null, or the warning that the view was not written. +function writeView(ctx, loc, before, parts, body, tag) { + const stale = `QUEUE.md changed since it was read; the view was not written and is stale. Check it, then run \`${FIX} render\``; + const now = readOrNull(ctx.io, loc.viewPath); + if (now === null || !now.equals(before)) return stale; + const tmp = `${loc.viewPath}.${tag}.tmp`; + try { + const mode = Number(ctx.io.stat(loc.viewPath).mode & 0o777n); + unlinkQuiet(ctx.io, tmp); + writeTemp(ctx.io, tmp, Buffer.from(parts.head + body + parts.tail), mode); + const again = readOrNull(ctx.io, loc.viewPath); + if (again === null || !again.equals(before)) { + unlinkQuiet(ctx.io, tmp); + return stale; + } + ctx.io.rename(tmp, loc.viewPath); + ctx.io.fsyncDir(loc.docsDir); + return null; + } catch (err) { + unlinkQuiet(ctx.io, tmp); + return `the view write failed (${errno(err)}); the op stands and the view is stale; run \`${FIX} render\``; + } +} + +// 8.5 steps 6–9. `exclusive` (genesis) links instead of renaming, so an +// existing queue.json is never replaced. +function writeQueue(ctx, loc, cur, bytes, op, rev, exclusive = false) { + const { io } = ctx; + const tmp = `${loc.queuePath}.${op}.tmp`; + // Only the lock holder writes these; one left here is from a killed writer. + unlinkQuiet(io, tmp); + try { + writeTemp(io, tmp, bytes, 0o644); + } catch (err) { + throw new QueueError(`cannot write ${QUEUE_REL}.${op}.tmp (${errno(err)}); nothing changed`, 1); + } + ctx.hook("temp-written"); + let unchanged; + try { + if (exclusive) { + unchanged = lstatOrNull(io, loc.queuePath) === null; + } else { + const st = lstatOrNull(io, loc.queuePath); + const now = st === null ? null : io.readFile(loc.queuePath); + unchanged = st !== null && sameStat(statKey(st), cur.stat) && now.equals(cur.bytes); + } + } catch (err) { + unlinkQuiet(io, tmp); + throw new QueueError(`cannot recheck ${QUEUE_REL} (${errno(err)}); nothing changed`, 1); + } + if (!unchanged) { + unlinkQuiet(io, tmp); + throw refuse(`${QUEUE_REL} changed outside the queue lock (git?) since it was read; nothing changed; retry the same op`); + } + try { + if (exclusive) { + io.link(tmp, loc.queuePath); + unlinkQuiet(io, tmp); + } else { + io.rename(tmp, loc.queuePath); + } + } catch (err) { + unlinkQuiet(io, tmp); + throw new QueueError(`cannot replace ${QUEUE_REL} (${errno(err)}); nothing changed`, 1); + } + ctx.hook("renamed"); + try { + io.fsyncDir(loc.docsDir); + } catch (err) { + throw new QueueError(`uncertain ${op} rev ${rev}: visible, durability not confirmed (${errno(err)})`, 3); + } + ctx.hook("dir-synced"); +} + +function commitWrite(ctx, loc, cur, doc, bytes, op, view, body, exclusive) { + const rev = doc.revision; + writeQueue(ctx, loc, cur, bytes, op, rev, exclusive); + try { + writeWitness(ctx, loc, doc, bytes); + } catch (err) { + throw new QueueError(`uncertain ${op} rev ${rev}: durable, witness not updated (${errno(err)})`, 3); + } + ctx.hook("witnessed"); + const warn = writeView(ctx, loc, view.bytes, view.parts, body, op); + ctx.hook("viewed"); + return warn; +} + +function withLock(ctx, loc, { op = null, verb }, fn) { + checkPlatform(ctx.io, [loc.docsDir, loc.gitDir]); + const handle = acquire({ gitDir: loc.gitDir, io: ctx.io, proc: ctx.proc, op, verb, waitMs: ctx.lockWaitMs, stepMs: ctx.lockStepMs, hook: ctx.hook }); + const res = { out: [], err: [], code: 0 }; + try { + ctx.hook("locked"); + fn(res); + } finally { + let msg; + try { msg = release(handle, ctx.io); } catch (err) { msg = `cannot release the queue lock (${errno(err)})`; } + if (msg) res.err.push(`warning: ${msg}`); + } + return res; +} + +function checkCallerOp(op) { + if (typeof op !== "string" || op === "") throw new QueueError("--op ID is required; choose it before the first attempt and reuse it on every retry", 4); + if (!CALLER_OP_RE.test(op)) throw refuse(`op id ${JSON.stringify(op)} must match ${CALLER_OP_RE.source} (8 to 72 characters)`); + if (op.endsWith(".outcome")) throw refuse(`op ids ending in .outcome are reserved`); +} + +// --- mutations --- + +function resolveFor(ctx, loc, cur, verb, args, cmp) { + if (verb === "add") { + const { path, anchor } = parseBriefSpec(args.brief); + return { brief: briefCheck(ctx, loc, path, anchor) }; + } + if (verb === "set" && args.field === "brief") { + const { path, anchor } = parseBriefSpec(args.value); + return { brief: briefCheck(ctx, loc, path, anchor) }; + } + if (verb === "move") { + const row = cur.state.rows.get(args.id); + if (args.candidate !== null && row?.state === "in-progress" && args.to === "in-review") return { candidate: resolveCandidate(ctx, loc, args.candidate) }; + return { candidate: null }; + } + if (verb === "accept-history") return { oldWitness: cmp.w ? { revision: cmp.w.revision, logDigest: cmp.w.logDigest } : null }; + return {}; +} + +// Every logged verb except genesis. `yes` is accept-history's confirmation; +// it is not part of the op's identity. +export function mutate(opts, { verb, op, args, by, yes = false }) { + const ctx = makeCtx(opts); + checkCallerOp(op); + const actor = actorOf(ctx, by); + const cargs = canonArgs(verb, args); + if (verb === "genesis") return genesis(ctx, op, cargs, actor); + const loc = locate(ctx); + checkActorSeat(loc.top, actor); + if (!headHas(ctx, loc, QUEUE_REL)) throw refuse(`genesis not committed: HEAD has no ${QUEUE_REL}; commit genesis with \`scripts/queue-commit.sh --genesis\` first`); + return withLock(ctx, loc, { op, verb }, (res) => { + const cur = readLocked(ctx, loc); + const cmp = witnessStep(ctx, loc, cur, { verb, op, args: cargs }); + for (const e of cmp.confirmed) res.err.push(`durable now, never acknowledged: ${e.op} by ${e.by} at ${e.at}`); + // Step 3: a recorded op answers with its receipt, before any view check. + const prior = cur.doc.log.find((e) => e.op === op); + if (prior) { + if (prior.verb !== verb || !sameJson(prior.args, cargs)) { + throw refuse(`op ${op} is recorded at rev ${prior.rev} as ${prior.verb} with other arguments; a new operation needs a new op id`); + } + const view = classifyView(readView(ctx, loc).text, cur.doc.log); + if (view.state === "stale") res.err.push(`warning: ${staleMessage(view, cur.doc.log)}`); + if (view.state === "unknown") res.err.push(`warning: ${unknownMessage(view)}`); + res.out.push(`${prior.result.receipt} (already recorded at rev ${prior.rev})`); + return; + } + if (verb === "accept-history") { + if (cmp.state !== "lost" && cmp.state !== "absent") throw refuse("history is not lost and the witness is present; accept-history has nothing to accept"); + const range = `${lostMessage(cmp, cur.doc)}. ops in that range are no longer deduplicated`; + if (!yes) throw refuse(`${range}. Re-run with --yes to accept this file as the queue's history`); + res.err.push(range); + } + // Step 4. + const viewRead = readView(ctx, loc); + const view = classifyView(viewRead.text, cur.doc.log); + if (view.state === "unknown") throw refuse(unknownMessage(view)); + if (view.state === "stale" && verb !== "accept-history") throw refuse(staleMessage(view, cur.doc.log)); + // Step 5. + const resolved = resolveFor(ctx, loc, cur, verb, cargs, cmp); + const rev = cur.doc.revision + 1; + const entry = { rev, op, verb, args: cargs, by: actor, at: ctx.now(), semantics: SEMANTICS, result: null, viewSha: null }; + const applied = applyEntry({ ...cur.state, revision: cur.doc.revision }, entry, resolved); + if (verb === "move" && applied.result.from === "briefed" && applied.result.to === "in-progress") { + const row = cur.state.rows.get(cargs.id); + if (!workingBriefMatches(loc, row.brief)) throw refuse(`row ${row.id}: brief differs from pinned blob; ask the lead to re-pin (${row.brief.path}, pinned ${row.brief.blob})`); + } + checkNewSeats(loc.top, cur.state.rows, applied.state.rows); + entry.result = applied.result; + const body = render(rowsArray(applied.state), rev); + entry.viewSha = sha256(body); + const doc = buildDoc(cur.doc.canonicalRoot, applied.state, [...cur.doc.log, entry]); + const bytes = Buffer.from(serialize(doc)); + loadDoc(bytes); + const warn = commitWrite(ctx, loc, cur, doc, bytes, op, { bytes: viewRead.bytes, parts: view.parts }, body, false); + if (warn) res.err.push(`warning: ${warn}`); + res.out.push(entry.result.receipt); + }); +} + +// log[0] (8.2). Runs before canonicalRoot exists; its arguments are checked +// against this checkout instead. +function genesis(ctx, op, args, actor) { + if (!PRIVILEGED.has(actor)) throw refuse("only a privileged actor (jason or sage) may run genesis"); + const loc = locate(ctx); + checkActorSeat(loc.top, actor); + if (args.root !== loc.top) throw refuse(`--root ${args.root} is not this checkout's toplevel ${loc.top}`); + checkBranch(loc, args.branch); + checkCode(loc.top); + return withLock(ctx, loc, { op, verb: "genesis" }, (res) => { + if (lstatOrNull(ctx.io, loc.queuePath) !== null) { + const cur = readLocked(ctx, loc); + const g = cur.doc.log[0]; + if (g.op !== op || !sameJson(g.args, args)) throw refuse(`${QUEUE_REL} exists; genesis runs once`); + const cmp = witnessStep(ctx, loc, cur, { verb: "genesis", op, args }); + for (const e of cmp.confirmed) res.err.push(`durable now, never acknowledged: ${e.op} by ${e.by} at ${e.at}`); + res.out.push(`${g.result.receipt} (already recorded at rev 0)`); + return; + } + if (headHas(ctx, loc, QUEUE_REL)) throw refuse(`HEAD already has ${QUEUE_REL}; genesis runs once`); + if (lstatOrNull(ctx.io, loc.witnessPath) !== null) throw refuse(`the witness .git/${WITNESS_NAME} exists without ${QUEUE_REL}; diagnose by hand`); + const mapEntry = headEntry(ctx, loc, args.map); + if (mapEntry === null || mapEntry.type !== "blob" || mapEntry.mode === "120000") throw refuse(`the migration map ${args.map} is not a committed file in HEAD`); + const map = parseMigrationMap(git(ctx, loc.top, ["cat-file", "blob", mapEntry.oid]).toString("utf8")); + const blobs = new Map(); + for (const r of map.rows) { + if (r.brief !== null) blobs.set(r.id, briefCheck(ctx, loc, r.brief.path, r.brief.anchor).blob); + } + const viewRead = readView(ctx, loc); + const parts = viewRead.text === null ? null : splitView(viewRead.text); + if (parts === null) throw refuse(`${VIEW_REL} needs the two queue markers, once each and in order`); + const at = ctx.now(); + const rows = genesisRows(map, blobs, op, at, actor); + checkNewSeats(loc.top, new Map(), new Map(rows.map((r) => [r.id, r]))); + const body = render(rows, 0); + const result = { + mapBlob: mapEntry.oid, highWater: map.highWater, retired: [...map.retired].sort((a, b) => a - b), rows, + legacyView: parts.body, receipt: genesisReceipt(op, rows.length), + }; + const entry = { rev: 0, op, verb: "genesis", args, by: actor, at, semantics: SEMANTICS, result, viewSha: sha256(body) }; + const doc = { version: VERSION, canonicalRoot: args.root, revision: 0, rows, log: [entry] }; + const bytes = Buffer.from(serialize(doc)); + loadDoc(bytes); + const warn = commitWrite(ctx, loc, null, doc, bytes, op, { bytes: viewRead.bytes, parts }, body, true); + if (warn) res.err.push(`warning: ${warn}`); + res.out.push(result.receipt); + }); +} + +// --- reads without the lock (8.4) --- + +function readUnlocked(ctx, loc) { + const { io } = ctx; + let w; + let bytes; + if (ctx.readOrder === "file-first") { + bytes = readOrNull(io, loc.queuePath); + ctx.hook("reader-between"); + w = parseWitness(readOrNull(io, loc.witnessPath)); + } else { + w = parseWitness(readOrNull(io, loc.witnessPath)); + ctx.hook("reader-between"); + bytes = readOrNull(io, loc.queuePath); + } + const notes = []; + if (bytes !== null) { + let d = null; + try { d = loadDoc(bytes); } catch (err) { if (!(err instanceof QueueError)) throw err; } + if (d !== null) { + checkCanonical(loc, d.doc.canonicalRoot, d.branch); + const cmp = compareWitness(d.doc, sha256(bytes), w); + if (cmp.state === "match" || cmp.state === "tail") { + if (cmp.state === "tail") notes.push(`rev ${d.doc.revision} visible, not confirmed durable`); + return { doc: d.doc, state: d.state, notes }; + } + } + } + // Adverse: recheck under the lock before reporting anything. + let out = null; + const res = withLock(ctx, loc, { verb: "read" }, () => { + const cur = readLocked(ctx, loc); + const cmp = compareWitness(cur.doc, cur.sha, readWitness(ctx, loc)); + if (cmp.state === "tail") notes.push(`unconfirmed tail: ${describeUnshown(cur.doc.log.slice(cmp.from))} visible but not confirmed durable; run \`${FIX} sync\``); + else if (cmp.state !== "match") throw lostRefusal(cmp, cur.doc); + out = { doc: cur.doc, state: cur.state }; + }); + return { ...out, notes: [...notes, ...res.err] }; +} + +function viewNotes(ctx, loc, log) { + const view = classifyView(readView(ctx, loc).text, log); + if (view.state === "stale") return [`warning: ${staleMessage(view, log)}`]; + if (view.state === "unknown") return [`warning: ${unknownMessage(view)}`]; + return []; +} + +function readState(opts) { + const ctx = makeCtx(opts); + const loc = locate(ctx); + const r = readUnlocked(ctx, loc); + return { ctx, loc, ...r, notes: [...r.notes, ...viewNotes(ctx, loc, r.doc.log)] }; +} + +export function list(opts) { + const r = readState(opts); + const out = rowsArray(r.state).map((row) => `${row.id}\t${row.state}\t${row.owner}\t${row.piece}`); + return { out, err: r.notes, code: 0 }; +} + +export function show(opts, id) { + const r = readState(opts); + const row = r.state.rows.get(id); + if (!row) throw refuse(`no row ${id}`); + return { out: [JSON.stringify(row, null, 2)], err: r.notes, code: 0 }; +} + +export function next(opts, seat) { + const ctx = makeCtx(opts); + const name = seat ?? ctx.env.MOSAIC_AGENT_NAME; + if (name === undefined || name === "") throw refuse("next needs a seat: pass SEAT or set MOSAIC_AGENT_NAME"); + checkName(name, "seat"); + const r = readState(opts); + const n = nextFor(r.state.rows, name, (row) => workingBriefMatches(r.loc, row.brief)); + if (n.action === "nothing") return { out: ["nothing"], err: r.notes, code: 0 }; + const row = n.row; + const brief = row.brief ? `; brief ${row.brief.path} § ${row.brief.anchor}` : ""; + const flag = n.briefDiffers ? "; brief differs from pinned blob; ask the lead to re-pin" : ""; + return { out: [`${n.action} row ${row.id}: ${row.piece}${brief}${flag}`], err: r.notes, code: 0 }; +} + +// --- locked verbs that log nothing --- + +function lockedRead(opts, verb, fn) { + const ctx = makeCtx(opts); + const loc = locate(ctx); + return withLock(ctx, loc, { verb }, (res) => { + const cur = readLocked(ctx, loc); + fn(ctx, loc, cur, res); + }); +} + +export function sync(opts, op = null) { + if (op !== null) checkCallerOp(op); + return lockedRead(opts, "sync", (ctx, loc, cur, res) => { + const cmp = witnessStep(ctx, loc, cur, { verb: "sync" }); + if (cmp.confirmed.length === 0) res.out.push(`nothing to confirm: rev ${cur.doc.revision} is durable and witnessed`); + for (const e of cmp.confirmed) res.out.push(`durable now, never acknowledged: ${e.op} by ${e.by} at ${e.at}`); + res.err.push(...viewNotes(ctx, loc, cur.doc.log)); + }); +} + +export function renderView(opts, { check = false } = {}) { + return lockedRead(opts, "render", (ctx, loc, cur, res) => { + witnessStep(ctx, loc, cur, { verb: "render" }); + const log = cur.doc.log; + const viewRead = readView(ctx, loc); + const view = classifyView(viewRead.text, log); + if (view.state === "unknown") throw refuse(unknownMessage(view)); + if (view.state === "current") { + res.out.push(`view current at rev ${cur.doc.revision}${check ? "" : "; nothing written"}`); + return; + } + if (check) throw refuse(staleMessage(view, log)); + const body = render(rowsArray(cur.state), cur.doc.revision); + const warn = writeView(ctx, loc, viewRead.bytes, view.parts, body, "render"); + if (warn) throw new QueueError(warn, 1); + res.out.push(`rendered rev ${cur.doc.revision} over rev ${view.shown}; newly shown: ${describeUnshown(view.unshown)}`); + }); +} + +function briefDrift(ctx, loc, rows) { + const drift = []; + for (const row of rowsArray({ rows })) { + if (row.brief === null || row.state === "done" || row.state === "parked") continue; + const e = headEntry(ctx, loc, row.brief.path); + if (e === null || e.type !== "blob" || e.mode === "120000") { + drift.push(`row ${row.id}: brief ${row.brief.path} is no longer a file in HEAD`); + } else if (e.oid !== row.brief.blob) { + drift.push(`row ${row.id}: brief ${row.brief.path} changed in HEAD (pinned ${row.brief.blob}, HEAD ${e.oid})`); + } + } + return drift; +} + +export function verify(opts, { current = false } = {}) { + return lockedRead(opts, "verify", (ctx, loc, cur, res) => { + witnessStep(ctx, loc, cur, { verb: "verify" }); + const view = classifyView(readView(ctx, loc).text, cur.doc.log); + if (view.state === "unknown") throw refuse(unknownMessage(view)); + if (view.state === "stale") throw refuse(staleMessage(view, cur.doc.log)); + if (current) { + const drift = briefDrift(ctx, loc, cur.state.rows); + if (drift.length) throw refuse(`brief drift against HEAD:\n${drift.join("\n")}`); + } + res.out.push(`ok verify rev ${cur.doc.revision}: file valid, witness matches, view current${current ? ", briefs match HEAD" : ""}`); + }); +} + +// 8.12 step 4. Reads only DIR/queue.json, DIR/QUEUE.md and the base: no git, +// no working directory, no lock. It does not certify witness continuity. +export function verifySnapshot(dir, { baseFile = null, baseAbsent = false } = {}) { + if (baseAbsent === (baseFile !== null)) throw new QueueError("verify --snapshot needs exactly one of --base-file F or --base-absent", 4); + const bytes = readFileOrRefuse(join(dir, "queue.json")); + const viewText = readFileOrRefuse(join(dir, "QUEUE.md")).toString("utf8"); + const { doc } = loadDoc(bytes); + const view = classifyView(viewText, doc.log); + if (view.state !== "current") throw refuse(`snapshot QUEUE.md is not the render of rev ${doc.revision} (${view.state})`); + if (baseAbsent) { + if (doc.log.length !== 1) throw refuse(`the base is absent, so the snapshot must hold genesis alone; it holds revs 0..${doc.revision}`); + } else { + const base = loadDoc(readFileOrRefuse(baseFile)).doc; + if (base.canonicalRoot !== doc.canonicalRoot) throw refuse("the snapshot's canonicalRoot differs from the base's"); + if (doc.revision < base.revision || logDigest(doc.log, base.revision) !== logDigest(base.log, base.revision)) { + throw refuse(`the snapshot's log does not extend the base's (base rev ${base.revision}, snapshot rev ${doc.revision})`); + } + } + return { out: [`ok verify --snapshot rev ${doc.revision}: pair valid, view current, ${baseAbsent ? "genesis alone" : "extends the base"}`], err: [], code: 0 }; +} + +function readFileOrRefuse(path) { + try { + return readFileSync(path); + } catch (err) { + throw refuse(`cannot read ${path} (${errno(err)})`); + } +} + +export function snapshot(opts, outDir) { + const ctx = makeCtx(opts); + const dir = resolvePath(ctx.cwd, outDir); + let real; + try { real = realpathSync(dir); } catch { throw refuse(`--out ${dir} does not exist`); } + if (!lstatSync(real).isDirectory()) throw refuse(`--out ${dir} is not a directory`); + if (readdirSync(real).length !== 0) throw refuse(`--out ${dir} is not empty`); + return lockedRead(opts, "snapshot", (c, loc, cur, res) => { + if (real === loc.top || real.startsWith(`${loc.top}/`)) throw refuse(`--out ${dir} is inside the repository`); + witnessStep(c, loc, cur, { verb: "snapshot" }); + const viewRead = readView(c, loc); + const view = classifyView(viewRead.text, cur.doc.log); + if (view.state !== "current") throw refuse(view.state === "stale" ? staleMessage(view, cur.doc.log) : unknownMessage(view)); + writeTemp(c.io, join(real, "queue.json"), cur.bytes); + writeTemp(c.io, join(real, "QUEUE.md"), viewRead.bytes); + res.out.push(`snapshot rev ${cur.doc.revision}: queue.json ${cur.sha}, QUEUE.md ${sha256(viewRead.bytes)}`); + }); +} + +// --- the lock by hand --- + +function unlockLoc(ctx) { + const loc = locate(ctx); + const bytes = readOrNull(ctx.io, loc.queuePath); + if (bytes !== null) { + let d = null; + try { d = loadDoc(bytes); } catch { /* an invalid file does not keep a dead lock in place */ } + if (d !== null) checkCanonical(loc, d.doc.canonicalRoot, d.branch); + } + checkPlatform(ctx.io, [loc.gitDir]); + return loc; +} + +export function unlock(opts, { checkGateOnly = false } = {}) { + const ctx = makeCtx(opts); + const loc = unlockLoc(ctx); + if (checkGateOnly) return { out: [checkGate({ gitDir: loc.gitDir, io: ctx.io, proc: ctx.proc }).line], err: [], code: 0 }; + return { out: [unlockLock({ gitDir: loc.gitDir, io: ctx.io, proc: ctx.proc, hook: ctx.hook })], err: [], code: 0 }; +} diff --git a/packages/queue/tests/commit.test.mjs b/packages/queue/tests/commit.test.mjs new file mode 100644 index 00000000..55105163 --- /dev/null +++ b/packages/queue/tests/commit.test.mjs @@ -0,0 +1,541 @@ +// scripts/queue-commit.sh and the queue guard (8.12): F1, the bootstrap (F3) +// and the general cases. Every run is in a scratch repository with the +// scripts committed; nothing touches this checkout's .git. +// +// Schedules use PATH shims for git and node. Each shim calls +// /on-git or /on-node, when present, before and after the real +// command, and the call blocks, so an action runs at an exact point in the +// procedure. Actions run with the original PATH, so they reach real git. +import { strict as assert } from "node:assert"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { sleepMs } from "../src/io.mjs"; +import { cli, scratchRepo, sh } from "./helpers.mjs"; + +const QJSON = "docs/plans/queue.json"; +const QMD = "docs/plans/QUEUE.md"; +const FIX = `git reset -q -- ${QJSON} ${QMD}`; +const GUARD = "mosaic queue guard: refused"; +const REAL_GIT = sh("sh", ["-c", "command -v git"]).stdout.trim(); + +function q(s) { + return `'${String(s).replaceAll("'", "'\\''")}'`; +} + +function sha256(buf) { + return createHash("sha256").update(buf).digest("hex"); +} + +// A repository with the scripts, the guard installed by sage and, unless +// `bootstrap` is set, genesis committed through queue-commit.sh. +function ready(t, { bootstrap = false } = {}) { + const repo = scratchRepo(t, { scripts: true }); + const ctl = join(repo.base, "ctl"); + const shimDir = join(repo.base, "shims"); + mkdirSync(ctl); + mkdirSync(shimDir); + const origPath = repo.env.PATH; + for (const [name, real] of [["git", REAL_GIT], ["node", process.execPath]]) { + const on = join(ctl, `on-${name}`); + writeFileSync(join(shimDir, name), [ + "#!/bin/sh", + `if [ -x ${q(on)} ]; then PATH=${q(origPath)} ${q(on)} before "$@" >${q(join(ctl, "log"))} 2>&1; fi`, + `${q(real)} "$@"; s=$?`, + `if [ -x ${q(on)} ]; then PATH=${q(origPath)} ${q(on)} after "$s" "$@" >${q(join(ctl, "log"))} 2>&1; fi`, + "exit $s", + "", + ].join("\n"), { mode: 0o755 }); + } + const r = { + ...repo, + ctl, + hook: join(repo.gitDir, "hooks/pre-commit"), + // Runs queue-commit.sh through the shims. + qc(args, env = {}) { + const res = sh("bash", [join(repo.root, "scripts/queue-commit.sh"), ...args], { + cwd: repo.root, env: { ...repo.env, PATH: `${shimDir}:${origPath}`, ...env }, allowFail: true, + }); + return { code: res.status, out: res.stdout, err: res.stderr }; + }, + // Installs an action. `body` is bash; $phase is before or after, $status + // the exit status after, $sub the git subcommand, $@ the arguments. + on(name, body) { + writeFileSync(join(ctl, `on-${name}`), [ + "#!/bin/bash", + "phase=$1; shift; status=", + 'if [ "$phase" = after ]; then status=$1; shift; fi', + 'a=("$@"); i=0; while [ "${a[i]:-}" = -C ]; do i=$((i+2)); done; sub=${a[i]:-}', + `CTL=${q(ctl)}; R=${q(repo.root)}`, + "unset GIT_INDEX_FILE", + 'once() { [ -e "$CTL/once-$1" ] && return 1; : > "$CTL/once-$1"; }', + 'echo "$phase $sub" >> "$CTL/calls-$(basename "$0")"', + body, + "", + ].join("\n"), { mode: 0o755 }); + }, + off(name) { + rmSync(join(ctl, `on-${name}`), { force: true }); + }, + calls(name) { + const p = join(ctl, `calls-on-${name}`); + return existsSync(p) ? readFileSync(p, "utf8").split("\n").filter(Boolean) : []; + }, + ctlFile(name) { + return readFileSync(join(ctl, name), "utf8").trim(); + }, + head() { + return repo.g("rev-parse", "HEAD").trim(); + }, + blob(rev, path) { + return sh("git", ["-C", repo.root, "cat-file", "blob", `${rev}:${path}`], { env: repo.env }).stdout; + }, + revAt(rev) { + return JSON.parse(r.blob(rev, QJSON)).revision; + }, + }; + const inst = r.qc(["--install-hook", "--by", "sage"]); + assert.equal(inst.code, 0, inst.err); + if (!bootstrap) { + genesis(r); + const c = r.qc(["--genesis", "-m", "queue genesis"]); + assert.equal(c.code, 0, c.err); + } + return r; +} + +function genesis(r) { + const g = cli(r, ["genesis", "--op", "genesis-2026-09-26", "--root", r.root, "--branch", "refactor", "--map", "agents/sage/work/queue-migration-map.md"], { by: "sage" }); + assert.equal(g.code, 0, g.err); +} + +let noteN = 0; +function note(r, id = 6) { + const res = cli(r, ["note", String(id), `note ${++noteN}`, "--op", `note-op-${noteN}`], { by: "darkwing" }); + assert.equal(res.code, 0, res.err); + return res; +} + +function stageFile(r, name, text = "source\n") { + writeFileSync(join(r.root, name), text); + r.g("add", "--", name); +} + +function assertUntouched(r, head, res, why) { + assert.equal(res.code, 2, `${why}: ${res.err}`); + assert.equal(r.head(), head, why); + assert.ok(!r.calls("git").includes("before update-ref"), `${why}: update-ref ran`); +} + +// --- F1 --- + +test("F1: an ordinary commit after update-ref is refused until step 8; then it commits and the queue stays at C", (t) => { + const r = ready(t); + note(r); + r.on("git", `if [ "$phase $sub $status" = "after update-ref 0" ]; then + echo src > "$R/src.txt"; git -C "$R" add src.txt + git -C "$R" commit -q -m ordinary 2> "$CTL/commit-err"; echo $? > "$CTL/commit-rc" +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); + assert.notEqual(r.ctlFile("commit-rc"), "0"); + assert.match(r.ctlFile("commit-err"), new RegExp(GUARD)); + assert.match(r.ctlFile("commit-err"), new RegExp(FIX.replaceAll(".", "\\."))); + const c = r.head(); + assert.equal(r.revAt(c), 1); + r.off("git"); + r.g("commit", "-q", "-m", "ordinary"); + assert.equal(r.g("rev-parse", "HEAD^").trim(), c); + assert.equal(r.revAt("HEAD"), 1); + assert.equal(r.blob("HEAD", "src.txt"), "src\n"); +}); + +test("F1: a commit whose guard ran before update-ref fails at its own HEAD update", async (t) => { + const r = ready(t); + note(r); + stageFile(r, "src.txt"); + const h = r.head(); + const started = join(r.ctl, "editor-started"); + const go = join(r.ctl, "editor-go"); + const editor = join(r.ctl, "editor.sh"); + writeFileSync(editor, `#!/bin/sh\n: > ${q(started)}\nwhile [ ! -e ${q(go)} ]; do sleep 0.05; done\necho "ordinary" > "$1"\n`, { mode: 0o755 }); + const child = spawn("git", ["-C", r.root, "commit", "-e", "-q"], { env: { ...r.env, GIT_EDITOR: editor }, stdio: ["ignore", "pipe", "pipe"] }); + let childErr = ""; + child.stderr.on("data", (d) => { childErr += d; }); + const exited = new Promise((resolve) => child.on("exit", resolve)); + for (let i = 0; i < 200 && !existsSync(started); i++) sleepMs(50); + assert.ok(existsSync(started), "the editor never started"); + // The paused commit ran its guard against H and holds index.lock. + const res = r.qc(["-m", "queue rev 1"]); + writeFileSync(go, ""); + const code = await exited; + // git 2.55 does not hold index.lock while the editor runs, so step 8 + // reconciles; the paused commit then loses at its HEAD update. + assert.equal(res.code, 0, res.err); + const c = r.head(); + assert.equal(r.g("rev-parse", "HEAD^").trim(), h); + assert.notEqual(code, 0); + assert.match(childErr, new RegExp(`cannot lock ref 'HEAD': is at ${c} but expected ${h}`)); + assert.equal(r.head(), c, "the old queue landed on top of C"); + assert.equal(r.g("diff", "--cached", "--name-only").trim(), "src.txt"); + r.g("commit", "-q", "-m", "ordinary"); + assert.equal(r.revAt("HEAD"), 1); +}); + +test("F1: step 8 with index.lock held exits 3, and ordinary commits stay refused until the printed command runs", (t) => { + const r = ready(t); + note(r); + r.on("git", `if [ "$phase $sub" = "before ls-files" ] && once lock; then : > "$R/.git/index.lock"; fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 3, res.err); + assert.match(res.err, /holds \.git\/index\.lock/); + assert.ok(res.err.includes(`run: ${FIX}`) || res.err.includes(`until this runs: ${FIX}`)); + const c = r.head(); + assert.equal(r.revAt(c), 1); + r.off("git"); + unlinkSync(join(r.gitDir, "index.lock")); + stageFile(r, "src.txt"); + const refused = sh("git", ["-C", r.root, "commit", "-q", "-m", "ordinary"], { env: r.env, allowFail: true }); + assert.notEqual(refused.status, 0); + assert.match(refused.stderr, new RegExp(GUARD)); + assert.equal(r.head(), c); + sh("sh", ["-c", FIX], { cwd: r.root, env: r.env }); + r.g("commit", "-q", "-m", "ordinary"); + assert.equal(r.revAt("HEAD"), 1); + assert.equal(r.g("rev-parse", "HEAD^").trim(), c); +}); + +test("F1: HEAD moving after commit-tree and before update-ref: refused, nothing published", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + r.on("git", `if [ "$phase $sub" = "after commit-tree" ] && once move; then + echo other > "$R/other.txt"; git -C "$R" add other.txt; git -C "$R" commit -q -m other +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 1, res.err); + assert.match(res.err, new RegExp(`refs/heads/refactor moved since ${h}; nothing published; start again`)); + assert.equal(r.g("rev-parse", "HEAD^").trim(), h); + assert.equal(r.revAt("HEAD"), 0); + assert.equal(r.g("log", "-1", "--format=%s").trim(), "other"); +}); + +test("F1: H is recorded before the canary, so HEAD moving during the canary makes update-ref fail", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + r.on("git", `if [ "$phase $sub" = "before hook" ] && once move; then + echo other > "$R/other.txt"; git -C "$R" add other.txt; git -C "$R" commit -q -m other +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 1, res.err); + assert.match(res.err, new RegExp(`moved since ${h}`)); + assert.equal(r.g("log", "-1", "--format=%s").trim(), "other"); + assert.equal(r.g("rev-parse", "HEAD^").trim(), h); + assert.equal(r.revAt("HEAD"), 0); + // Both guard checks ran in full (two canary runs each), so only update-ref caught the move. + assert.equal(r.calls("git").filter((c) => c === "before hook").length, 4); +}); + +test("F1: a shared-index change during the procedure is not committed", (t) => { + const r = ready(t); + note(r); + r.on("git", `if [ "$phase $sub" = "before commit-tree" ]; then echo late > "$R/late.txt"; git -C "$R" add late.txt; fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); + assert.deepEqual(r.g("diff-tree", "-r", "--name-only", "HEAD^", "HEAD").trim().split("\n"), [QMD, QJSON]); + assert.equal(r.g("diff", "--cached", "--name-only").trim(), "late.txt"); +}); + +test("F1: a queue path staged after update-ref: step 8 stops and touches nothing", (t) => { + const r = ready(t); + note(r); + r.on("git", `if [ "$phase $sub $status" = "after update-ref 0" ]; then + git -C "$R" add docs/plans/queue.json; sha256sum "$R/.git/index" | cut -d' ' -f1 > "$CTL/index-sha" +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 3, res.err); + assert.match(res.err, /someone staged a queue path; the index was not touched/); + assert.equal(sha256(readFileSync(join(r.gitDir, "index"))), r.ctlFile("index-sha")); + assert.equal(r.revAt("HEAD"), 1); +}); + +test("F1: a missing or a different hook refuses", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + const bytes = readFileSync(r.hook); + r.on("git", ""); + unlinkSync(r.hook); + assertUntouched(r, h, r.qc(["-m", "x"]), "missing"); + assert.match(r.qc(["-m", "x"]).err, /queue guard is not installed/); + writeFileSync(r.hook, `${bytes}\n# edited\n`, { mode: 0o755 }); + const diff = r.qc(["-m", "x"]); + assertUntouched(r, h, diff, "different"); + assert.match(diff.err, /differs from scripts\/git-hooks\/pre-commit/); + const inst = r.qc(["--install-hook", "--by", "sage"]); + assert.equal(inst.code, 2); + assert.match(inst.err, /a different pre-commit hook exists/); +}); + +test("F1: same bytes without the exec bit, a symlinked hook, and core.hooksPath in the local or global scope each refuse before update-ref", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + const bytes = readFileSync(r.hook); + r.on("git", ""); + chmodSync(r.hook, 0o644); + let res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "no exec bit"); + assert.match(res.err, /not executable/); + chmodSync(r.hook, 0o755); + const target = join(r.base, "hook-copy"); + writeFileSync(target, bytes, { mode: 0o755 }); + unlinkSync(r.hook); + symlinkSync(target, r.hook); + res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "symlink"); + assert.match(res.err, /is a symlink/); + unlinkSync(r.hook); + writeFileSync(r.hook, bytes, { mode: 0o755 }); + r.g("config", "core.hooksPath", ".githooks"); + res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "local hooksPath"); + assert.match(res.err, /core\.hooksPath is set \(local/); + r.g("config", "--unset", "core.hooksPath"); + r.g("config", "--global", "core.hooksPath", "/nonexistent-hooks"); + res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "global hooksPath"); + assert.match(res.err, /core\.hooksPath is set \(global/); + r.g("config", "--global", "--unset", "core.hooksPath"); + res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); +}); + +test("F1: the canary refuses a hook that git would not run", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + r.on("git", `if [ "$phase $sub" = "before hook" ] && once chmod; then chmod 0644 "$R/.git/hooks/pre-commit"; fi`); + const res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "canary"); + assert.match(res.err, /refused \(step1\): the canary's clean run failed/); +}); + +test("F1: the guard deactivated after step 1 is refused at the step-7 recheck", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + r.on("git", `if [ "$phase $sub" = "after commit-tree" ]; then chmod 0644 "$R/.git/hooks/pre-commit"; fi`); + const res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "step 7"); + assert.match(res.err, /refused \(step7\): \.git\/hooks\/pre-commit is not executable/); + assert.ok(r.calls("git").includes("after commit-tree")); +}); + +// --- bootstrap (F3) --- + +test("bootstrap: implementation-only HEAD, the guard, genesis, the --genesis commit, then an extending commit", (t) => { + const r = ready(t, { bootstrap: true }); + const impl = r.head(); + assert.equal(sh("git", ["-C", r.root, "cat-file", "-e", `HEAD:${QJSON}`], { env: r.env, allowFail: true }).status, 128); + assert.equal(readFileSync(r.hook, "utf8"), r.blob("HEAD", "scripts/git-hooks/pre-commit")); + genesis(r); + const g = r.qc(["--genesis", "-m", "queue genesis"]); + assert.equal(g.code, 0, g.err); + assert.match(g.err, /ok verify --snapshot rev 0: pair valid, view current, genesis alone/); + assert.equal(r.g("rev-parse", "HEAD^").trim(), impl); + assert.equal(r.revAt("HEAD"), 0); + assert.equal(r.g("status", "--porcelain").trim(), ""); + note(r); + const c = r.qc(["-m", "queue rev 1"]); + assert.equal(c.code, 0, c.err); + assert.match(c.err, /extends the base/); + assert.equal(r.revAt("HEAD"), 1); + assert.equal(r.g("status", "--porcelain").trim(), ""); +}); + +test("bootstrap: --genesis with a base present, no base without --genesis, an op before the first commit, a changed map, another branch", (t) => { + const r = ready(t, { bootstrap: true }); + let res = r.qc(["-m", "x"]); + assert.equal(res.code, 2); + assert.match(res.err, /HEAD has no docs\/plans\/queue\.json; the first queue commit needs --genesis/); + genesis(r); + res = r.qc(["-m", "x"]); + assert.equal(res.code, 2); + assert.match(res.err, /needs --genesis/); + const op = cli(r, ["note", "6", "early", "--op", "note-early-1"], { by: "darkwing" }); + assert.equal(op.code, 2); + assert.match(op.err, /genesis not committed/); + // The map changed in HEAD after genesis read it. + const mapPath = join(r.root, "agents/sage/work/queue-migration-map.md"); + const mapText = readFileSync(mapPath, "utf8"); + writeFileSync(mapPath, `${mapText}\nEdited.\n`); + r.g("add", "agents/sage/work/queue-migration-map.md"); + r.g("commit", "-q", "-m", "map edit"); + res = r.qc(["--genesis", "-m", "x"]); + assert.equal(res.code, 2); + assert.match(res.err, /genesis read map blob [0-9a-f]{40}, but HEAD's agents\/sage\/work\/queue-migration-map\.md is/); + writeFileSync(mapPath, mapText); + r.g("add", "agents/sage/work/queue-migration-map.md"); + r.g("commit", "-q", "-m", "map restored"); + r.g("checkout", "-q", "-b", "other"); + res = r.qc(["--genesis", "-m", "x"]); + assert.equal(res.code, 2); + r.g("checkout", "-q", "refactor"); + res = r.qc(["--genesis", "-m", "queue genesis"]); + assert.equal(res.code, 0, res.err); + res = r.qc(["--genesis", "-m", "x"]); + assert.equal(res.code, 2); + assert.match(res.err, /--genesis, but HEAD already has docs\/plans\/queue\.json/); +}); + +test("bootstrap: the archived tests and validator run outside any repository", (t) => { + const r = ready(t); + note(r); + r.on("node", `if [ "$phase" = before ] && { [ "$1" = --test ] || [ "$3" = --snapshot ]; }; then + { echo "$1|$(pwd -P)|\${GIT_CEILING_DIRECTORIES:-}|$(git rev-parse --show-toplevel >/dev/null 2>&1; echo $?)"; } >> "$CTL/node-where" +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); + const lines = r.ctlFile("node-where").split("\n"); + assert.equal(lines.length, 2); + for (const line of lines) { + const [first, cwd, ceiling, rc] = line.split("|"); + assert.ok(first === "--test" || first === "packages/queue/src/cli.mjs", first); + assert.ok(!cwd.startsWith(`${r.root}/`) && cwd !== r.root, cwd); + assert.ok(cwd.startsWith(`${ceiling}/`), `${cwd} under ${ceiling}`); + assert.notEqual(rc, "0", "a repository was found around the archive"); + } +}); + +// --- general --- + +test("general: an unrelated staged file stays staged, and the committed blobs are the snapshot bytes", (t) => { + const r = ready(t); + note(r); + stageFile(r, "src.txt"); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); + const m = /snapshot rev 1: queue\.json ([0-9a-f]{64}), QUEUE\.md ([0-9a-f]{64})/.exec(res.out); + assert.ok(m, res.out); + assert.equal(sha256(r.blob("HEAD", QJSON)), m[1]); + assert.equal(sha256(r.blob("HEAD", QMD)), m[2]); + assert.equal(r.blob("HEAD", QJSON), readFileSync(join(r.root, QJSON), "utf8")); + assert.equal(r.g("diff", "--cached", "--name-only").trim(), "src.txt"); + assert.equal(sh("git", ["-C", r.root, "cat-file", "-e", "HEAD:src.txt"], { env: r.env, allowFail: true }).status, 128); +}); + +test("general: a queue write after the snapshot is not committed", (t) => { + const r = ready(t); + note(r); + const cliPath = join(r.root, "packages/queue/src/cli.mjs"); + r.on("node", `if [ "$phase $status $2" = "after 0 snapshot" ] && once write; then + MOSAIC_AGENT_NAME=darkwing node ${q(cliPath)} note 6 late --op note-late-1 > "$CTL/late-out" 2>&1 +fi`); + const res = r.qc(["-m", "queue rev 1"]); + assert.equal(res.code, 0, res.err); + assert.match(r.ctlFile("late-out"), /ok note-late-1 rev 2/); + assert.equal(r.revAt("HEAD"), 1); + assert.equal(JSON.parse(readFileSync(join(r.root, QJSON), "utf8")).revision, 2); + r.off("node"); + const again = r.qc(["-m", "queue rev 2"]); + assert.equal(again.code, 0, again.err); + assert.equal(r.revAt("HEAD"), 2); +}); + +test("general: a snapshot whose log does not extend the base refuses", (t) => { + const r = ready(t); + note(r); + // A bypass commit of rev 1, then the working files rolled back to rev 0 + // and the loss accepted: the working log forks from HEAD's. + r.g("add", QJSON, QMD); + r.g("commit", "-q", "--no-verify", "-m", "bypass"); + writeFileSync(join(r.root, QJSON), r.blob("HEAD^", QJSON)); + writeFileSync(join(r.root, QMD), r.blob("HEAD^", QMD)); + const acc = cli(r, ["accept-history", "--op", "accept-1", "--reason", "test rollback", "--yes"], { by: "sage" }); + assert.equal(acc.code, 0, acc.err); + const h = r.head(); + r.on("git", ""); + const res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "fork"); + assert.match(res.err, /the snapshot's log does not extend the base's \(base rev 1, snapshot rev 1\)/); +}); + +test("general: install-hook privilege, repair of a same-bytes hook, and its refusals", (t) => { + const r = ready(t, { bootstrap: true }); + let res = r.qc(["--install-hook"]); + assert.equal(res.code, 2); + assert.match(res.err, /no actor/); + res = r.qc(["--install-hook"], { MOSAIC_AGENT_NAME: "darkwing" }); + assert.equal(res.code, 2); + assert.match(res.err, /privileged \(jason or sage\), not darkwing/); + chmodSync(r.hook, 0o644); + res = r.qc(["--install-hook"], { MOSAIC_AGENT_NAME: "jason" }); + assert.equal(res.code, 0, res.err); + assert.match(res.out, /the same bytes were already there/); + assert.equal(sh("stat", ["-c", "%a", r.hook]).stdout.trim(), "755"); + const bytes = readFileSync(r.hook); + unlinkSync(r.hook); + const target = join(r.base, "hook-copy"); + writeFileSync(target, bytes, { mode: 0o755 }); + symlinkSync(target, r.hook); + res = r.qc(["--install-hook", "--by", "sage"]); + assert.equal(res.code, 2); + assert.match(res.err, /is a symlink/); + unlinkSync(r.hook); + r.g("config", "core.hooksPath", ".githooks"); + res = r.qc(["--install-hook", "--by", "sage"]); + assert.equal(res.code, 2); + assert.match(res.err, /core\.hooksPath is set/); + assert.ok(!existsSync(r.hook)); +}); + +test("general: environment overrides, a linked worktree and usage", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + for (const k of ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"]) { + const res = r.qc(["-m", "x"], { [k]: join(r.root, ".git") }); + assert.equal(res.code, 2, k); + assert.match(res.err, new RegExp(`${k} is set`)); + } + const wt = join(r.base, "wt"); + r.g("worktree", "add", "-q", "-b", "wt", wt); + const w = sh("bash", [join(wt, "scripts/queue-commit.sh"), "-m", "x"], { cwd: wt, env: r.env, allowFail: true }); + assert.equal(w.status, 2); + assert.match(w.stderr, /linked worktree/); + for (const args of [[], ["-m", ""], ["--by", "sage", "-m", "x"], ["--install-hook", "-m", "x"], ["--bogus"], ["-m"]]) { + assert.equal(r.qc(args).code, 4, JSON.stringify(args)); + } + assert.equal(r.head(), h); +}); + +test("general: a queue path staged before the run refuses at step 1", (t) => { + const r = ready(t); + note(r); + const h = r.head(); + r.g("add", QMD); + r.on("git", ""); + const res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "staged"); + assert.match(res.err, /the shared index has staged changes to docs\/plans\/queue\.json or docs\/plans\/QUEUE\.md/); + assert.ok(res.err.includes(FIX)); + assert.ok(!r.calls("git").includes("before commit-tree")); +}); + +test("general: HEAD's queue tests failing in the archive refuse", (t) => { + const r = ready(t); + writeFileSync(join(r.root, "packages/queue/tests/fail.test.mjs"), 'import { test } from "node:test";\ntest("fails", () => { throw new Error("archived failure"); });\n'); + r.g("add", "packages/queue/tests/fail.test.mjs"); + r.g("commit", "-q", "-m", "failing test"); + note(r); + const h = r.head(); + r.on("git", ""); + const res = r.qc(["-m", "x"]); + assertUntouched(r, h, res, "archive tests"); + assert.match(res.err, /archived failure/); + assert.match(res.err, /refused: HEAD's queue tests failed in the archive/); +}); diff --git a/packages/queue/tests/data.test.mjs b/packages/queue/tests/data.test.mjs new file mode 100644 index 00000000..2e6c019f --- /dev/null +++ b/packages/queue/tests/data.test.mjs @@ -0,0 +1,361 @@ +// The pure layer: schema, serialization, the 8.7 matrix, replay, render, +// view classification and `next` (8.8). No file or git access. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + CALLER_OP_RE, LOG_OP_RE, applyEntry, buildDoc, canonArgs, classifyView, countHeading, genesisReceipt, genesisRows, + gitBlobId, loadDoc, nextFor, parseManifest, parseMigrationMap, render, rowsArray, serialize, sha256, splitView, +} from "../src/queue.mjs"; +import { QueueError } from "../src/errors.mjs"; +import { MAP_ROWS, mapText } from "./helpers.mjs"; + +const ROOT = "/srv/repo"; +const BLOB = "a".repeat(40); +const BLOB2 = "b".repeat(40); +let clock = 0; +const at = () => new Date(Date.UTC(2026, 8, 26, 12, 0, clock++)).toISOString(); + +function brief(path = "docs/plans/brief-b.md", anchor = "Queue", blob = BLOB) { + return { path, anchor, blob }; +} + +// A genesis document built the way store.mjs builds one. +function genesisDoc(rows = MAP_ROWS) { + const map = parseMigrationMap(mapText(rows)); + const blobs = new Map(map.rows.filter((r) => r.brief).map((r) => [r.id, BLOB])); + const t = at(); + const grows = genesisRows(map, blobs, "genesis-op-0001", t, "sage"); + const body = render(grows, 0); + const entry = { + rev: 0, op: "genesis-op-0001", verb: "genesis", args: { root: ROOT, branch: "refactor", map: "agents/sage/work/queue-migration-map.md" }, + by: "sage", at: t, semantics: 1, + result: { mapBlob: BLOB2, highWater: map.highWater, retired: map.retired, rows: grows, legacyView: "legacy\n", receipt: genesisReceipt("genesis-op-0001", grows.length) }, + viewSha: sha256(body), + }; + return { version: 1, canonicalRoot: ROOT, revision: 0, rows: grows, log: [entry] }; +} + +// Appends one op the way store.mjs does and returns the new document. +function step(doc, verb, args, by, resolved = {}, op = `op-${verb}-${doc.revision + 1}-xxxx`) { + const { state } = loadDoc(Buffer.from(serialize(doc))); + const entry = { rev: doc.revision + 1, op, verb, args: canonArgs(verb, args), by, at: at(), semantics: 1, result: null, viewSha: null }; + const out = applyEntry(state, entry, resolved); + entry.result = out.result; + entry.viewSha = sha256(render(rowsArray(out.state), entry.rev)); + return buildDoc(doc.canonicalRoot, out.state, [...doc.log, entry]); +} + +function refused(fn, re) { + assert.throws(fn, (err) => err instanceof QueueError && err.code === 2 && re.test(err.message)); +} + +const mv = (id, to, extra = {}) => ({ id, to, reason: null, candidate: null, evidence: null, ...extra }); +const row = (doc, id) => doc.rows.find((r) => r.id === id); +const MANIFEST = `${"c".repeat(64)} packages/queue/src/queue.mjs\n`; +const CAND = { kind: "manifest", digest: sha256(MANIFEST), text: MANIFEST }; + +// Row 9 in progress (row 6 blocked first, since row 9 waits on 6 settled). +function row9Started() { + let d = genesisDoc(); + d = step(d, "move", mv(6, "blocked", { reason: "paused" }), "darkwing"); + return step(d, "move", mv(9, "in-progress"), "darkwing"); +} + +test("genesis document serializes deterministically and replays", () => { + const doc = genesisDoc(); + const text = serialize(doc); + assert.equal(serialize(JSON.parse(text)), text); + assert.ok(text.endsWith("}\n")); + const loaded = loadDoc(Buffer.from(text)); + assert.equal(loaded.branch, "refactor"); + assert.deepEqual(rowsArray(loaded.state), doc.rows); + assert.deepEqual(row(doc, 6).claim, { seat: "darkwing", op: "genesis-op-0001" }); + assert.equal(row(doc, 9).claim, null); +}); + +test("a hand edit that stays valid JSON fails replay; a formatting-only edit fails re-serialization", () => { + const doc = step(genesisDoc(), "note", { id: 9, text: "hello" }, "darkwing"); + const edited = JSON.parse(serialize(doc)); + edited.rows.find((r) => r.id === 9).note = "changed by hand"; + refused(() => loadDoc(Buffer.from(serialize(edited))), /do not equal the replay/); + const loose = serialize(doc).replace('"revision": 1', '"revision": 1'); + refused(() => loadDoc(Buffer.from(loose)), /re-serialize byte for byte/); + const reordered = serialize(doc).replace(/"version": 1,\n "canonicalRoot": "[^"]*",/, `"canonicalRoot": "${ROOT}",\n "version": 1,`); + refused(() => loadDoc(Buffer.from(reordered)), /keys must be exactly/); +}); + +test("a tampered result, receipt or viewSha fails replay", () => { + const doc = step(genesisDoc(), "note", { id: 9, text: "hello" }, "darkwing"); + const a = JSON.parse(serialize(doc)); + a.log[1].result.receipt = "ok forged"; + refused(() => loadDoc(Buffer.from(serialize(a))), /result differs from its replay/); + const b = JSON.parse(serialize(doc)); + b.log[1].viewSha = "0".repeat(64); + refused(() => loadDoc(Buffer.from(serialize(b))), /viewSha is not the render/); + const c = JSON.parse(serialize(doc)); + c.log[1].by = "rocko"; + refused(() => loadDoc(Buffer.from(serialize(c))), /does not replay/); +}); + +test("op ids: 8 to 72 characters for callers, 80 in the log for .outcome entries", () => { + assert.ok(CALLER_OP_RE.test("a".repeat(72))); + assert.ok(!CALLER_OP_RE.test("a".repeat(73))); + assert.ok(!CALLER_OP_RE.test("short")); + assert.ok(!CALLER_OP_RE.test("-leading-dash")); + assert.ok(LOG_OP_RE.test(`${"a".repeat(72)}.outcome`)); + assert.ok(!LOG_OP_RE.test(`${"a".repeat(73)}.outcome`)); +}); + +test("add: defaults for an ordinary seat, privileged extras, refusals", () => { + const d = genesisDoc(); + const add = { piece: "New", gate: "tests pass", brief: "docs/plans/brief-b.md#Queue", issues: [1508] }; + const d1 = step(d, "add", add, "dewey", { brief: brief() }); + const r = row(d1, 12); + assert.equal(r.owner, "dewey"); + assert.equal(r.state, "queued"); + assert.deepEqual(r.closes, [1508]); + assert.equal(r.gateOwner, "jason"); + assert.deepEqual([r.after, r.reviewers, r.required, r.claim], [[], [], false, null]); + assert.match(d1.log[1].result.receipt, /row 12 none→queued$/); + refused(() => step(d, "add", { ...add, owner: "rocko" }, "dewey", { brief: brief() }), /only a privileged actor may set owner/); + refused(() => step(d, "add", { ...add, reviewers: ["rocko"] }, "dewey", { brief: brief() }), /reviewers/); + refused(() => step(d, "add", { ...add, required: true }, "dewey", { brief: brief() }), /required/); + const d2 = step(d, "add", { ...add, owner: "rocko", gateOwner: "filbert", after: [{ id: 9, when: "done" }], reviewers: ["dewey"], required: true }, "sage", { brief: brief() }); + const r2 = row(d2, 12); + assert.deepEqual([r2.owner, r2.gateOwner, r2.required, r2.after], ["rocko", "filbert", true, [{ id: 9, when: "done" }]]); + refused(() => step(d, "add", { ...add, after: [{ id: 99, when: "done" }] }, "sage", { brief: brief() }), /missing row 99/); + // ids come from the high-water mark, so a retired id is never reused. + const d3 = step(d2, "add", add, "dewey", { brief: brief() }); + assert.ok(row(d3, 13)); +}); + +test("matrix: queued→briefed privileged; briefed→in-progress owner with after satisfied", () => { + let d = step(genesisDoc(), "add", { piece: "N", gate: "g", brief: "docs/plans/brief-b.md#Queue" }, "dewey", { brief: brief() }); + refused(() => step(d, "move", mv(12, "briefed"), "dewey"), /privileged/); + d = step(d, "move", mv(12, "briefed"), "sage"); + refused(() => step(d, "move", mv(12, "in-progress"), "sage"), /only the owner \(dewey\) may start/); + d = step(d, "move", mv(12, "in-progress"), "dewey"); + assert.deepEqual(row(d, 12).claim.seat, "dewey"); + refused(() => step(d, "move", mv(12, "in-progress"), "dewey"), /not a transition/); + // row 9 waits on row 6 settled; row 6 is in progress at genesis. + refused(() => step(genesisDoc(), "move", mv(9, "in-progress"), "darkwing"), /waits on 6 \(settled; now in-progress\)/); + const d9 = row9Started(); + assert.equal(row(d9, 9).state, "in-progress"); +}); + +test("matrix: release, review round, changes requested and waiting-on-jason", () => { + let d = row9Started(); + refused(() => step(d, "move", mv(9, "briefed"), "darkwing"), /use `queue release 9`/); + refused(() => step(d, "release", { id: 9 }, "dewey"), /only the claimant/); + const released = step(d, "release", { id: 9 }, "darkwing"); + assert.deepEqual([row(released, 9).state, row(released, 9).claim], ["briefed", null]); + step(d, "release", { id: 9 }, "sage"); + refused(() => step(d, "move", mv(9, "in-review"), "darkwing"), /needs --candidate/); + refused(() => step(d, "move", mv(9, "in-review", { candidate: "x.sha256" }), "dewey", { candidate: CAND }), /only the claimant/); + d = step(d, "move", mv(9, "in-review", { candidate: "x.sha256" }), "darkwing", { candidate: CAND }); + const rv = row(d, 9).review; + assert.equal(rv.issue, 1508); + assert.deepEqual(rv.rounds.map((r) => [r.n, r.request, r.candidate.digest]), [[1, "none", CAND.digest]]); + assert.match(d.log.at(-1).result.receipt, /in-progress→in-review round 1$/); + refused(() => step(d, "move", mv(9, "in-progress"), "dewey"), /claimed by darkwing/); + d = step(d, "move", mv(9, "in-progress"), "darkwing"); + assert.equal(row(d, 9).claim.seat, "darkwing"); + d = step(d, "move", mv(9, "in-review", { candidate: "y" }), "darkwing", { candidate: { kind: "commit", digest: BLOB2, text: null } }); + assert.equal(row(d, 9).review.rounds.length, 2); + const w = step(d, "move", mv(9, "waiting-on-jason"), "sage"); + assert.equal(row(w, 9).claim.seat, "darkwing"); + refused(() => step(w, "move", mv(9, "done"), "sage"), /sage closes a waiting-on-jason row only with --evidence/); + refused(() => step(w, "move", mv(9, "done"), "darkwing"), /only jason/); + const done = step(w, "move", mv(9, "done", { evidence: "Jason approved in thread X" }), "sage"); + assert.deepEqual([row(done, 9).state, row(done, 9).claim], ["done", null]); + const doneJ = step(w, "move", mv(9, "done"), "jason"); + refused(() => step(doneJ, "move", mv(9, "blocked", { reason: "x" }), "jason"), /done rows never change/); + refused(() => step(doneJ, "note", { id: 9, text: "x" }, "jason"), /notes are closed/); +}); + +test("matrix J5: in-review→done by the gate owner with evidence naming the current round", () => { + let d = row9Started(); + d = step(d, "move", mv(9, "in-review", { candidate: "x" }), "darkwing", { candidate: CAND }); + const ev = `comment=4242,candidate=${CAND.digest}`; + refused(() => step(d, "move", mv(9, "done"), "filbert"), /--evidence comment=,candidate=/); + refused(() => step(d, "move", mv(9, "done", { evidence: `comment=1,candidate=${"d".repeat(64)}` }), "filbert"), /is not round 1's candidate/); + refused(() => step(d, "move", mv(9, "done", { evidence: ev }), "rocko"), /only the gate owner \(filbert\)/); + const done = step(d, "move", mv(9, "done", { evidence: ev }), "filbert"); + assert.equal(row(done, 9).state, "done"); + step(d, "move", mv(9, "done", { evidence: ev }), "sage"); + // Row 11's gate is Jason's: in-review→done is refused for everyone. + let e = step(genesisDoc(), "move", mv(11, "in-progress"), "dewey"); + e = step(e, "move", mv(11, "in-review", { candidate: "x" }), "dewey", { candidate: CAND }); + refused(() => step(e, "move", mv(11, "done", { evidence: ev }), "jason"), /gate is Jason's/); +}); + +test("matrix: blocked keeps the claim and returns only to previousState", () => { + let d = row9Started(); + refused(() => step(d, "move", mv(9, "blocked"), "darkwing"), /needs --reason/); + refused(() => step(d, "move", mv(9, "blocked", { reason: "x" }), "rocko"), /claimed by darkwing/); + d = step(d, "move", mv(9, "blocked", { reason: "waiting on git" }), "darkwing"); + const r = row(d, 9); + assert.deepEqual([r.state, r.previousState, r.blockedReason, r.claim.seat], ["blocked", "in-progress", "waiting on git", "darkwing"]); + refused(() => step(d, "move", mv(9, "blocked", { reason: "y" }), "darkwing"), /already blocked; update the reason with note/); + const noted = step(d, "note", { id: 9, text: "now waiting on review" }, "darkwing"); + assert.equal(row(noted, 9).blockedReason, "now waiting on review"); + refused(() => step(d, "note", { id: 9, text: "" }, "darkwing"), /reason cannot be empty/); + refused(() => step(d, "move", mv(9, "briefed"), "darkwing"), /returns only there/); + const back = step(d, "move", mv(9, "in-progress"), "sage"); + assert.deepEqual([row(back, 9).state, row(back, 9).previousState, row(back, 9).blockedReason, row(back, 9).claim.seat], ["in-progress", null, null, "darkwing"]); + refused(() => step(genesisDoc(), "move", mv(1, "blocked", { reason: "x" }), "sage"), /done rows never change/); + refused(() => step(genesisDoc(), "move", mv(8, "blocked", { reason: "x" }), "jason"), /not a transition/); +}); + +test("matrix J4: parking is Jason's, refused while required; unpark returns to queued", () => { + const d = genesisDoc(); + refused(() => step(d, "move", mv(11, "parked"), "sage"), /only jason may park/); + refused(() => step(d, "move", mv(9, "parked"), "jason"), /required and cannot be parked/); + const p = step(d, "move", mv(11, "parked"), "jason"); + refused(() => step(p, "move", mv(11, "briefed"), "jason"), /not a transition/); + refused(() => step(p, "move", mv(11, "queued"), "sage"), /only jason may unpark/); + const u = step(p, "move", mv(11, "queued"), "jason"); + assert.equal(row(u, 11).state, "queued"); + refused(() => step(p, "set", { id: 11, field: "required", value: true }, "sage"), /parked; unpark it/); + refused(() => step(p, "note", { id: 11, text: "x" }, "jason"), /notes are closed/); + refused(() => step(row9Started(), "move", mv(9, "parked"), "jason"), /not a transition|required/); +}); + +test("field edits: who may change what", () => { + const d = genesisDoc(); + const set = (id, field, value, reason = null) => ({ id, field, value, reason }); + refused(() => step(d, "set", set(11, "piece", "x"), "dewey"), /privileged/); + assert.match(step(d, "set", set(11, "piece", "Renamed"), "sage").log.at(-1).result.receipt, /row 11 piece: Brief template→Renamed$/); + refused(() => step(d, "set", set(9, "after", []), "sage"), /only jason may change after on a required row/); + step(d, "set", set(9, "after", []), "jason"); + step(d, "set", set(11, "after", [{ id: 9, when: "done" }]), "sage"); + refused(() => step(d, "set", set(11, "after", [{ id: 11, when: "done" }]), "sage"), /lists itself/); + refused(() => step(d, "set", set(9, "required", false), "sage"), /only jason may clear required/); + const req = step(d, "set", set(11, "required", true), "sage"); + assert.equal(row(req, 11).requiredSince, req.log.at(-1).at); + const cleared = step(req, "set", set(11, "required", false), "jason"); + assert.equal(row(cleared, 11).requiredSince, null); + refused(() => step(d, "set", set(9, "closes", []), "sage"), /needs --reason/); + const narrowed = step(d, "set", set(9, "closes", [], "closed elsewhere"), "sage"); + assert.deepEqual(row(narrowed, 9).closes, []); + refused(() => step(d, "set", set(9, "closes", [1]), "sage", {}), /--reason|not one of/); + refused(() => step(d, "set", set(9, "closes", [1], "r"), "sage"), /not one of row 9's issues/); + const issues = step(narrowed, "set", set(9, "issues", [1508, 1510]), "sage"); + assert.deepEqual(row(issues, 9).closes, [1508, 1510]); + refused(() => step(d, "set", set(9, "piece", "Queue as data"), "sage"), /already/); + refused(() => step(d, "set", set(1, "piece", "x"), "jason"), /done rows never change/); + const repin = step(d, "set", set(9, "brief", "docs/plans/brief-b.md#Queue"), "sage", { brief: brief(undefined, undefined, BLOB2) }); + assert.equal(row(repin, 9).brief.blob, BLOB2); + refused(() => step(d, "set", set(9, "brief", "docs/plans/brief-b.md#Queue"), "sage", { brief: brief() }), /already pinned/); + refused(() => step(d, "set", set(9, "brief", "docs/plans/brief-b.md#Queue"), "darkwing", { brief: brief() }), /privileged/); +}); + +test("note: owner, listed reviewer or privileged; empty clears", () => { + const d = genesisDoc(); + const n = step(d, "note", { id: 9, text: "from the reviewer" }, "filbert"); + assert.equal(row(n, 9).note, "from the reviewer"); + refused(() => step(d, "note", { id: 9, text: "x" }, "rocko"), /only the owner/); + assert.equal(row(step(n, "note", { id: 9, text: "" }, "darkwing"), 9).note, null); + refused(() => step(d, "note", { id: 9, text: "two\nlines" }, "darkwing"), /one line/); + refused(() => step(d, "note", { id: 9, text: `sep${String.fromCharCode(0x2028)}x` }, "darkwing"), /one line/); +}); + +test("assign moves the claim with the owner; done clears it", () => { + let d = row9Started(); + refused(() => step(d, "assign", { id: 9, seat: "dewey" }, "darkwing"), /privileged/); + d = step(d, "assign", { id: 9, seat: "dewey" }, "sage", {}, "assign-row-9-dewey"); + assert.deepEqual(row(d, 9).claim, { seat: "dewey", op: "assign-row-9-dewey" }); + assert.match(d.log.at(-1).result.receipt, /owner: darkwing→dewey$/); + refused(() => step(d, "move", mv(9, "in-review", { candidate: "x" }), "darkwing", { candidate: CAND }), /only the claimant \(dewey\)/); + const unclaimed = step(genesisDoc(), "assign", { id: 11, seat: "rocko" }, "sage"); + assert.equal(row(unclaimed, 11).claim, null); +}); + +test("render is byte-stable and escapes pipes", () => { + const d = step(genesisDoc(), "set", { id: 11, field: "piece", value: "a | b", reason: null }, "sage"); + const a = render(d.rows, d.revision); + assert.equal(render(JSON.parse(JSON.stringify(d.rows)).reverse(), d.revision), a); + assert.match(a, /\| 11 \| a \\\| b \|/); + assert.match(a, /revision 1 by/); + assert.notEqual(sha256(render(d.rows, 2)), sha256(a)); +}); + +test("view classification: current, genuine stale, edited stale marker, changed current body, markers", () => { + const d0 = genesisDoc(); + const d1 = step(d0, "note", { id: 9, text: "one" }, "darkwing"); + const wrap = (body) => `# Q\n\n\n${body}\ntail\n`; + const cur = wrap(render(d1.rows, 1)); + assert.equal(classifyView(cur, d1.log).state, "current"); + const stale = classifyView(wrap(render(d0.rows, 0)), d1.log); + assert.equal(stale.state, "stale"); + assert.deepEqual(stale.unshown.map((e) => e.rev), [1]); + assert.equal(classifyView(wrap(render(d0.rows, 0).replace("Row six", "Row 6")), d1.log).state, "unknown"); + assert.equal(classifyView(cur.replace("| one |", "| two |"), d1.log).state, "unknown"); + assert.equal(classifyView(cur.replace("\n", ""), d1.log).state, "unknown"); + assert.equal(classifyView(`${cur}\n`, d1.log).state, "unknown"); + assert.equal(splitView("\n\n"), null); + assert.equal(classifyView(null, d1.log).state, "unknown"); +}); + +test("next: resume, then review, then start, then wait, then nothing; lowest id first", () => { + let d = genesisDoc(); + const add = (owner, extra = {}) => ({ piece: `p-${owner}`, gate: "g", brief: "docs/plans/brief-b.md#Queue", owner, ...extra }); + d = step(d, "add", add("rocko"), "sage", { brief: brief() }); // 12 + d = step(d, "add", add("rocko", { reviewers: ["darkwing"] }), "sage", { brief: brief() }); // 13 + d = step(d, "add", add("darkwing"), "sage", { brief: brief() }); // 14 + for (const id of [12, 13, 14]) d = step(d, "move", mv(id, "briefed"), "sage"); + const rows = () => loadDoc(Buffer.from(serialize(d))).state.rows; + const all = () => true; + // darkwing resumes row 6 (claimed at genesis). + assert.deepEqual([nextFor(rows(), "darkwing", all).action, nextFor(rows(), "darkwing", all).row.id], ["resume", 6]); + d = step(d, "move", mv(13, "in-progress"), "rocko"); + d = step(d, "move", mv(13, "in-review", { candidate: "x" }), "rocko", { candidate: CAND }); + assert.equal(nextFor(rows(), "darkwing", all).row.id, 6); + d = step(d, "move", mv(6, "blocked", { reason: "r" }), "darkwing"); + const r = nextFor(rows(), "darkwing", all); + assert.deepEqual([r.action, r.row.id], ["review", 13]); + // rocko: row 12 briefed, row 13 in review as author: start before wait. + assert.deepEqual([nextFor(rows(), "rocko", all).action, nextFor(rows(), "rocko", all).row.id], ["start", 12]); + const flagged = nextFor(rows(), "rocko", () => false); + assert.equal(flagged.briefDiffers, true); + d = step(d, "move", mv(12, "parked"), "jason"); + assert.deepEqual([nextFor(rows(), "rocko", all).action, nextFor(rows(), "rocko", all).row.id], ["wait", 13]); + assert.equal(nextFor(rows(), "researcher", all).action, "nothing"); + // row 9 (darkwing) waits on 6 settled, which blocked satisfies: start comes after review. + d = step(d, "move", mv(13, "in-progress"), "rocko"); + assert.deepEqual([nextFor(rows(), "darkwing", all).action, nextFor(rows(), "darkwing", all).row.id], ["start", 9]); +}); + +test("canonical args make a retry's identity independent of list order", () => { + const a = canonArgs("add", { piece: "p", gate: "g", brief: "a.md#X", issues: [3, 1], reviewers: ["rocko", "dewey"] }); + assert.deepEqual([a.issues, a.reviewers, a.after, a.required], [[1, 3], ["dewey", "rocko"], null, null]); + refused(() => canonArgs("add", { piece: "p", gate: "g", brief: "a.md#X", issues: [1, 1] }), /repeated/); + refused(() => canonArgs("add", { piece: "p", gate: "g", brief: "a.md" }), /PATH#ANCHOR/); + refused(() => canonArgs("add", { piece: "p", gate: "g", brief: "../x.md#A" }), /repo-relative/); + refused(() => canonArgs("move", { id: 1, to: "finished" }), /unknown state/); + refused(() => canonArgs("set", { id: 1, field: "owner", value: "x" }), /set cannot change/); +}); + +test("manifests, headings and blob ids", () => { + assert.equal(parseManifest(MANIFEST), 1); + refused(() => parseManifest(`${"c".repeat(64)} one-space\n`), /not " "/); + refused(() => parseManifest(`${"c".repeat(64)} a\n${"d".repeat(64)} a\n`), /twice/); + refused(() => parseManifest("no newline"), /end with a newline/); + const text = "# T\n\n## A\n\n```\n## A\n```\n\n### A \n"; + assert.equal(countHeading(text, "A"), 2); + assert.equal(countHeading("~~~\n## A\n~~~\n## A\n", "A"), 1); + // git hash-object of "hello\n" + assert.equal(gitBlobId(Buffer.from("hello\n")), "ce013625030ba8dba906f756967f9e9ca394464a"); +}); + +test("the migration map: one queue-map block, exact keys", () => { + assert.equal(parseMigrationMap(mapText()).rows.length, MAP_ROWS.length); + refused(() => parseMigrationMap(mapText() + mapText()), /exactly one/); + refused(() => parseMigrationMap("no block\n"), /exactly one/); + const bad = MAP_ROWS.map((r) => ({ ...r })); + delete bad[0].note; + refused(() => parseMigrationMap(mapText(bad)), /keys must be exactly/); + const noBrief = MAP_ROWS.map((r) => (r.id === 9 ? { ...r, brief: null } : r)); + const map = parseMigrationMap(mapText(noBrief)); + refused(() => loadDoc(Buffer.from(serialize(genesisDoc(map.rows)))), /only a done row may lack one/); +}); diff --git a/packages/queue/tests/fixtures/kill-at.mjs b/packages/queue/tests/fixtures/kill-at.mjs new file mode 100644 index 00000000..fd480dcc --- /dev/null +++ b/packages/queue/tests/fixtures/kill-at.mjs @@ -0,0 +1,10 @@ +// Child process for the SIGKILL tests. argv: SRC_DIR STEP REQUEST_JSON. +// Runs one mutation from the working directory and kills itself with +// SIGKILL when the write path reaches STEP (8.5). +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const [src, step, json] = process.argv.slice(2); +const store = await import(pathToFileURL(join(src, "store.mjs")).href); +store.mutate({ hook: (name) => { if (name === step) process.kill(process.pid, "SIGKILL"); } }, JSON.parse(json)); +process.stdout.write("finished without reaching the step\n"); diff --git a/packages/queue/tests/fixtures/lock-child.mjs b/packages/queue/tests/fixtures/lock-child.mjs new file mode 100644 index 00000000..3338d350 --- /dev/null +++ b/packages/queue/tests/fixtures/lock-child.mjs @@ -0,0 +1,14 @@ +// Child process for the lock tests. argv: GITDIR MODE +// kill-before-link SIGKILL itself after the temp record is written +// hold take the lock, print "held", then wait to be killed +import { realIo, sleepMs } from "../../src/io.mjs"; +import { acquire } from "../../src/lock.mjs"; + +const [gitDir, mode] = process.argv.slice(2); +if (mode === "kill-before-link") { + acquire({ gitDir, io: realIo, verb: "move", hook: (n) => { if (n === "lock-temp-written") process.kill(process.pid, "SIGKILL"); } }); +} else if (mode === "hold") { + acquire({ gitDir, io: realIo, op: "holder-op-1", verb: "move" }); + process.stdout.write("held\n"); + sleepMs(60000); +} diff --git a/packages/queue/tests/helpers.mjs b/packages/queue/tests/helpers.mjs new file mode 100644 index 00000000..ecc61709 --- /dev/null +++ b/packages/queue/tests/helpers.mjs @@ -0,0 +1,147 @@ +// Scratch repositories for the queue tests. Every test works under +// os.tmpdir(): a fresh repository holding a copy of packages/queue/src and +// the two discord helpers it imports, so the code-under-root check (8.3) +// passes there and nothing touches this checkout's .git. +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +export const PKG_SRC = join(HERE, "..", "src"); +export const DISCORD_SRC = join(HERE, "..", "..", "discord", "src"); +export const SCRIPTS = join(HERE, "..", "..", "..", "scripts"); + +export const BEGIN = ""; +export const END = ""; + +// Git reads no user or system config here: HOME is a scratch directory. +export function gitEnv(home, extra = {}) { + const env = { ...process.env, HOME: home, XDG_CONFIG_HOME: join(home, ".config"), GIT_CONFIG_NOSYSTEM: "1", + GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@example.invalid", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@example.invalid", ...extra }; + for (const k of ["GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_INDEX_FILE", "MOSAIC_AGENT_NAME"]) if (!(k in extra)) delete env[k]; + return env; +} + +export function sh(cmd, args, { cwd, env, input, allowFail = false } = {}) { + const r = spawnSync(cmd, args, { cwd, env, input, encoding: "utf8", maxBuffer: 64 << 20 }); + if (r.error) throw r.error; + if (r.status !== 0 && !allowFail) throw new Error(`${cmd} ${args.join(" ")} exited ${r.status}: ${r.stderr}`); + return r; +} + +export const MAP_ROWS = [ + { + id: 1, piece: "Finished thing", owner: "darkwing", issues: [1503], closes: [1503], state: "done", previousState: null, + required: false, requiredSince: null, gate: "B passed", gateOwner: "jason", brief: null, after: [], reviewers: [], + note: null, blockedReason: null, createdAt: "unknown", + }, + { + id: 6, piece: "Row six", owner: "darkwing", issues: [1511], closes: [1511], state: "in-progress", previousState: null, + required: false, requiredSince: null, gate: "pilot accepted", gateOwner: "jason", brief: { path: "docs/plans/brief-a.md", anchor: "Row six" }, + after: [], reviewers: ["filbert"], note: "legacy note", blockedReason: null, createdAt: "2026-09-13", + }, + { + id: 8, piece: "Parked thing", owner: "unassigned", issues: [], closes: [], state: "parked", previousState: null, + required: false, requiredSince: null, gate: "Jason says", gateOwner: "jason", brief: { path: "docs/plans/brief-a.md", anchor: "Parked" }, + after: [], reviewers: [], note: null, blockedReason: null, createdAt: "unknown", + }, + { + id: 9, piece: "Queue as data", owner: "darkwing", issues: [1508], closes: [1508], state: "briefed", previousState: null, + required: true, requiredSince: "unknown", gate: "filbert approves", gateOwner: "filbert", brief: { path: "docs/plans/brief-b.md", anchor: "Queue" }, + after: [{ id: 6, when: "settled" }], reviewers: ["filbert"], note: null, blockedReason: null, createdAt: "2026-09-13", + }, + { + id: 11, piece: "Brief template", owner: "dewey", issues: [1508], closes: [1508], state: "briefed", previousState: null, + required: false, requiredSince: null, gate: "two briefs accepted", gateOwner: "jason", brief: { path: "docs/plans/brief-b.md", anchor: "Template" }, + after: [], reviewers: [], note: null, blockedReason: null, createdAt: "unknown", + }, +]; + +export function mapText(rows = MAP_ROWS, { retired = [7], highWater = 11 } = {}) { + return `# Migration map\n\nReviewed.\n\n\`\`\`json queue-map\n${JSON.stringify({ rows, retired, highWater }, null, 2)}\n\`\`\`\n`; +} + +export const LEGACY = "| # | Piece |\n|---|---|\n| 1 | old row |\n"; + +export function queueMd(body = LEGACY) { + return `# QUEUE\n\nHeader prose.\n\n${BEGIN}\n${body}${END}\n\nParked entries below.\n`; +} + +// A committed scratch checkout on branch `refactor` with the queue code, +// seats, briefs, a map and QUEUE.md. Returns paths and runners. With +// `scripts`, it also commits queue-commit.sh, the guard and a one-test +// packages/queue/tests, so the commit procedure has an archive to run. +export function scratchRepo(t, { map = mapText(), commitMap = true, name = "repo", scripts = false } = {}) { + const base = realpathSync(mkdtempSync(join(tmpdir(), "mosaic-queue-test-"))); + if (t) t.after(() => rmSync(base, { recursive: true, force: true })); + const home = join(base, "home"); + mkdirSync(join(home, ".config"), { recursive: true }); + const root = join(base, name); + mkdirSync(root); + const env = gitEnv(home); + const g = (...args) => sh("git", ["-C", root, ...args], { env }).stdout; + g("init", "-q", "-b", "refactor"); + cpSync(PKG_SRC, join(root, "packages/queue/src"), { recursive: true }); + mkdirSync(join(root, "packages/discord/src"), { recursive: true }); + for (const f of ["journal.mjs", "errors.mjs"]) cpSync(join(DISCORD_SRC, f), join(root, "packages/discord/src", f)); + for (const s of ["darkwing", "dewey", "filbert", "rocko", "sage"]) { + mkdirSync(join(root, "agents", s, "work"), { recursive: true }); + writeFileSync(join(root, "agents", s, ".keep"), ""); + } + mkdirSync(join(root, "docs/plans"), { recursive: true }); + writeFileSync(join(root, "docs/plans/brief-a.md"), "# Briefs A\n\n## Row six\n\nText.\n\n## Parked\n\nStub.\n"); + writeFileSync(join(root, "docs/plans/brief-b.md"), "# Briefs B\n\n## Queue\n\nText.\n\n## Template\n\nText.\n\n```\n## Queue\n```\n"); + writeFileSync(join(root, "docs/plans/QUEUE.md"), queueMd()); + if (map !== null) writeFileSync(join(root, "agents/sage/work/queue-migration-map.md"), map); + if (scripts) { + mkdirSync(join(root, "scripts/git-hooks"), { recursive: true }); + cpSync(join(SCRIPTS, "queue-commit.sh"), join(root, "scripts/queue-commit.sh")); + cpSync(join(SCRIPTS, "git-hooks/pre-commit"), join(root, "scripts/git-hooks/pre-commit")); + mkdirSync(join(root, "packages/queue/tests"), { recursive: true }); + writeFileSync(join(root, "packages/queue/tests/stub.test.mjs"), 'import { test } from "node:test";\ntest("archive stub", () => {});\n'); + } + if (!commitMap && map !== null) { + g("add", "-A", "--", ".", ":!agents/sage/work/queue-migration-map.md"); + } else { + g("add", "-A"); + } + g("commit", "-q", "-m", "scratch"); + return { base, home, root, env, g, gitDir: join(root, ".git"), queuePath: join(root, "docs/plans/queue.json"), viewPath: join(root, "docs/plans/QUEUE.md") }; +} + +// The copied code, imported from inside the scratch repository. +export async function load(repo) { + const src = join(repo.root, "packages/queue/src"); + const store = await import(pathToFileURL(join(src, "store.mjs")).href); + const cli = await import(pathToFileURL(join(src, "cli.mjs")).href); + const queue = await import(pathToFileURL(join(src, "queue.mjs")).href); + const io = await import(pathToFileURL(join(src, "io.mjs")).href); + const lock = await import(pathToFileURL(join(src, "lock.mjs")).href); + const errors = await import(pathToFileURL(join(src, "errors.mjs")).href); + return { store, cli, queue, io, lock, errors }; +} + +// Runs the copied CLI as a child process in `cwd`. +export function cli(repo, args, { cwd = repo.root, by = null, env = {} } = {}) { + const e = { ...repo.env, ...env }; + if (by !== null) e.MOSAIC_AGENT_NAME = by; + const r = spawnSync(process.execPath, [join(repo.root, "packages/queue/src/cli.mjs"), ...args], { cwd, env: e, encoding: "utf8" }); + if (r.error) throw r.error; + return { code: r.status, signal: r.signal, out: r.stdout, err: r.stderr }; +} + +// Genesis in the working tree, then the genesis commit by plain git (the +// scratch repo has no guard installed), so later ops pass the F3 check. +export function genesisCommitted(repo, { by = "sage", op = "genesis-2026-09-26" } = {}) { + const r = cli(repo, ["genesis", "--op", op, "--root", repo.root, "--branch", "refactor", "--map", "agents/sage/work/queue-migration-map.md"], { by }); + if (r.code !== 0) throw new Error(`genesis failed: ${r.err}`); + repo.g("add", "docs/plans/queue.json", "docs/plans/QUEUE.md"); + repo.g("commit", "-q", "-m", "queue genesis"); + return r; +} + +export function opts(repo, extra = {}) { + return { cwd: repo.root, env: repo.env, ...extra }; +} diff --git a/packages/queue/tests/lock.test.mjs b/packages/queue/tests/lock.test.mjs new file mode 100644 index 00000000..9b6b2174 --- /dev/null +++ b/packages/queue/tests/lock.test.mjs @@ -0,0 +1,249 @@ +// The lock and the unlock gate (8.4): each schedule in the 8.4 test list. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { bootId, processStart } from "../../discord/src/journal.mjs"; +import { QueueError } from "../src/errors.mjs"; +import { realIo } from "../src/io.mjs"; +import { GATE_NAME, LOCK_NAME, acquire, checkGate, classify, realProc, release, unlock } from "../src/lock.mjs"; + +const HERE = new URL(".", import.meta.url).pathname; + +function dir(t) { + const d = realpathSync(mkdtempSync(join(tmpdir(), "mosaic-queue-lock-"))); + t.after(() => rmSync(d, { recursive: true, force: true })); + return d; +} + +const OTHER_BOOT = "00000000-0000-4000-8000-000000000000"; + +function record(over = {}) { + return Buffer.from(JSON.stringify({ + pid: process.pid, start: processStart(process.pid), boot: bootId(), host: realProc.host(), op: "some-op-0001", verb: "move", + at: "2026-09-26T00:00:00.000Z", ...over, + }) + "\n"); +} + +// A pid that is not running: a child that has already exited. +async function deadPid() { + const c = spawn(process.execPath, ["-e", ""]); + await new Promise((r) => c.on("exit", r)); + return c.pid; +} + +// A live process that the tests never signal except to clean up. +async function sleeper(t) { + const c = spawn("sleep", ["60"], { stdio: "ignore" }); + await new Promise((r) => c.on("spawn", r)); + t.after(() => { try { c.kill("SIGKILL"); } catch { /* gone */ } }); + return c; +} + +function refused(fn, re, code = 2) { + assert.throws(fn, (err) => err instanceof QueueError && err.code === code && re.test(err.message)); +} + +const tmps = (d) => readdirSync(d).filter((n) => n.endsWith(".tmp")); + +test("acquire publishes the record by link; release removes only its own lock", (t) => { + const d = dir(t); + const h = acquire({ gitDir: d, io: realIo, op: "abcdefgh-1", verb: "move" }); + const rec = JSON.parse(readFileSync(join(d, LOCK_NAME), "utf8")); + assert.deepEqual(Object.keys(rec), ["pid", "start", "boot", "host", "op", "verb", "at"]); + assert.equal(classify(readFileSync(join(d, LOCK_NAME)), realProc).state, "live"); + assert.deepEqual(tmps(d), []); + assert.equal(release(h, realIo), null); + assert.equal(existsSync(join(d, LOCK_NAME)), false); +}); + +test("a kill between the temp write and the link leaves no lock", async (t) => { + const d = dir(t); + const child = spawn(process.execPath, [join(HERE, "fixtures", "lock-child.mjs"), d, "kill-before-link"], { stdio: "ignore" }); + const [, signal] = await new Promise((r) => child.on("exit", (...a) => r(a))); + assert.equal(signal, "SIGKILL"); + assert.equal(existsSync(join(d, LOCK_NAME)), false); + assert.equal(tmps(d).length, 1, "the killed writer's temp file stays; it is not the lock"); + release(acquire({ gitDir: d, io: realIo, verb: "move" }), realIo); +}); + +test("a short or failed temp write refuses and leaves no lock and no temp", (t) => { + const d = dir(t); + const short = { ...realIo, write: () => 0 }; + refused(() => acquire({ gitDir: d, io: short, verb: "move" }), /cannot write the lock record .*ESHORT/, 1); + const enospc = { ...realIo, write: () => { throw Object.assign(new Error("full"), { code: "ENOSPC" }); } }; + refused(() => acquire({ gitDir: d, io: enospc, verb: "move" }), /ENOSPC; no lock taken/, 1); + const badFsync = { ...realIo, fsync: () => { throw Object.assign(new Error("io"), { code: "EIO" }); } }; + refused(() => acquire({ gitDir: d, io: badFsync, verb: "move" }), /EIO/, 1); + const badBack = { ...realIo, readFile: (p) => (p.endsWith(".tmp") ? Buffer.from("x") : realIo.readFile(p)) }; + refused(() => acquire({ gitDir: d, io: badBack, verb: "move" }), /EREADBACK/, 1); + assert.deepEqual(readdirSync(d), []); +}); + +test("a link error other than EEXIST refuses", (t) => { + const d = dir(t); + const io = { ...realIo, link: () => { throw Object.assign(new Error("no"), { code: "EPERM" }); } }; + refused(() => acquire({ gitDir: d, io, verb: "move" }), /cannot link .*EPERM; no lock taken/, 1); + assert.deepEqual(readdirSync(d), []); +}); + +test("a paused holder: another writer waits 10 s, then refuses naming it live", async (t) => { + const d = dir(t); + const child = spawn(process.execPath, [join(HERE, "fixtures", "lock-child.mjs"), d, "hold"], { stdio: ["ignore", "pipe", "ignore"] }); + t.after(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } }); + await new Promise((r) => child.stdout.once("data", r)); + const t0 = Date.now(); + refused(() => acquire({ gitDir: d, io: realIo, op: "waiter-op-1", verb: "move" }), /queue lock held by move holder-op-1 since .*; retry the same op later/); + assert.ok(Date.now() - t0 >= 10000, `waited ${Date.now() - t0} ms`); + child.kill("SIGKILL"); + await new Promise((r) => child.on("exit", r)); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), /owner is dead: pid .*; run `scripts\/mosaic queue unlock` once nothing is running/); + assert.match(unlock({ gitDir: d, io: realIo }), /^removed queue lock \(dead: pid/); + release(acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), realIo); +}); + +test("two concurrent unlockers: the second refuses on the gate", async (t) => { + const d = dir(t); + writeFileSync(join(d, LOCK_NAME), record({ pid: await deadPid() })); + let inner; + const outer = unlock({ + gitDir: d, io: realIo, + hook: (name) => { + if (name !== "gate-held") return; + try { unlock({ gitDir: d, io: realIo }); } catch (err) { inner = err; } + }, + }); + assert.match(inner.message, /unlock gate .* is held \(live: pid \d+, unlock since/); + assert.match(outer, /^removed queue lock \(dead/); + assert.equal(existsSync(join(d, GATE_NAME)), false); +}); + +test("a writer publishing during an unlock, lock first: unlock sees it live and refuses", (t) => { + const d = dir(t); + let unlockErr; + const h = acquire({ + gitDir: d, io: realIo, verb: "move", + hook: (name) => { + if (name !== "lock-linked") return; + try { unlock({ gitDir: d, io: realIo }); } catch (err) { unlockErr = err; } + }, + }); + assert.match(unlockErr.message, /queue lock owner is live: .*; unlock refuses/); + assert.equal(existsSync(join(d, GATE_NAME)), false); + assert.equal(release(h, realIo), null); +}); + +test("a writer publishing during an unlock, gate first: the writer releases and refuses", (t) => { + const d = dir(t); + let writerErr; + const out = unlock({ + gitDir: d, io: realIo, + hook: (name) => { + if (name !== "gate-held") return; + try { acquire({ gitDir: d, io: realIo, verb: "move" }); } catch (err) { writerErr = err; } + }, + }); + assert.match(writerErr.message, /unlock gate .* is present \(live: .*unlock/); + assert.equal(out, "no queue lock present; nothing removed"); + assert.deepEqual(readdirSync(d), []); +}); + +test("a reused pid within one boot is mismatch; unlock removes the lock and never signals the process", async (t) => { + const d = dir(t); + const s = await sleeper(t); + const start = processStart(s.pid); + const wrong = String(BigInt(start) + 1n); + writeFileSync(join(d, LOCK_NAME), record({ pid: s.pid, start: wrong })); + assert.equal(classify(readFileSync(join(d, LOCK_NAME)), realProc).state, "mismatch"); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), /mismatch: .*was reused/); + assert.match(unlock({ gitDir: d, io: realIo }), /removed queue lock \(mismatch/); + assert.equal(s.exitCode, null); + assert.equal(processStart(s.pid), start, "the process still runs, unsignalled"); +}); + +test("the same pid and start on a different boot is mismatch", (t) => { + const d = dir(t); + writeFileSync(join(d, LOCK_NAME), record({ boot: OTHER_BOOT })); + const c = classify(readFileSync(join(d, LOCK_NAME)), realProc); + assert.deepEqual([c.state, c.why], ["mismatch", "recorded in a previous boot"]); +}); + +test("a foreign host is unknown whatever the local pid says; unlock refuses", async (t) => { + const d = dir(t); + writeFileSync(join(d, LOCK_NAME), record({ pid: await deadPid(), host: "some-other-host" })); + assert.equal(classify(readFileSync(join(d, LOCK_NAME)), realProc).state, "unknown"); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), /unknown: .*recorded on host some-other-host; unlock refuses this too/); + refused(() => unlock({ gitDir: d, io: realIo }), /owner is unknown: .*; unlock refuses/); + assert.equal(existsSync(join(d, LOCK_NAME)), true); +}); + +test("unreadable /proc: classification is unknown and acquire refuses", (t) => { + const d = dir(t); + const blind = { ...realProc, processStart: () => null }; + assert.equal(classify(record(), blind).state, "unknown"); + const alive = { ...realProc, processStart: (pid) => (pid === process.pid ? realProc.processStart(pid) : null) }; + assert.match(classify(record({ pid: 1 }), alive).why, /start time is unreadable/); + refused(() => acquire({ gitDir: d, io: realIo, proc: blind, verb: "move" }), /cannot read this process's start time or boot id/); + refused(() => acquire({ gitDir: d, io: realIo, proc: { ...realProc, bootId: () => null }, verb: "move" }), /cannot read/); +}); + +test("invalid records: empty, unparsable, wrong keys, bad start or boot", () => { + for (const b of [null, "", "{", "[]", JSON.stringify({ pid: 1 }), record({ start: "x" }).toString(), record({ boot: "nope" }).toString()]) { + assert.equal(classify(b === null ? null : Buffer.from(b), realProc).state, "invalid"); + } +}); + +test("a stale gate blocks writers; --check-gate says mismatch for a reused pid", async (t) => { + const d = dir(t); + const s = await sleeper(t); + writeFileSync(join(d, GATE_NAME), record({ pid: s.pid, start: String(BigInt(processStart(s.pid)) + 1n), verb: "unlock", op: null })); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move" }), /unlock gate .* is present \(mismatch: .*--check-gate/); + assert.equal(existsSync(join(d, LOCK_NAME)), false, "the writer released its lock"); + const g = checkGate({ gitDir: d, io: realIo }); + assert.equal(g.state, "mismatch"); + assert.match(g.line, /remove .* by hand only once no queue command is running/); + refused(() => unlock({ gitDir: d, io: realIo }), /unlock gate .* is held \(mismatch/); + writeFileSync(join(d, GATE_NAME), record({ host: "elsewhere", verb: "unlock", op: null })); + assert.match(checkGate({ gitDir: d, io: realIo }).line, /unknown: .*leave it for diagnosis/); + rmSync(join(d, GATE_NAME)); + assert.equal(checkGate({ gitDir: d, io: realIo }).state, "absent"); +}); + +test("a delayed release by a dead owner, after unlock and a new owner: the inode check keeps the new lock", (t) => { + const d = dir(t); + const a = acquire({ gitDir: d, io: realIo, op: "owner-a-op", verb: "move" }); + // Unlock judges A dead (as it would after a crash); B then takes the lock. + const judge = { ...realProc, pidAlive: () => false }; + assert.match(unlock({ gitDir: d, io: realIo, proc: judge }), /removed queue lock \(dead/); + const b = acquire({ gitDir: d, io: realIo, op: "owner-b-op", verb: "move" }); + assert.match(release(a, realIo), /is not the one this process took; left in place/); + assert.match(readFileSync(join(d, LOCK_NAME), "utf8"), /owner-b-op/); + assert.equal(release(b, realIo), null); + assert.match(release(b, realIo), /already gone/); +}); + +test("release checks the inode too: a byte-identical lock file with a new inode is left in place", (t) => { + const d = dir(t); + const a = acquire({ gitDir: d, io: realIo, op: "owner-a-op", verb: "move" }); + const bytes = readFileSync(join(d, LOCK_NAME)); + // The copy exists alongside the original before the rename, so its inode differs. + writeFileSync(join(d, "copy"), bytes); + renameSync(join(d, "copy"), join(d, LOCK_NAME)); + assert.match(release(a, realIo), /is not the one this process took; left in place/); + assert.deepEqual(readFileSync(join(d, LOCK_NAME)), bytes); +}); + +test("unlock refuses a live, unknown or invalid lock, and does nothing without one", async (t) => { + const d = dir(t); + assert.equal(unlock({ gitDir: d, io: realIo }), "no queue lock present; nothing removed"); + writeFileSync(join(d, LOCK_NAME), ""); + refused(() => unlock({ gitDir: d, io: realIo }), /owner is invalid/); + writeFileSync(join(d, LOCK_NAME), record()); + refused(() => unlock({ gitDir: d, io: realIo }), /owner is live/); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), /held by move some-op-0001/); + writeFileSync(join(d, LOCK_NAME), "not json"); + refused(() => acquire({ gitDir: d, io: realIo, verb: "move", waitMs: 0 }), /record is invalid; inspect .* by hand/); + assert.equal(existsSync(join(d, GATE_NAME)), false); +}); diff --git a/packages/queue/tests/store.test.mjs b/packages/queue/tests/store.test.mjs new file mode 100644 index 00000000..7033b076 --- /dev/null +++ b/packages/queue/tests/store.test.mjs @@ -0,0 +1,333 @@ +// The CLI against scratch repositories: genesis, the canonical checks (8.3), +// op ids (8.6), claims, the view (8.5 outcomes), brief checks (8.13), +// verify, render and snapshot. +import assert from "node:assert/strict"; +import { mkdirSync, readFileSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { LEGACY, MAP_ROWS, PKG_SRC, cli, genesisCommitted, mapText, queueMd, scratchRepo, sh } from "./helpers.mjs"; + +const MAP = "agents/sage/work/queue-migration-map.md"; + +function ok(r, re) { + assert.equal(r.code, 0, `exit ${r.code}: ${r.err}`); + if (re) assert.match(r.out, re); + return r; +} + +function no(r, code, re) { + assert.equal(r.code, code, `expected exit ${code}, got ${r.code}: ${r.out}${r.err}`); + if (re) assert.match(r.err, re); + return r; +} + +const genesisArgs = (repo, over = {}) => [ + "genesis", "--op", over.op ?? "genesis-2026-09-26", "--root", over.root ?? repo.root, "--branch", over.branch ?? "refactor", "--map", over.map ?? MAP, +]; + +function ready(t, o = {}) { + const repo = scratchRepo(t, o); + genesisCommitted(repo); + return repo; +} + +const doc = (repo) => JSON.parse(readFileSync(repo.queuePath, "utf8")); +const row = (repo, id) => doc(repo).rows.find((r) => r.id === id); + +test("genesis: refusals before anything is written", (t) => { + const repo = scratchRepo(t); + no(cli(repo, ["list"]), 2, /no docs\/plans\/queue.json; genesis has not run/); + no(cli(repo, genesisArgs(repo), { by: "darkwing" }), 2, /only a privileged actor/); + no(cli(repo, genesisArgs(repo)), 2, /no actor: pass --by NAME or set MOSAIC_AGENT_NAME/); + no(cli(repo, genesisArgs(repo, { root: repo.base }), { by: "sage" }), 2, /is not this checkout's toplevel/); + no(cli(repo, genesisArgs(repo, { branch: "main" }), { by: "sage" }), 2, /HEAD is refs\/heads\/refactor; the queue runs only on branch main/); + no(cli(repo, genesisArgs(repo, { map: "agents/sage/work/nope.md" }), { by: "sage" }), 2, /not a committed file in HEAD/); + no(cli(repo, genesisArgs(repo, { op: "short" }), { by: "sage" }), 2, /must match/); + const missingOp = genesisArgs(repo).filter((a, i, all) => a !== "--op" && all[i - 1] !== "--op"); + no(cli(repo, missingOp, { by: "sage" }), 4, /--op ID is required/); + assert.throws(() => readFileSync(repo.queuePath)); +}); + +test("genesis: the map must be committed, well formed, with committed briefs and seats", (t) => { + const uncommitted = scratchRepo(t, { commitMap: false }); + no(cli(uncommitted, genesisArgs(uncommitted), { by: "sage" }), 2, /not a committed file in HEAD/); + const twice = scratchRepo(t, { map: mapText() + mapText() }); + no(cli(twice, genesisArgs(twice), { by: "sage" }), 2, /exactly one ```json queue-map block/); + const anchor = scratchRepo(t, { map: mapText(MAP_ROWS.map((r) => (r.id === 11 ? { ...r, brief: { path: "docs/plans/brief-b.md", anchor: "Nope" } } : r))) }); + no(cli(anchor, genesisArgs(anchor), { by: "sage" }), 2, /has 0 headings "Nope" in HEAD/); + const staged = scratchRepo(t, { map: mapText(MAP_ROWS.map((r) => (r.id === 11 ? { ...r, brief: { path: "docs/plans/new.md", anchor: "New" } } : r))) }); + writeFileSync(join(staged.root, "docs/plans/new.md"), "## New\n"); + staged.g("add", "docs/plans/new.md"); + no(cli(staged, genesisArgs(staged), { by: "sage" }), 2, /new.md is not committed in HEAD; commit it first \(a staged brief is refused\)/); + const ghost = scratchRepo(t, { map: mapText(MAP_ROWS.map((r) => (r.id === 11 ? { ...r, owner: "ghost" } : r))) }); + no(cli(ghost, genesisArgs(ghost), { by: "sage" }), 2, /row 11 owner ghost is not jason, coordinator, unassigned or a seat/); + const noBrief = scratchRepo(t, { map: mapText(MAP_ROWS.map((r) => (r.id === 11 ? { ...r, brief: null } : r))) }); + no(cli(noBrief, genesisArgs(noBrief), { by: "sage" }), 2, /only a done row may lack one/); +}); + +test("genesis: markers, a stray witness, once only; a retry returns the receipt", (t) => { + const repo = scratchRepo(t); + writeFileSync(repo.viewPath, "# QUEUE\n\nno markers\n"); + no(cli(repo, genesisArgs(repo), { by: "sage" }), 2, /needs the two queue markers/); + writeFileSync(repo.viewPath, queueMd()); + writeFileSync(join(repo.gitDir, "mosaic-queue.head"), "{}\n"); + no(cli(repo, genesisArgs(repo), { by: "sage" }), 2, /the witness .* exists without docs\/plans\/queue.json/); + unlinkSync(join(repo.gitDir, "mosaic-queue.head")); + const g = ok(cli(repo, genesisArgs(repo), { by: "sage" }), /^ok genesis-2026-09-26 rev 0 genesis 5 rows$/m); + const d = doc(repo); + assert.equal(d.canonicalRoot, repo.root); + assert.equal(d.log[0].result.legacyView, LEGACY); + assert.equal(d.log[0].result.mapBlob, repo.g("rev-parse", `HEAD:${MAP}`).trim()); + assert.match(readFileSync(repo.viewPath, "utf8"), /^# QUEUE\n\nHeader prose\.\n\n\n\nGenerated from `docs\/plans\/queue.json` revision 0/); + assert.match(readFileSync(repo.viewPath, "utf8"), /\n\nParked entries below\.\n$/); + ok(cli(repo, genesisArgs(repo), { by: "sage" }), new RegExp(`^${g.out.trim()} \\(already recorded at rev 0\\)`)); + no(cli(repo, genesisArgs(repo, { op: "genesis-other-01" }), { by: "sage" }), 2, /exists; genesis runs once/); + // Before the genesis commit, reads work and ops refuse. + ok(cli(repo, ["list"]), /^1\tdone\tdarkwing\tFinished thing\n6\tin-progress/); + no(cli(repo, ["note", "9", "x", "--op", "note-row9-001"], { by: "darkwing" }), 2, /genesis not committed: HEAD has no docs\/plans\/queue.json/); + repo.g("add", "docs/plans/queue.json", "docs/plans/QUEUE.md"); + repo.g("commit", "-q", "-m", "genesis"); + rmSync(repo.queuePath); + no(cli(repo, genesisArgs(repo, { op: "genesis-again-1" }), { by: "sage" }), 2, /HEAD already has docs\/plans\/queue.json|witness .* exists/); +}); + +test("genesis: a file holding genesis alone with no witness is confirmed by sync or a retry", (t) => { + const repo = ready(t); + unlinkSync(join(repo.gitDir, "mosaic-queue.head")); + no(cli(repo, ["list"]), 2, /witness .* is missing; .*run `scripts\/mosaic queue sync` \(the file holds genesis alone\)/); + ok(cli(repo, genesisArgs(repo), { by: "sage" }), /already recorded at rev 0/); + unlinkSync(join(repo.gitDir, "mosaic-queue.head")); + ok(cli(repo, ["sync"]), /durable now, never acknowledged: genesis-2026-09-26 by sage/); + ok(cli(repo, ["sync"]), /nothing to confirm: rev 0 is durable and witnessed/); +}); + +test("canonical checks: worktree, second clone, detached HEAD, wrong branch, GIT_DIR, foreign code; a symlink works", (t) => { + const repo = ready(t); + ok(cli(repo, ["verify"]), /^ok verify rev 0/); + const wt = join(repo.base, "wt"); + repo.g("worktree", "add", "-q", "--detach", wt); + no(cli(repo, ["list"], { cwd: wt }), 2, /is a linked worktree; the queue runs only in the canonical checkout/); + const clone = join(repo.base, "clone"); + sh("git", ["clone", "-q", "--branch", "refactor", repo.root, clone], { env: repo.env }); + no(cli(repo, ["list"], { cwd: clone }), 2, new RegExp(`this checkout is ${clone}; the queue's canonical root is ${repo.root}`)); + const cloneCli = { ...repo, root: clone }; + no(cli(cloneCli, ["list"], { cwd: clone }), 2, /the queue's canonical root is/); + repo.g("checkout", "-q", "--detach"); + no(cli(repo, ["list"]), 2, /HEAD is detached; the queue runs only on branch refactor/); + repo.g("checkout", "-q", "-b", "other"); + no(cli(repo, ["list"]), 2, /HEAD is refs\/heads\/other; the queue runs only on branch refactor/); + repo.g("checkout", "-q", "refactor"); + no(cli(repo, ["list"], { env: { GIT_DIR: repo.gitDir } }), 2, /GIT_DIR is set/); + no(cli(repo, ["list"], { env: { GIT_WORK_TREE: repo.root } }), 2, /GIT_WORK_TREE is set/); + no(cli(repo, ["list"], { cwd: repo.base }), 2, /is not inside a git checkout/); + const foreign = sh(process.execPath, [join(PKG_SRC, "cli.mjs"), "list"], { cwd: repo.root, env: repo.env, allowFail: true }); + assert.equal(foreign.status, 2); + assert.match(foreign.stderr, /this queue code .* is not under the canonical root/); + const link = join(repo.base, "link"); + symlinkSync(repo.root, link); + ok(cli(repo, ["list"], { cwd: join(link, "docs") }), /^1\tdone/); +}); + +test("op ids: missing, too long, reserved; a retry answers; another payload refuses", (t) => { + const repo = ready(t); + no(cli(repo, ["note", "9", "hello"], { by: "darkwing" }), 4, /--op ID is required/); + no(cli(repo, ["note", "9", "hello", "--op", "note-by-ghost"], { by: "ghost" }), 2, /actor ghost is not jason or a seat under agents/); + no(cli(repo, ["note", "9", "hello", "--op", "note-by-nobody"]), 2, /no actor/); + no(cli(repo, ["note", "9", "hello", "--op", "a".repeat(73)], { by: "darkwing" }), 2, /8 to 72 characters/); + ok(cli(repo, ["note", "9", "hello", "--op", "a".repeat(72)], { by: "darkwing" })); + no(cli(repo, ["note", "9", "hello", "--op", "note-row9.outcome"], { by: "darkwing" }), 2, /reserved/); + const first = ok(cli(repo, ["note", "9", "hi", "--op", "note-row9-0001"], { by: "darkwing" }), /^ok note-row9-0001 rev 2 row 9 note$/m); + ok(cli(repo, ["note", "9", "hi", "--op", "note-row9-0001"], { by: "darkwing" }), new RegExp(`^${first.out.trim()} \\(already recorded at rev 2\\)$`, "m")); + no(cli(repo, ["note", "9", "other", "--op", "note-row9-0001"], { by: "darkwing" }), 2, /recorded at rev 2 as note with other arguments; a new operation needs a new op id/); + no(cli(repo, ["release", "9", "--op", "note-row9-0001"], { by: "darkwing" }), 2, /as note with other arguments/); + no(cli(repo, ["move", "6", "in-progress", "--op", "move-row6-again"], { by: "darkwing" }), 2, /not a transition/); + assert.equal(doc(repo).revision, 2); +}); + +test("a retried add returns the id it first allocated, after reassignment and after done", (t) => { + const repo = ready(t); + const add = ["add", "--op", "add-new-row-01", "--piece", "New thing", "--gate", "Jason says", "--brief", "docs/plans/brief-b.md#Template", "--owner", "rocko", "--issue", "#1508"]; + const first = ok(cli(repo, add, { by: "sage" }), /row 12 none→queued$/m).out.trim(); + ok(cli(repo, ["assign", "12", "dewey", "--op", "assign-12-dewey"], { by: "sage" }), /owner: rocko→dewey/); + ok(cli(repo, add, { by: "sage" }), new RegExp(`^${first} \\(already recorded at rev 1\\)$`, "m")); + ok(cli(repo, ["move", "12", "briefed", "--op", "brief-12-0001"], { by: "sage" })); + ok(cli(repo, ["move", "12", "in-progress", "--op", "start-12-0001"], { by: "dewey" })); + ok(cli(repo, ["move", "12", "in-review", "--candidate", "HEAD", "--op", "review-12-0001"], { by: "dewey" }), /in-progress→in-review round 1/); + ok(cli(repo, ["move", "12", "waiting-on-jason", "--op", "wait-12-00001"], { by: "sage" })); + ok(cli(repo, ["move", "12", "done", "--op", "done-12-00001"], { by: "jason" })); + ok(cli(repo, add, { by: "sage" }), new RegExp(`^${first} \\(already recorded at rev 1\\)$`, "m")); + assert.equal(doc(repo).rows.filter((r) => r.piece === "New thing").length, 1); +}); + +test("Rocko's S4 schedule: a lost result, another writer, then the retry opens no second round", (t) => { + const repo = ready(t); + ok(cli(repo, ["move", "6", "blocked", "--reason", "paused", "--op", "block-6-00001"], { by: "darkwing" })); + ok(cli(repo, ["move", "9", "in-progress", "--op", "start-9-00001"], { by: "darkwing" })); + const review = ["move", "9", "in-review", "--candidate", "HEAD", "--op", "review-9-00001"]; + cli(repo, review, { by: "darkwing" }); // result lost + ok(cli(repo, ["note", "9", "looking now", "--op", "note-9-000001"], { by: "filbert" })); + ok(cli(repo, review, { by: "darkwing" }), /round 1 \(already recorded at rev 3\)/); + assert.equal(row(repo, 9).review.rounds.length, 1); +}); + +test("claims and add defaults through the CLI; candidates are manifests or reachable commits", (t) => { + const repo = ready(t); + ok(cli(repo, ["add", "--op", "add-by-dewey-1", "--piece", "Mine", "--gate", "tests", "--brief", "docs/plans/brief-b.md#Template"], { by: "dewey" })); + const r = row(repo, 12); + assert.deepEqual([r.owner, r.gateOwner, r.state, r.claim, r.brief.blob], ["dewey", "jason", "queued", null, repo.g("rev-parse", "HEAD:docs/plans/brief-b.md").trim()]); + no(cli(repo, ["add", "--op", "add-by-dewey-2", "--piece", "x", "--gate", "g", "--brief", "docs/plans/brief-b.md#Template", "--reviewer", "ghost"], { by: "sage" }), 2, /reviewer ghost is not jason or a seat/); + no(cli(repo, ["add", "--op", "add-by-dewey-3", "--piece", "x", "--gate", "g", "--brief", "docs/plans/brief-b.md"], { by: "dewey" }), 2, /PATH#ANCHOR/); + no(cli(repo, ["add", "--op", "add-by-dewey-4", "--piece", "x", "--gate", "g", "--brief", "docs/plans/brief-b.md#Queue", "--issue", "x"], { by: "dewey" }), 4); + ok(cli(repo, ["move", "11", "in-progress", "--op", "start-11-0001"], { by: "dewey" })); + assert.deepEqual(row(repo, 11).claim, { seat: "dewey", op: "start-11-0001" }); + no(cli(repo, ["move", "11", "in-review", "--candidate", "no-such-thing", "--op", "review-11-001"], { by: "dewey" }), 2, /neither a manifest file nor a commit/); + const dangling = repo.g("commit-tree", "-m", "x", `${repo.g("rev-parse", "HEAD^{tree}").trim()}`).trim(); + no(cli(repo, ["move", "11", "in-review", "--candidate", dangling, "--op", "review-11-002"], { by: "dewey" }), 2, /not reachable from any local branch or tag/); + const manifest = join(repo.base, "cand.sha256"); + writeFileSync(manifest, `${"e".repeat(64)} packages/x.mjs\n`); + ok(cli(repo, ["move", "11", "in-review", "--candidate", manifest, "--op", "review-11-003"], { by: "dewey" })); + assert.equal(row(repo, 11).review.rounds[0].candidate.kind, "manifest"); + ok(cli(repo, ["release", "6", "--op", "release-6-001"], { by: "sage" }), /in-progress→briefed/); + assert.equal(row(repo, 6).claim, null); +}); + +test("the working-brief check: a changed working copy refuses the start and flags next", (t) => { + const repo = ready(t); + writeFileSync(join(repo.root, "docs/plans/brief-b.md"), "# Briefs B\n\n## Queue\n\nEdited.\n\n## Template\n\nEdited.\n"); + ok(cli(repo, ["next", "dewey"]), /^start row 11: Brief template; brief docs\/plans\/brief-b.md § Template; brief differs from pinned blob; ask the lead to re-pin$/m); + no(cli(repo, ["move", "11", "in-progress", "--op", "start-11-0001"], { by: "dewey" }), 2, /row 11: brief differs from pinned blob; ask the lead to re-pin/); + repo.g("commit", "-q", "-am", "edit brief"); + no(cli(repo, ["move", "11", "in-progress", "--op", "start-11-0001"], { by: "dewey" }), 2, /brief differs from pinned blob/); + no(cli(repo, ["verify", "--current"]), 2, /brief drift against HEAD:\nrow 9: brief docs\/plans\/brief-b.md changed in HEAD .*\nrow 11:/); + ok(cli(repo, ["verify"])); + ok(cli(repo, ["set", "11", "brief", "docs/plans/brief-b.md#Template", "--op", "repin-11-0001"], { by: "sage" }), /row 11 brief: docs\/plans\/brief-b.md § Template @\w{12}→docs\/plans\/brief-b.md § Template @\w{12}/); + ok(cli(repo, ["next", "dewey"]), /^start row 11: Brief template; brief docs\/plans\/brief-b.md § Template$/m); + ok(cli(repo, ["move", "11", "in-progress", "--op", "start-11-0001"], { by: "dewey" })); +}); + +test("next: resume first, then nothing for an idle seat; needs a seat", (t) => { + const repo = ready(t); + ok(cli(repo, ["next", "darkwing"]), /^resume row 6: Row six/m); + ok(cli(repo, ["next"], { by: "dewey" }), /^start row 11/m); + ok(cli(repo, ["next", "rocko"]), /^nothing$/m); + no(cli(repo, ["next"]), 2, /next needs a seat/); +}); + +test("view stale: new ops and verify refuse naming the unshown op; retries answer; reads warn; render fixes", (t) => { + const repo = ready(t); + const first = ok(cli(repo, ["note", "9", "one", "--op", "note-9-000001"], { by: "darkwing" })).out.trim(); + repo.g("checkout", "--", "docs/plans/QUEUE.md"); + no(cli(repo, ["note", "9", "two", "--op", "note-9-000002"], { by: "darkwing" }), 2, /view stale: QUEUE.md shows rev 0; rev 1 \(op note-9-000001 by darkwing at .*\) is recorded .*Tell darkwing, then run `scripts\/mosaic queue render`/); + no(cli(repo, ["verify"]), 2, /view stale/); + const retry = ok(cli(repo, ["note", "9", "one", "--op", "note-9-000001"], { by: "darkwing" })); + assert.equal(retry.out.trim(), `${first} (already recorded at rev 1)`); + assert.match(retry.err, /warning: view stale/); + const read = ok(cli(repo, ["list"])); + assert.match(read.err, /warning: view stale/); + no(cli(repo, ["render", "--check"]), 2, /view stale/); + ok(cli(repo, ["render"]), /rendered rev 1 over rev 0; newly shown: rev 1 \(op note-9-000001 by darkwing/); + ok(cli(repo, ["render"]), /view current at rev 1; nothing written/); + ok(cli(repo, ["verify"])); +}); + +test("view unknown: a hand edit, an old marker over an edited body, missing or duplicate markers", (t) => { + const repo = ready(t); + ok(cli(repo, ["note", "9", "one", "--op", "note-9-000001"], { by: "darkwing" })); + const good = readFileSync(repo.viewPath, "utf8"); + const cases = [ + good.replace("| one |", "| two |"), + repo.g("show", "HEAD:docs/plans/QUEUE.md").replace("Row six", "Row 6"), + good.replace("\n", ""), + good + "\n", + ]; + for (const text of cases) { + writeFileSync(repo.viewPath, text); + no(cli(repo, ["note", "9", "two", "--op", "note-9-000002"], { by: "darkwing" }), 2, /view unknown: .*restore the table with git or re-apply the edit as queue ops/); + no(cli(repo, ["render"]), 2, /view unknown/); + no(cli(repo, ["verify"]), 2, /view unknown/); + assert.match(ok(cli(repo, ["show", "9"])).err, /warning: view unknown/); + assert.equal(readFileSync(repo.viewPath, "utf8"), text, "nothing rewrote an unknown view"); + } + writeFileSync(repo.viewPath, good.replace("Header prose.", "Header prose, edited by hand.")); + ok(cli(repo, ["note", "9", "two", "--op", "note-9-000002"], { by: "darkwing" })); + assert.match(readFileSync(repo.viewPath, "utf8"), /edited by hand/); +}); + +test("a hand edit to queue.json refuses every verb, reads included", (t) => { + const repo = ready(t); + const text = readFileSync(repo.queuePath, "utf8"); + writeFileSync(repo.queuePath, text.replace('"piece": "Row six"', '"piece": "Row 6"')); + for (const args of [["list"], ["show", "6"], ["next", "darkwing"], ["verify"], ["render"], ["sync"]]) no(cli(repo, args), 2, /do not equal the replay/); + no(cli(repo, ["note", "9", "x", "--op", "note-9-000001"], { by: "darkwing" }), 2, /do not equal the replay/); + writeFileSync(repo.queuePath, text.replace("\n", "\n\n")); + no(cli(repo, ["list"]), 2, /re-serialize byte for byte/); + writeFileSync(repo.queuePath, "{ not json"); + no(cli(repo, ["list"]), 2, /not valid JSON|is not JSON/); +}); + +test("verify and render --check leave bytes and mtimes unchanged", (t) => { + const repo = ready(t); + const snap = () => [repo.queuePath, repo.viewPath, join(repo.gitDir, "mosaic-queue.head")].map((p) => [readFileSync(p, "hex"), statSync(p, { bigint: true }).mtimeNs]); + const before = snap(); + ok(cli(repo, ["verify"])); + ok(cli(repo, ["verify", "--current"])); + ok(cli(repo, ["render", "--check"])); + ok(cli(repo, ["render"])); + ok(cli(repo, ["list"])); + ok(cli(repo, ["next", "darkwing"])); + assert.deepEqual(snap(), before); +}); + +test("render is byte-stable across runs and repositories", (t) => { + const a = ready(t); + const b = ready(t); + const body = (repo) => readFileSync(repo.viewPath, "utf8").replace(repo.root, "ROOT"); + assert.equal(body(a), body(b)); + const bytes = readFileSync(a.viewPath); + writeFileSync(join(a.base, "x"), ""); + ok(cli(a, ["render"])); + assert.ok(readFileSync(a.viewPath).equals(bytes)); +}); + +test("snapshot and verify --snapshot", (t) => { + const repo = ready(t); + const out = join(repo.base, "snap"); + mkdirSync(out); + no(cli(repo, ["snapshot", "--out", join(repo.root, "docs")]), 2, /is not empty/); + mkdirSync(join(repo.root, "empty-inside")); + no(cli(repo, ["snapshot", "--out", "empty-inside"]), 2, /is inside the repository/); + no(cli(repo, ["snapshot", "--out", join(repo.base, "missing")]), 2, /does not exist/); + ok(cli(repo, ["snapshot", "--out", out]), /^snapshot rev 0: queue.json [0-9a-f]{64}, QUEUE.md [0-9a-f]{64}$/m); + ok(cli(repo, ["verify", "--snapshot", out, "--base-absent"], { cwd: repo.base }), /genesis alone/); + no(cli(repo, ["verify", "--snapshot", out]), 4, /exactly one of/); + const base = join(repo.base, "base.json"); + writeFileSync(base, readFileSync(repo.queuePath)); + ok(cli(repo, ["note", "9", "one", "--op", "note-9-000001"], { by: "darkwing" })); + const out2 = join(repo.base, "snap2"); + mkdirSync(out2); + ok(cli(repo, ["snapshot", "--out", out2])); + ok(cli(repo, ["verify", "--snapshot", out2, "--base-file", base], { cwd: repo.base }), /extends the base/); + no(cli(repo, ["verify", "--snapshot", out2, "--base-absent"]), 2, /must hold genesis alone/); + no(cli(repo, ["verify", "--snapshot", out, "--base-file", join(out2, "queue.json")]), 2, /does not extend the base/); + writeFileSync(join(out2, "QUEUE.md"), readFileSync(join(out, "QUEUE.md"))); + no(cli(repo, ["verify", "--snapshot", out2, "--base-file", base]), 2, /is not the render of rev 1 \(stale\)/); + repo.g("checkout", "--", "docs/plans/QUEUE.md"); + const out3 = join(repo.base, "snap3"); + mkdirSync(out3); + no(cli(repo, ["snapshot", "--out", out3]), 2, /view stale/); +}); + +test("usage errors exit 4", (t) => { + const repo = ready(t); + no(cli(repo, []), 4); + no(cli(repo, ["frobnicate"]), 4, /unknown verb/); + no(cli(repo, ["list", "--bogus"]), 4, /unknown option --bogus/); + no(cli(repo, ["show"]), 4, /expected show ID/); + no(cli(repo, ["show", "0"]), 4, /positive integer/); + no(cli(repo, ["move", "9", "done", "--op", "x-00000001", "--op", "y-00000001"], { by: "sage" }), 4, /--op given twice/); + no(cli(repo, ["set", "9", "owner", "x", "--op", "set-9-000001"], { by: "sage" }), 4, /set fields/); + no(cli(repo, ["set", "9", "required", "yes", "--op", "set-9-000001"], { by: "sage" }), 4, /true or false/); + no(cli(repo, ["add", "--op", "add-0000001", "--piece", "x"], { by: "sage" }), 4, /add needs --gate/); + no(cli(repo, ["list", "--check"]), 4, /does not apply here/); + ok(cli(repo, ["--help"]), /^usage: queue list/); +}); diff --git a/packages/queue/tests/write.test.mjs b/packages/queue/tests/write.test.mjs new file mode 100644 index 00000000..0d0781c0 --- /dev/null +++ b/packages/queue/tests/write.test.mjs @@ -0,0 +1,313 @@ +// The write path (8.5): injected faults, SIGKILL at each step, git +// interference, the witness, sync and accept-history, and unlocked reads +// racing a writer (8.4, F2). +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { readFileSync, readdirSync, unlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { cli, genesisCommitted, load, scratchRepo } from "./helpers.mjs"; + +const HERE = new URL(".", import.meta.url).pathname; + +async function ready(t) { + const repo = scratchRepo(t); + genesisCommitted(repo); + return { repo, m: await load(repo) }; +} + +const note = (id, text, op, by = "darkwing") => ({ verb: "note", op, args: { id, text }, by }); +const o = (repo, extra = {}) => ({ cwd: repo.root, env: repo.env, lockWaitMs: 300, lockStepMs: 50, ...extra }); +const read = (p) => readFileSync(p, "utf8"); +const revOf = (repo) => JSON.parse(read(repo.queuePath)).revision; +const witness = (repo) => JSON.parse(read(join(repo.gitDir, "mosaic-queue.head"))); +const shownRev = (repo) => Number(/revision (\d+) by/.exec(read(repo.viewPath))[1]); +const tmps = (repo) => [...readdirSync(join(repo.root, "docs/plans")), ...readdirSync(repo.gitDir)].filter((n) => n.endsWith(".tmp")); + +function throwsCode(fn, code, re) { + assert.throws(fn, (err) => { + assert.equal(err.code, code, err.message); + assert.match(err.message, re); + return true; + }); +} + +// realIo with one fault: `name` fails for paths that `match`. +function faultIo(m, name, match, code) { + const real = m.io.realIo; + const paths = new Map(); + const fail = () => { throw Object.assign(new Error(code), { code }); }; + const hit = (n, p) => n === name && p !== undefined && match(p); + return { + ...real, + openExcl: (p, mode) => { const fd = real.openExcl(p, mode); paths.set(fd, p); return fd; }, + close: (fd) => { paths.delete(fd); real.close(fd); }, + write: (fd, b, off, len) => (hit("write", paths.get(fd)) ? (code === "SHORT" ? 0 : fail()) : real.write(fd, b, off, len)), + fsync: (fd) => (hit("fsync", paths.get(fd)) ? fail() : real.fsync(fd)), + rename: (a, b) => (hit("rename", b) ? fail() : real.rename(a, b)), + fsyncDir: (d) => (hit("fsyncDir", d) ? fail() : real.fsyncDir(d)), + }; +} + +const queueTmp = (p) => /queue\.json\.[^/]*\.tmp$/.test(p); + +test("a short write, ENOSPC or a file fsync failure: nothing visible, temp removed", async (t) => { + const { repo, m } = await ready(t); + const before = read(repo.queuePath); + for (const [name, code] of [["write", "SHORT"], ["write", "ENOSPC"], ["fsync", "EIO"]]) { + const io = faultIo(m, name, queueTmp, code); + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 1, /cannot write docs\/plans\/queue.json.note-9-000001.tmp \((ESHORT|ENOSPC|EIO)\); nothing changed/); + assert.equal(read(repo.queuePath), before); + assert.deepEqual(tmps(repo), []); + assert.equal(witness(repo).revision, 0); + } + assert.match(m.store.mutate(o(repo), note(9, "x", "note-9-000001")).out[0], /^ok note-9-000001 rev 1/); +}); + +test("a rename failure: nothing visible, temp removed", async (t) => { + const { repo, m } = await ready(t); + const before = read(repo.queuePath); + const io = faultIo(m, "rename", (p) => p === repo.queuePath, "EXDEV"); + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 1, /cannot replace docs\/plans\/queue.json \(EXDEV\); nothing changed/); + assert.equal(read(repo.queuePath), before); + assert.deepEqual(tmps(repo), []); +}); + +test("a directory fsync failure: uncertain, exit 3, no receipt; the tail refuses new ops; a retry confirms", async (t) => { + const { repo, m } = await ready(t); + const view = read(repo.viewPath); + const io = faultIo(m, "fsyncDir", (d) => d === join(repo.root, "docs/plans"), "EIO"); + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /^uncertain note-9-000001 rev 1: visible, durability not confirmed \(EIO\)$/); + assert.equal(revOf(repo), 1); + assert.equal(witness(repo).revision, 0); + assert.equal(read(repo.viewPath), view); + throwsCode(() => m.store.mutate(o(repo), note(9, "y", "note-9-000002")), 2, /unconfirmed tail: rev 1 \(op note-9-000001 by darkwing at .*\) visible but not confirmed durable; run `scripts\/mosaic queue sync` or retry that op/); + const unlocked = m.store.list(o(repo)); + assert.match(unlocked.err.join("\n"), /rev 1 visible, not confirmed durable/); + const retry = m.store.mutate(o(repo), note(9, "x", "note-9-000001")); + assert.match(retry.err.join("\n"), /durable now, never acknowledged: note-9-000001 by darkwing/); + assert.match(retry.err.join("\n"), /warning: view stale/); + assert.match(retry.out[0], /\(already recorded at rev 1\)$/); + assert.equal(witness(repo).revision, 1); +}); + +test("a directory fsync failure, then sync names the op", async (t) => { + const { repo, m } = await ready(t); + const io = faultIo(m, "fsyncDir", (d) => d === join(repo.root, "docs/plans"), "EIO"); + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /uncertain/); + const r = cli(repo, ["sync"]); + assert.equal(r.code, 0, r.err); + assert.match(r.out, /^durable now, never acknowledged: note-9-000001 by darkwing at /); + assert.match(r.err, /view stale/); + assert.equal(cli(repo, ["render"]).code, 0); + assert.equal(cli(repo, ["verify"]).code, 0); +}); + +test("a witness write failure: uncertain, durable, exit 3; the view is untouched", async (t) => { + const { repo, m } = await ready(t); + const io = faultIo(m, "rename", (p) => p.endsWith("mosaic-queue.head"), "EIO"); + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")), 3, /^uncertain note-9-000001 rev 1: durable, witness not updated \(EIO\)$/); + assert.equal(revOf(repo), 1); + assert.equal(witness(repo).revision, 0); + assert.equal(shownRev(repo), 0); + assert.deepEqual(tmps(repo), []); + assert.match(m.store.mutate(o(repo), note(9, "x", "note-9-000001")).out[0], /already recorded at rev 1/); +}); + +test("a view write that fails keeps the op and reports a stale view", async (t) => { + const { repo, m } = await ready(t); + const io = faultIo(m, "rename", (p) => p === repo.viewPath, "EIO"); + const r = m.store.mutate(o(repo, { io }), note(9, "x", "note-9-000001")); + assert.match(r.out[0], /^ok note-9-000001 rev 1/); + assert.match(r.err.join("\n"), /the view write failed \(EIO\); the op stands and the view is stale/); + assert.equal(shownRev(repo), 0); + assert.deepEqual(tmps(repo), []); +}); + +function killAt(repo, m, step, req) { + const r = spawnSync(process.execPath, [join(HERE, "fixtures", "kill-at.mjs"), join(repo.root, "packages/queue/src"), step, JSON.stringify(req)], { + cwd: repo.root, env: repo.env, encoding: "utf8", + }); + assert.equal(r.signal, "SIGKILL", `child did not die at ${step}: ${r.stdout}${r.stderr}`); + // The dead child's lock stays; nothing removes it by age. + // The 10 s wait is lock.test.mjs's; here the wait is short. + throwsCode(() => m.store.mutate(o(repo), note(9, "z", "note-9-blocked")), 2, /queue lock owner is dead: pid \d+, note .*; run `scripts\/mosaic queue unlock`/); + const u = cli(repo, ["unlock"]); + assert.equal(u.code, 0, u.err); + assert.match(u.out, /^removed queue lock \(dead/); +} + +test("SIGKILL before the rename: nothing recorded; the retry removes the leftover temp", async (t) => { + const { repo, m } = await ready(t); + const before = read(repo.queuePath); + killAt(repo, m, "temp-written", note(9, "x", "note-9-000001")); + assert.equal(read(repo.queuePath), before); + assert.deepEqual(tmps(repo), ["queue.json.note-9-000001.tmp"]); + const r = cli(repo, ["note", "9", "x", "--op", "note-9-000001"], { by: "darkwing" }); + assert.match(r.out, /^ok note-9-000001 rev 1 row 9 note$/m); + assert.deepEqual(tmps(repo), []); +}); + +test("SIGKILL after the rename, before the witness: the tail refuses new ops and sync names the op", async (t) => { + const { repo, m } = await ready(t); + killAt(repo, m, "renamed", note(9, "x", "note-9-000001")); + assert.equal(revOf(repo), 1); + assert.equal(witness(repo).revision, 0); + const r = cli(repo, ["note", "9", "y", "--op", "note-9-000002"], { by: "darkwing" }); + assert.equal(r.code, 2); + assert.match(r.err, /unconfirmed tail: rev 1 \(op note-9-000001 by darkwing/); + assert.match(cli(repo, ["sync"]).out, /durable now, never acknowledged: note-9-000001 by darkwing/); +}); + +test("SIGKILL after the witness, before the view: the stale refusal names the op", async (t) => { + const { repo, m } = await ready(t); + killAt(repo, m, "witnessed", note(9, "x", "note-9-000001")); + assert.equal(witness(repo).revision, 1); + const r = cli(repo, ["note", "9", "y", "--op", "note-9-000002"], { by: "darkwing" }); + assert.equal(r.code, 2); + assert.match(r.err, /view stale: QUEUE.md shows rev 0; rev 1 \(op note-9-000001 by darkwing at .*\) is recorded but the table shows rev 0 and may never have been acknowledged\. Tell darkwing/); +}); + +test("SIGKILL after the view, before the receipt: the retry returns the receipt", async (t) => { + const { repo, m } = await ready(t); + killAt(repo, m, "viewed", note(9, "x", "note-9-000001")); + assert.equal(shownRev(repo), 1); + const r = cli(repo, ["note", "9", "x", "--op", "note-9-000001"], { by: "darkwing" }); + assert.equal(r.code, 0); + assert.equal(r.out, "ok note-9-000001 rev 1 row 9 note (already recorded at rev 1)\n"); + assert.equal(r.err, ""); +}); + +test("git checkout between steps 1 and 7: step 7 refuses and nothing is written", async (t) => { + const { repo, m } = await ready(t); + m.store.mutate(o(repo), note(9, "one", "note-9-000001")); + const hook = (n) => { if (n === "temp-written") repo.g("checkout", "--", "docs/plans/queue.json"); }; + throwsCode(() => m.store.mutate(o(repo, { hook }), note(9, "two", "note-9-000002")), 2, /changed outside the queue lock \(git\?\) since it was read; nothing changed/); + assert.equal(revOf(repo), 0); + assert.deepEqual(tmps(repo), []); + throwsCode(() => m.store.list(o(repo)), 2, /history lost: the witness recorded rev 1/); +}); + +test("git stash restoring an older valid pair: history lost; accept-history needs privilege, a reason and --yes", async (t) => { + const { repo, m } = await ready(t); + m.store.mutate(o(repo), note(9, "one", "note-9-000001")); + repo.g("commit", "-q", "-am", "queue rev 1"); + m.store.mutate(o(repo), note(9, "two", "note-9-000002")); + repo.g("stash", "-q"); + assert.equal(revOf(repo), 1); + for (const args of [["list"], ["show", "9"], ["verify"], ["render"], ["sync"]]) { + const r = cli(repo, args); + assert.equal(r.code, 2, args.join(" ")); + assert.match(r.err, /history lost: the witness recorded rev 2 \(logDigest [0-9a-f]{12}…, at .*\); this file holds revs 0\.\.1 and does not extend it; every verb refuses except `scripts\/mosaic queue accept-history`/); + } + assert.equal(cli(repo, ["note", "9", "x", "--op", "note-9-000003"], { by: "darkwing" }).code, 2); + const noYes = cli(repo, ["accept-history", "--reason", "stash by mistake", "--op", "accept-hist-01"], { by: "sage" }); + assert.equal(noYes.code, 2); + assert.match(noYes.err, /history lost: .*revs 0\.\.1.*\. ops in that range are no longer deduplicated\. Re-run with --yes/); + const seat = cli(repo, ["accept-history", "--reason", "x", "--yes", "--op", "accept-hist-01"], { by: "darkwing" }); + assert.equal(seat.code, 2); + assert.match(seat.err, /privileged/); + assert.equal(cli(repo, ["accept-history", "--yes", "--op", "accept-hist-01"], { by: "sage" }).code, 4); + const ok = cli(repo, ["accept-history", "--reason", "stash by mistake", "--yes", "--op", "accept-hist-01"], { by: "sage" }); + assert.equal(ok.code, 0, ok.err); + assert.match(ok.err, /ops in that range are no longer deduplicated/); + assert.match(ok.out, /^ok accept-hist-01 rev 2 accept-history over witness rev 2/m); + const d = JSON.parse(read(repo.queuePath)); + assert.equal(d.log[2].result.oldWitness.revision, 2); + assert.equal(witness(repo).revision, 2); + assert.equal(cli(repo, ["verify"]).code, 0); + // The lost op id is no longer deduplicated: it records again. + assert.match(cli(repo, ["note", "9", "two", "--op", "note-9-000002"], { by: "darkwing" }).out, /^ok note-9-000002 rev 3/m); + const again = cli(repo, ["accept-history", "--reason", "x", "--yes", "--op", "accept-hist-02"], { by: "sage" }); + assert.equal(again.code, 2); + assert.match(again.err, /history is not lost/); +}); + +test("a deleted witness: refused after the locked recheck; accept-history records it absent", async (t) => { + const { repo, m } = await ready(t); + m.store.mutate(o(repo), note(9, "one", "note-9-000001")); + unlinkSync(join(repo.gitDir, "mosaic-queue.head")); + throwsCode(() => m.store.list(o(repo)), 2, /the witness \.git\/mosaic-queue.head is missing; this file holds revs 0\.\.1, and ops recorded after them may be lost; every verb refuses except/); + throwsCode(() => m.store.sync(o(repo)), 2, /is missing/); + writeFileSync(join(repo.gitDir, "mosaic-queue.head"), "not json\n"); + throwsCode(() => m.store.list(o(repo)), 2, /the witness .* is invalid/); + const r = cli(repo, ["accept-history", "--reason", "witness deleted", "--yes", "--op", "accept-hist-01"], { by: "jason" }); + assert.equal(r.code, 0, r.err); + assert.match(r.out, /accept-history over witness absent/); + assert.equal(JSON.parse(read(repo.queuePath)).log[2].result.oldWitness, null); +}); + +test("a header edit during a write: the op stands, the view write is skipped with a warning", async (t) => { + const { repo, m } = await ready(t); + const hook = (n) => { if (n === "witnessed") writeFileSync(repo.viewPath, read(repo.viewPath).replace("Header prose.", "Header prose, edited.")); }; + const r = m.store.mutate(o(repo, { hook }), note(9, "one", "note-9-000001")); + assert.match(r.out[0], /^ok note-9-000001 rev 1/); + assert.match(r.err.join("\n"), /QUEUE.md changed since it was read; the view was not written and is stale/); + assert.equal(shownRev(repo), 0); + assert.match(m.store.renderView(o(repo)).out[0], /rendered rev 1 over rev 0/); + assert.match(read(repo.viewPath), /Header prose, edited\./); +}); + +test("a reader paused between the witness and the file while a writer finishes: no lost-history report", async (t) => { + const { repo, m } = await ready(t); + const hook = (n) => { if (n === "reader-between") m.store.mutate(o(repo), note(9, "one", "note-9-000001")); }; + const r = m.store.list(o(repo, { hook })); + assert.equal(r.code, 0); + assert.deepEqual(r.err, ["rev 1 visible, not confirmed durable"]); +}); + +test("file-then-witness order forced by a hook: the locked recheck prevents a false report", async (t) => { + const { repo, m } = await ready(t); + const hook = (n) => { if (n === "reader-between") m.store.mutate(o(repo), note(9, "one", "note-9-000001")); }; + const r = m.store.show(o(repo, { hook, readOrder: "file-first" }), 9); + assert.equal(r.code, 0); + assert.deepEqual(r.err, []); + assert.equal(JSON.parse(r.out[0]).note, "one", "the recheck reports the current revision"); +}); + +test("a writer paused before and after the witness rename: readers see a tail, then a match", async (t) => { + const { repo, m } = await ready(t); + const seen = {}; + const hook = (n) => { + if (n === "dir-synced" || n === "witnessed") seen[n] = m.store.list(o(repo)).err; + }; + m.store.mutate(o(repo, { hook }), note(9, "one", "note-9-000001")); + assert.equal(seen["dir-synced"][0], "rev 1 visible, not confirmed durable"); + assert.equal(seen.witnessed.some((l) => /lost|visible/.test(l)), false); + // Both readers also warn that the view (written after the witness) is behind. + for (const lines of Object.values(seen)) assert.match(lines.at(-1), /^warning: view stale: QUEUE.md shows rev 0/); +}); + +test("a true rollback is reported only after the locked recheck; a held lock names its holder instead", async (t) => { + const { repo, m } = await ready(t); + m.store.mutate(o(repo), note(9, "one", "note-9-000001")); + repo.g("checkout", "--", "docs/plans/queue.json", "docs/plans/QUEUE.md"); + const child = spawn(process.execPath, [join(HERE, "fixtures", "lock-child.mjs"), repo.gitDir, "hold"], { stdio: ["ignore", "pipe", "ignore"] }); + t.after(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } }); + await new Promise((r) => child.stdout.once("data", r)); + throwsCode(() => m.store.list(o(repo)), 2, /^queue lock held by move holder-op-1 since/); + child.kill("SIGKILL"); + await new Promise((r) => child.on("exit", r)); + m.store.unlock(o(repo)); + throwsCode(() => m.store.list(o(repo)), 2, /history lost: the witness recorded rev 1/); +}); + +test("an accept-history in progress: an unlocked reader waits on the lock and never reports lost history", async (t) => { + const { repo, m } = await ready(t); + m.store.mutate(o(repo), note(9, "one", "note-9-000001")); + unlinkSync(join(repo.gitDir, "mosaic-queue.head")); + let during; + const hook = (n) => { + if (n !== "renamed") return; + try { during = m.store.list(o(repo)); } catch (err) { during = err; } + }; + m.store.mutate(o(repo, { hook }), { verb: "accept-history", op: "accept-hist-01", args: { reason: "witness deleted" }, by: "sage", yes: true }); + assert.match(during.message, /^queue lock held by accept-history accept-hist-01/); + assert.equal(m.store.list(o(repo)).err.length, 0); +}); + +test("the platform check refuses other filesystems", async (t) => { + const { repo, m } = await ready(t); + const io = { ...m.io.realIo, statfsType: () => 0x6969 }; + throwsCode(() => m.store.mutate(o(repo, { io }), note(9, "one", "note-9-000001")), 2, /unsupported filesystem .*type 0x6969/); +}); diff --git a/scripts/git-hooks/pre-commit b/scripts/git-hooks/pre-commit new file mode 100755 index 00000000..abdf14e7 --- /dev/null +++ b/scripts/git-hooks/pre-commit @@ -0,0 +1,12 @@ +#!/bin/sh +# Mosaic queue guard (docs 8.12). Installed as .git/hooks/pre-commit by +# `scripts/queue-commit.sh --install-hook`; never edit the installed copy. +# It refuses any commit whose index entries for the queue's two files differ +# from HEAD's. Only scripts/queue-commit.sh commits them, through +# commit-tree, which runs no hooks. Any failure of the check refuses too. +if git diff --cached --quiet HEAD -- docs/plans/queue.json docs/plans/QUEUE.md; then + exit 0 +fi +echo "mosaic queue guard: refused: this commit would change docs/plans/queue.json or docs/plans/QUEUE.md; only scripts/queue-commit.sh commits them" >&2 +echo "fix: git reset -q -- docs/plans/queue.json docs/plans/QUEUE.md" >&2 +exit 1 diff --git a/scripts/queue-commit.sh b/scripts/queue-commit.sh new file mode 100755 index 00000000..516e4a9d --- /dev/null +++ b/scripts/queue-commit.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# queue-commit.sh: the lead's commit procedure for the queue (queue-as-data +# plan 8.12). It commits exactly the tested bytes of docs/plans/queue.json and +# docs/plans/QUEUE.md on top of HEAD through a temporary index and +# commit-tree, so nothing anyone else has staged is swept in. +# +# Usage: +# scripts/queue-commit.sh -m MSG commit the queue's next revisions +# scripts/queue-commit.sh --genesis -m MSG the first queue commit (HEAD has no queue.json) +# scripts/queue-commit.sh --install-hook [--by NAME] +# install the queue guard (jason or sage) +# +# Exit codes: 0 ok; 1 failed, nothing published; 2 refused, nothing +# published; 3 committed but the shared index was not reconciled (the +# printed command finishes it); 4 usage. +# +# Committing still needs its own authorization. This procedure doesn't give +# it, and it never pushes. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +QJSON=docs/plans/queue.json +QMD=docs/plans/QUEUE.md +HOOK_REL=scripts/git-hooks/pre-commit +FIX="git reset -q -- $QJSON $QMD" +TMPD="" + +say() { printf 'queue-commit: %s\n' "$*" >&2; } +die() { local code=$1; shift; say "$*"; exit "$code"; } +cleanup() { if [ -n "$TMPD" ]; then rm -rf -- "$TMPD"; fi; } +trap cleanup EXIT + +usage() { + sed -n '8,12p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + exit 4 +} + +g() { git -C "$ROOT" "$@"; } + +# --- arguments --- +MODE=commit +GENESIS=0 +MSG="" +HAVE_MSG=0 +BY="" +while [ $# -gt 0 ]; do + case "$1" in + -m) [ $# -ge 2 ] || usage; MSG=$2; HAVE_MSG=1; shift 2 ;; + --genesis) GENESIS=1; shift ;; + --install-hook) MODE=install; shift ;; + --by) [ $# -ge 2 ] || usage; BY=$2; shift 2 ;; + -h|--help) usage ;; + *) say "unknown argument: $1"; usage ;; + esac +done +if [ "$MODE" = install ]; then + { [ "$HAVE_MSG" = 0 ] && [ "$GENESIS" = 0 ]; } || { say "--install-hook takes only --by"; usage; } +else + [ -z "$BY" ] || { say "--by applies only to --install-hook"; usage; } + [ "$HAVE_MSG" = 1 ] && [ -n "$MSG" ] || { say "a commit needs -m MSG"; usage; } +fi + +# --- the repository (8.3): the canonical checkout, located only from this script --- +for v in GIT_DIR GIT_WORK_TREE GIT_COMMON_DIR GIT_INDEX_FILE GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES; do + if [ -n "${!v+x}" ]; then die 2 "refused: $v is set; run with the normal git environment"; fi +done +top=$(g rev-parse --show-toplevel 2>/dev/null) || die 2 "refused: $ROOT is not a git checkout" +[ "$(cd "$top" && pwd -P)" = "$ROOT" ] || die 2 "refused: $ROOT is not the top of its checkout" +gd=$(g rev-parse --path-format=absolute --git-dir) || die 1 "cannot read the git directory" +cd_=$(g rev-parse --path-format=absolute --git-common-dir) || die 1 "cannot read the git common directory" +[ "$(cd "$gd" && pwd -P)" = "$ROOT/.git" ] && [ "$(cd "$cd_" && pwd -P)" = "$ROOT/.git" ] \ + || die 2 "refused: $ROOT's git directory is not $ROOT/.git (a linked worktree or a separate git dir)" +HOOK="$ROOT/.git/hooks/pre-commit" + +TMPD=$(mktemp -d "${TMPDIR:-/tmp}/queue-commit.XXXXXX") || die 1 "mktemp failed" + +# --- the queue guard, active and not just present (8.12, G2). $1 is the +# commit whose hook blob and tree the checks use. --- +guard_check() { + local h=$1 when=$2 idx out rc + [ -e "$HOOK" ] || [ -L "$HOOK" ] || die 2 "refused ($when): the queue guard is not installed at .git/hooks/pre-commit; run \`scripts/queue-commit.sh --install-hook\`" + [ ! -L "$HOOK" ] || die 2 "refused ($when): .git/hooks/pre-commit is a symlink" + [ -f "$HOOK" ] || die 2 "refused ($when): .git/hooks/pre-commit is not a regular file" + [ -O "$HOOK" ] || die 2 "refused ($when): .git/hooks/pre-commit is not owned by this user" + [ -x "$HOOK" ] || die 2 "refused ($when): .git/hooks/pre-commit is not executable, so git would skip it" + g cat-file -e "$h:$HOOK_REL" 2>/dev/null || die 2 "refused ($when): $h has no $HOOK_REL" + g cat-file blob "$h:$HOOK_REL" | cmp -s - "$HOOK" \ + || die 2 "refused ($when): .git/hooks/pre-commit differs from $HOOK_REL at $h" + out=$(g config --show-scope --get-all core.hooksPath 2>/dev/null) + [ -z "$out" ] || die 2 "refused ($when): core.hooksPath is set ($(printf '%s' "$out" | tr '\n\t' '; ')), so git would not run the queue guard" + # The canary: git itself runs the hook it would run for a commit. + idx="$TMPD/canary-$when/index" + mkdir -p "$(dirname "$idx")" + GIT_INDEX_FILE=$idx g read-tree "$h" || die 1 "canary ($when): read-tree failed" + if ! out=$(cd "$ROOT" && GIT_INDEX_FILE=$idx git hook run pre-commit 2>&1); then + die 2 "refused ($when): the canary's clean run failed, so git is not running the queue guard as installed: $out" + fi + GIT_INDEX_FILE=$idx g update-index --add --cacheinfo "100644,$(g rev-parse "$h:$HOOK_REL"),$QMD" \ + || die 1 "canary ($when): update-index failed" + out=$(cd "$ROOT" && GIT_INDEX_FILE=$idx git hook run pre-commit 2>&1); rc=$? + [ "$rc" -ne 0 ] && [[ "$out" == *"mosaic queue guard: refused"* ]] \ + || die 2 "refused ($when): the canary's changed run was not refused by the queue guard (exit $rc)" + rm -rf -- "$(dirname "$idx")" +} + +# --- --install-hook --- +if [ "$MODE" = install ]; then + [ -n "$BY" ] || BY=${MOSAIC_AGENT_NAME:-} + [ -n "$BY" ] || die 2 "refused: no actor; pass --by NAME or set MOSAIC_AGENT_NAME" + case "$BY" in jason|sage) ;; *) die 2 "refused: installing the queue guard is privileged (jason or sage), not $BY" ;; esac + H=$(g rev-parse --verify -q HEAD) || die 2 "refused: HEAD has no commit" + out=$(g config --show-scope --get-all core.hooksPath 2>/dev/null) + [ -z "$out" ] || die 2 "refused: core.hooksPath is set ($(printf '%s' "$out" | tr '\n\t' '; ')); unset it first" + g cat-file -e "$H:$HOOK_REL" 2>/dev/null || die 2 "refused: HEAD has no $HOOK_REL" + if [ -L "$HOOK" ]; then die 2 "refused: .git/hooks/pre-commit is a symlink; remove it by hand if it is not wanted"; fi + if [ -e "$HOOK" ]; then + [ -f "$HOOK" ] || die 2 "refused: .git/hooks/pre-commit is not a regular file" + g cat-file blob "$H:$HOOK_REL" | cmp -s - "$HOOK" || die 2 "refused: a different pre-commit hook exists at .git/hooks/pre-commit" + chmod 0755 "$HOOK" || die 1 "chmod failed" + note="the same bytes were already there" + else + mkdir -p "$ROOT/.git/hooks" || die 1 "cannot create .git/hooks" + tmp="$ROOT/.git/hooks/.pre-commit.queue-commit.$$" + g cat-file blob "$H:$HOOK_REL" > "$tmp" && chmod 0755 "$tmp" || { rm -f -- "$tmp"; die 1 "cannot write .git/hooks/pre-commit"; } + # link() never replaces, so a hook that appeared meanwhile is kept. + if ! ln -- "$tmp" "$HOOK" 2>/dev/null; then rm -f -- "$tmp"; die 2 "refused: a pre-commit hook appeared while installing"; fi + rm -f -- "$tmp" + note="copied from HEAD" + fi + guard_check "$H" install + printf 'ok installed the queue guard at .git/hooks/pre-commit (%s, blob %s, mode 0755); canary passed\n' \ + "$note" "$(g rev-parse "$H:$HOOK_REL")" + exit 0 +fi + +# --- 1. guard. H is recorded first, before the canary, so a branch that moves +# at any later point makes update-ref in step 7 fail. --- +H=$(g rev-parse --verify -q HEAD) || die 2 "refused: HEAD has no commit" +BRANCH=$(g symbolic-ref -q --short HEAD) || die 2 "refused: HEAD is detached" +guard_check "$H" step1 +g diff-index --cached --quiet "$H" -- "$QJSON" "$QMD" \ + || die 2 "refused: the shared index has staged changes to $QJSON or $QMD; seats never stage them; run: $FIX" +if g cat-file -e "$H:$QJSON" 2>/dev/null; then + [ "$GENESIS" = 0 ] || die 2 "refused: --genesis, but HEAD already has $QJSON" +else + [ "$GENESIS" = 1 ] || die 2 "refused: HEAD has no $QJSON; the first queue commit needs --genesis" +fi + +# --- 2. snapshot, under the queue lock, with this checkout's code --- +SNAP="$TMPD/snapshot" +mkdir "$SNAP" || die 1 "mkdir failed" +snapline=$(cd "$ROOT" && node "$ROOT/packages/queue/src/cli.mjs" snapshot --out "$SNAP") || die 2 "refused: queue snapshot failed (see above)" +say "$snapline" +# shellcheck disable=SC2016 +info=$(node -e ' + const d = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); + const g = d.log[0]; + process.stdout.write([g.args.branch, g.args.root, g.args.map, g.result.mapBlob].join("\n")); +' "$SNAP/queue.json") || die 2 "refused: cannot read the snapshot's genesis entry" +{ read -r gbranch; read -r groot; read -r gmap; read -r gblob; } <<<"$info" +[ "$gbranch" = "$BRANCH" ] || die 2 "refused: HEAD is on $BRANCH, but the queue's genesis branch is $gbranch" +[ "$groot" = "$ROOT" ] || die 2 "refused: the queue's canonical root is $groot, not $ROOT" +if [ "$GENESIS" = 1 ]; then + hblob=$(g rev-parse -q --verify "$H:$gmap" 2>/dev/null) || die 2 "refused: the migration map $gmap is not in HEAD" + [ "$hblob" = "$gblob" ] || die 2 "refused: genesis read map blob $gblob, but HEAD's $gmap is $hblob" +fi + +# --- 3. base, from the object database --- +TREE=$(g rev-parse "$H^{tree}") || die 1 "cannot read HEAD's tree" +printf '%s %s\n' "$H" "$TREE" > "$SNAP/base.id" +if [ "$GENESIS" = 1 ]; then + BASEARG=(--base-absent) +else + g cat-file blob "$H:$QJSON" > "$SNAP/base.json" || die 1 "cannot read the base $QJSON" + BASEARG=(--base-file "$SNAP/base.json") +fi + +# --- 4. verify with HEAD's code, outside any repository --- +ARCH="$TMPD/archive/tree" +mkdir -p "$ARCH" || die 1 "mkdir failed" +g archive --format=tar "$H" | tar -x -C "$ARCH" || die 1 "cannot unpack HEAD's archive" +[ -f "$ARCH/packages/queue/src/cli.mjs" ] || die 2 "refused: HEAD has no packages/queue" +# NODE_TEST_CONTEXT is cleared: under a parent test runner, a nested +# `node --test` reports to that runner and exits 0 whatever its tests do. +if ! (cd "$ARCH" && GIT_CEILING_DIRECTORIES="$TMPD/archive" env -u NODE_TEST_CONTEXT node --test packages/queue/tests/ >"$TMPD/archive-tests.log" 2>&1); then + tail -n 40 "$TMPD/archive-tests.log" >&2 + die 2 "refused: HEAD's queue tests failed in the archive" +fi +(cd "$ARCH" && GIT_CEILING_DIRECTORIES="$TMPD/archive" node packages/queue/src/cli.mjs verify --snapshot "$SNAP" "${BASEARG[@]}") >&2 \ + || die 2 "refused: HEAD's validator rejected the snapshot" + +# --- 5. blobs --- +B1=$(g hash-object -w --no-filters "$SNAP/queue.json") || die 1 "hash-object failed" +B2=$(g hash-object -w --no-filters "$SNAP/QUEUE.md") || die 1 "hash-object failed" + +# --- 6. tree, in a temporary index at a path that doesn't exist yet --- +TIDX="$TMPD/tree/index" +mkdir -p "$TMPD/tree" +GIT_INDEX_FILE=$TIDX g read-tree "$H" || die 1 "read-tree failed" +GIT_INDEX_FILE=$TIDX g update-index --add --cacheinfo "100644,$B1,$QJSON" --cacheinfo "100644,$B2,$QMD" \ + || die 1 "update-index failed" +T=$(GIT_INDEX_FILE=$TIDX g write-tree) || die 1 "write-tree failed" +changed=$(g diff-tree -r --name-only "$H" "$T") || die 1 "diff-tree failed" +[ -n "$changed" ] || die 2 "refused: the snapshot equals HEAD's queue files; nothing to commit" +while IFS= read -r p; do + [ "$p" = "$QJSON" ] || [ "$p" = "$QMD" ] || die 1 "the new tree changes $p, not only the queue files" +done <<<"$changed" + +# --- 7. commit, recheck the guard, publish --- +printf '%s\n' "$MSG" > "$TMPD/msg" +C=$(g commit-tree "$T" -p "$H" -F "$TMPD/msg") || die 1 "commit-tree failed" +guard_check "$H" step7 +if ! g update-ref -m queue-commit "refs/heads/$BRANCH" "$C" "$H"; then + die 1 "refs/heads/$BRANCH moved since $H; nothing published; start again" +fi +say "committed $C on $BRANCH (parent $H)" + +# --- 8. reconcile the shared index --- +unreconciled() { say "committed $C, but the shared index was not reconciled: $*"; say "the queue guard refuses ordinary commits until this runs: $FIX"; exit 3; } +head_now=$(g rev-parse HEAD) +[ "$head_now" = "$C" ] || unreconciled "HEAD is $head_now, not $C; the index was not touched" +want=$(g ls-tree --format='%(objectmode) %(objectname) 0 %(path)' "$H" -- "$QJSON" "$QMD") || unreconciled "cannot read HEAD's entries" +have=$(g ls-files --format='%(objectmode) %(objectname) %(stage) %(path)' -- "$QJSON" "$QMD") || unreconciled "cannot read the index" +[ "$have" = "$want" ] || unreconciled "the index's queue entries differ from $H's, so someone staged a queue path; the index was not touched" +[ ! -e "$ROOT/.git/index.lock" ] || unreconciled "another git process holds .git/index.lock" +g reset -q -- "$QJSON" "$QMD" || unreconciled "git reset failed" +printf 'ok committed %s: %s\n' "$C" "$snapline" +exit 0 diff --git a/scripts/test-queue.sh b/scripts/test-queue.sh new file mode 100755 index 00000000..eb11e98c --- /dev/null +++ b/scripts/test-queue.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Suite for packages/queue (issue #1508). Every test runs in scratch +# repositories under the system temp directory; nothing touches this +# checkout's .git, lock, witness or queue files. +# +# scripts/test-queue.sh full run +# NO_COLOR=1 scripts/test-queue.sh plain output +# +# Once HEAD holds docs/plans/queue.json (the genesis commit), the suite also +# runs `verify` on the live queue, which takes the queue lock for a moment +# and writes nothing. +set -uo pipefail +cd "$(dirname "$0")/.." + +SANDBOX="$(mktemp -d)" +trap 'rm -rf "$SANDBOX"' EXIT +PASS=0 +FAIL=0 +if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then + C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m' +else + C_OK=""; C_FAIL=""; C_RESET="" +fi +check() { + if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi +} + +echo "toolchain: node $(node --version), $(git --version)" +echo + +# --- syntax --- +for f in packages/queue/src/*.mjs packages/queue/tests/*.mjs packages/queue/tests/fixtures/*.mjs scripts/queue-commit.sh scripts/git-hooks/pre-commit; do + case "$f" in + *.sh) bash -n "$f" >/dev/null 2>&1 ;; + */pre-commit) sh -n "$f" >/dev/null 2>&1 ;; + *) node --check "$f" >/dev/null 2>&1 ;; + esac + check "syntax: $f" $? +done +[ -x scripts/queue-commit.sh ] && [ -x scripts/git-hooks/pre-commit ] +check "queue-commit.sh and the guard are executable" $? +node -e 'const p=require("./packages/queue/package.json"); process.exit(p.dependencies||p.devDependencies?1:0)' >/dev/null 2>&1 +check "packages/queue declares no dependencies" $? + +# --- the package tests --- +env -u NODE_TEST_CONTEXT node --test --test-reporter=spec packages/queue/tests/ > "$SANDBOX/tests.log" 2>&1 +rc=$? +grep -E '^ℹ (tests|pass|fail) ' "$SANDBOX/tests.log" +if [ "$rc" != 0 ]; then grep -E '^✖|Error' "$SANDBOX/tests.log" | head -40; fi +check "node --test packages/queue/tests/" "$rc" + +# --- the live queue, once genesis is committed --- +if git cat-file -e HEAD:docs/plans/queue.json 2>/dev/null; then + node packages/queue/src/cli.mjs verify + check "queue verify (live queue)" $? +else + echo "skip queue verify: HEAD has no docs/plans/queue.json (before the genesis commit)" +fi + +echo +echo "queue suite: $PASS passed, $FAIL failed" +[ "$FAIL" = 0 ]