// Row 270 (264c, #1775, T1 item 7 D6): the count of completed compactions over // one thread session's lifetime, across its runs and native branches. // // The host collects it from the session's native store and appends what it // found to a ledger, /compactions.jsonl (0600, append-only). The // store dir holds the mounted session/ and claude-projects/ but is not mounted // itself (sandbox.mjs binds()), so the agent can't reach the ledger. The // launcher is its only writer, and collects while it holds the session's // workspace lock (session.mjs lockSessionWorkspace); each collection is one // O_APPEND write. The count is rebuilt from the ledger, never from a page. // // What a compaction is depends on the harness, so core never reads a native // stream itself (row 307, #1814, Vikunja 598). Each collection calls the route's // harness module's `compactions` (packages/plugins/README.md has the contract), // checks its answer, and writes the ledger lines from it. A compaction's key is // (entry, completedAt) over the whole store, and the stream it was first seen in // is recorded beside it. A harness with no pinned compaction format answers // harness-unsupported, and so does one that can't count the store's history: a // session any such collection ran for is unknown for good, never 0. // // One collection reads at most a budget of stream bytes (row 270 r2, B1). // The budget is the system config's compactionReadBytes, which only a human // writes (mos-1352, DECISIONS Q222); readBudget() takes the default for a // missing or bad value. A collection the budget stops short is unknown with // not-collected, and its cursors stop at the last whole line read, so a later // collection reads on from there. // // One collection opens at most a cap of stream files, in name order (row 270 // r3, B4). The cap is the system config's compactionStreams, human-only too // (sage-3038); readStreams() takes the default for a missing or bad value. // Past the cap a collection is not-collected, and the streams it didn't open // get no cursor line, so neither a collection's work nor the ledger lines it // writes grow with the number of files the agent makes. // // Ledger lines, each one JSON object with v 1: // { kind: "compaction", entry, completedAt, stream, at } once per key // { kind: "cursor", stream, bytes, tail, at } read through `bytes`; `tail` // is the sha256 of the up to 4096 bytes that end there // { kind: "collected", at, known, reason, harness, version } // A cursor lets the next collection read only what was appended. A stream // that shrank, changed under its cursor or went away is store-incomplete from // then on: entries may be lost, so the count stays a lower bound for good. import { closeSync, constants as fsc, fstatSync, lstatSync, openSync, readFileSync, writeSync } from "node:fs"; import { join } from "node:path"; import { ID_PATTERN } from "../../organization/src/vocabulary.mjs"; export const LEDGER = "compactions.jsonl"; const CHUNK = 1 << 20; // BUILD.md (queue-270, round 2) has the measured stream sizes behind the default. export const READ_BYTES = 64 * CHUNK; const READ_MIN = CHUNK, READ_MAX = 1024 * CHUNK; // BUILD.md (queue-270, round 3) has the measured stream counts behind the default. export const STREAMS = 1024; const STREAMS_MIN = 32, STREAMS_MAX = 65536; const keyOf = (entry, completedAt) => JSON.stringify([entry, completedAt]); const segment = (x) => typeof x === "string" && ID_PATTERN.test(x) && !x.includes(".."); // The store directory session.mjs sessionStore() makes, as a path only: the // reader never creates it. Null for an id that isn't one. export function storeDir(dataRoot, business, thread, session, { legacy = false } = {}) { if (![business, thread, session].every(segment)) return null; const dir = join(dataRoot, "threads", business, thread); return legacy ? dir : join(dir, "sessions", session); } // The budget from the system config's compactionReadBytes: a whole number of // bytes from 1 MiB to 1 GiB, or the default for anything else, missing included. export function readBudget(value) { return Number.isSafeInteger(value) && value >= READ_MIN && value <= READ_MAX ? value : READ_BYTES; } // The stream cap from the system config's compactionStreams: a whole number // from 32 to 65536, or the default for anything else, missing included. export function readStreams(value) { return Number.isSafeInteger(value) && value >= STREAMS_MIN && value <= STREAMS_MAX ? value : STREAMS; } // The ledger as recorded. Throws on a ledger that can't be read. `bad` is set // by any line that isn't a v 1 record, a torn last line included. export function readLedger(dir) { const events = new Map(), cursors = {}; let last = null, incomplete = false, unsupported = false, bad = false, torn = false, text; const file = join(dir, LEDGER); try { if (!lstatSync(file).isFile()) throw Object.assign(new Error(`${file} isn't a regular file`), { code: "ledger-unsafe" }); text = readFileSync(file, { encoding: "utf8", flag: fsc.O_RDONLY | fsc.O_NOFOLLOW }); } catch (e) { if (e.code === "ENOENT") return null; throw e; } const lines = text.split("\n"); if (lines.pop() !== "") bad = torn = true; for (const line of lines) { let r; try { r = JSON.parse(line); } catch { r = null; } if (!r || typeof r !== "object" || r.v !== 1) { bad = true; continue; } if (r.kind === "compaction") { const key = keyOf(r.entry, r.completedAt); if (!events.has(key)) events.set(key, r); } else if (r.kind === "cursor") cursors[r.stream] = { bytes: r.bytes, tail: r.tail }; else if (r.kind === "collected") { last = r; if (r.reason === "store-incomplete") incomplete = true; if (r.reason === "harness-unsupported") unsupported = true; } else bad = true; } return { events, cursors, last, incomplete, unsupported, bad, torn }; } // What session.list shows: { count, known, reason, at }. Known only when the // ledger is whole, no collection was harness-unsupported and none lost // entries. Any other answer is unknown with a reason and a lower bound, or null // for none: never 0. export function compactionCount(dir) { const unknown = (reason, count = null, at = null) => ({ count: count || null, known: false, reason, at }); let ledger; try { ledger = readLedger(dir); } catch { return unknown("store-unreadable"); } if (!ledger) return unknown("not-collected"); const count = ledger.events.size; const at = ledger.last?.at ?? null; if (ledger.bad) return unknown("store-incomplete", count, at); if (!ledger.last) return unknown("not-collected", count); if (ledger.unsupported) return unknown("harness-unsupported", count, at); if (ledger.incomplete) return unknown("store-incomplete", count, at); if (!ledger.last.known) return unknown(ledger.last.reason, count, at); return { count, known: true, reason: null, at }; } const REASONS = ["store-incomplete", "store-unreadable", "not-collected", "harness-unsupported"]; const HEX = /^[0-9a-f]{64}$/; // The harness module's answer, checked before anything is written: what the // export's contract allows and nothing else. Returns what's wrong, or null. function checkAnswer(a) { const plain = (x) => x !== null && typeof x === "object" && !Array.isArray(x); if (!plain(a)) return "not an object"; if (!Array.isArray(a.found)) return "found isn't a list"; for (const f of a.found) { if (!plain(f) || typeof f.entry !== "string" || !f.entry) return "a found entry has no entry id"; if (f.completedAt !== null && typeof f.completedAt !== "string") return `found ${f.entry} has a completedAt that isn't a string or null`; if (typeof f.stream !== "string" || !f.stream) return `found ${f.entry} has no stream`; } if (!plain(a.cursors)) return "cursors isn't an object"; for (const [stream, c] of Object.entries(a.cursors)) { if (!plain(c) || !Number.isSafeInteger(c.bytes) || c.bytes < 0 || typeof c.tail !== "string" || !HEX.test(c.tail)) return `cursor ${JSON.stringify(stream)} isn't { bytes, tail }`; } if (a.reason !== null && !REASONS.includes(a.reason)) return `reason ${JSON.stringify(a.reason)} isn't one of ${REASONS.join(", ")}`; if (a.known !== (a.reason === null)) return "known must be true exactly when reason is null"; if (a.version !== null && typeof a.version !== "string") return "version isn't a string or null"; return null; } // One collection for a session's store ({ dir, session, projects } from // sessionStore), at run start (resume) and run end, under the session's // workspace lock. `harness` is { name, compactions }: the route's harness name // and its loaded module's export. Appends the new compactions, the moved // cursors and one collected line in a single O_APPEND write, then answers the // count. A key already in the ledger is never written again, and nothing // rewrites the file. A module that throws or answers out of contract throws // here, before anything is written. export function collectCompactions(store, harness, { now = () => new Date().toISOString(), budget = READ_BYTES, streams = STREAMS } = {}) { const prior = readLedger(store.dir); const ledger = prior ?? { events: new Map(), cursors: {}, torn: false }; const at = now(); const answer = harness.compactions({ store: { session: store.session, projects: store.projects }, cursors: { ...ledger.cursors }, budget, streams, first: prior === null }); const wrong = checkAnswer(answer); if (wrong) throw new Error(`harness ${harness.name} compactions: bad answer: ${wrong}`); const out = []; for (const f of answer.found) { const key = keyOf(f.entry, f.completedAt); if (ledger.events.has(key)) continue; ledger.events.set(key, f); out.push({ v: 1, kind: "compaction", entry: f.entry, completedAt: f.completedAt, stream: f.stream, at }); } for (const [stream, c] of Object.entries(answer.cursors)) { const old = ledger.cursors[stream]; if (!old || old.bytes !== c.bytes || old.tail !== c.tail) out.push({ v: 1, kind: "cursor", stream, bytes: c.bytes, tail: c.tail, at }); } out.push({ v: 1, kind: "collected", at, known: answer.known, reason: answer.reason, harness: harness.name, version: answer.version }); const fd = openSync(join(store.dir, LEDGER), fsc.O_WRONLY | fsc.O_APPEND | fsc.O_CREAT | fsc.O_NOFOLLOW, 0o600); try { if (!fstatSync(fd).isFile()) throw Object.assign(new Error(`${join(store.dir, LEDGER)} isn't a regular file`), { code: "ledger-unsafe" }); // After a torn line, start on a fresh one: the torn line stays bad and // keeps the count unknown, and the new lines stay whole. const buf = Buffer.from((ledger.torn ? "\n" : "") + out.map((r) => JSON.stringify(r)).join("\n") + "\n"); let off = 0; while (off < buf.length) off += writeSync(fd, buf, off, buf.length - off); } finally { closeSync(fd); } return compactionCount(store.dir); }