git mv packages/business -> packages/organization (@mosaic/organization). The old package stays as one-line re-export shims of each moved module, with src/business.mjs re-exporting src/organization-config.mjs and src/cli.mjs forwarding to the moved executable. BusinessError is OrganizationConfigError (one constructor, name kept BusinessError); OrganizationError stays distinct. The bus adapter gains organization-context.mjs (busOrganization), with business.mjs (busBusiness) as the same function. Import consumers, manifests, the lockfile, scripts/mosaic, stack-copy.sh COPY_PATHS (both roots), seat sandbox RUNNER_FILES and the copy closure walker (organization-imports.mjs, both roots required) move to the new path. v1 serialization, v1 writers and the stage A fixtures are unchanged; the stage A contract tests stay where they were and pass as before.
206 lines
11 KiB
JavaScript
206 lines
11 KiB
JavaScript
// Row 270 (264c, #1775, T1 item 7 D6): the count of completed compactions over
|
|
// one thread session's lifetime, across its runs and native branches.
|
|
//
|
|
// The host collects it from the session's native store and appends what it
|
|
// found to a ledger, <store dir>/compactions.jsonl (0600, append-only). The
|
|
// store dir holds the mounted session/ and claude-projects/ but is not mounted
|
|
// itself (sandbox.mjs binds()), so the agent can't reach the ledger. The
|
|
// launcher is its only writer, and collects while it holds the session's
|
|
// workspace lock (session.mjs lockSessionWorkspace); each collection is one
|
|
// O_APPEND write. The count is rebuilt from the ledger, never from a page.
|
|
//
|
|
// What a compaction is depends on the harness, so core never reads a native
|
|
// stream itself (row 307, #1814, Vikunja 598). Each collection calls the route's
|
|
// harness module's `compactions` (packages/plugins/README.md has the contract),
|
|
// checks its answer, and writes the ledger lines from it. A compaction's key is
|
|
// (entry, completedAt) over the whole store, and the stream it was first seen in
|
|
// is recorded beside it. A harness with no pinned compaction format answers
|
|
// harness-unsupported, and so does one that can't count the store's history: a
|
|
// session any such collection ran for is unknown for good, never 0.
|
|
//
|
|
// One collection reads at most a budget of stream bytes (row 270 r2, B1).
|
|
// The budget is the system config's compactionReadBytes, which only a human
|
|
// writes (mos-1352, DECISIONS Q222); readBudget() takes the default for a
|
|
// missing or bad value. A collection the budget stops short is unknown with
|
|
// not-collected, and its cursors stop at the last whole line read, so a later
|
|
// collection reads on from there.
|
|
//
|
|
// One collection opens at most a cap of stream files, in name order (row 270
|
|
// r3, B4). The cap is the system config's compactionStreams, human-only too
|
|
// (sage-3038); readStreams() takes the default for a missing or bad value.
|
|
// Past the cap a collection is not-collected, and the streams it didn't open
|
|
// get no cursor line, so neither a collection's work nor the ledger lines it
|
|
// writes grow with the number of files the agent makes.
|
|
//
|
|
// Ledger lines, each one JSON object with v 1:
|
|
// { kind: "compaction", entry, completedAt, stream, at } once per key
|
|
// { kind: "cursor", stream, bytes, tail, at } read through `bytes`; `tail`
|
|
// is the sha256 of the up to 4096 bytes that end there
|
|
// { kind: "collected", at, known, reason, harness, version }
|
|
// A cursor lets the next collection read only what was appended. A stream
|
|
// that shrank, changed under its cursor or went away is store-incomplete from
|
|
// then on: entries may be lost, so the count stays a lower bound for good.
|
|
|
|
import { closeSync, constants as fsc, fstatSync, lstatSync, openSync, readFileSync, writeSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { ID_PATTERN } from "../../organization/src/vocabulary.mjs";
|
|
|
|
export const LEDGER = "compactions.jsonl";
|
|
const CHUNK = 1 << 20;
|
|
// BUILD.md (queue-270, round 2) has the measured stream sizes behind the default.
|
|
export const READ_BYTES = 64 * CHUNK;
|
|
const READ_MIN = CHUNK, READ_MAX = 1024 * CHUNK;
|
|
// BUILD.md (queue-270, round 3) has the measured stream counts behind the default.
|
|
export const STREAMS = 1024;
|
|
const STREAMS_MIN = 32, STREAMS_MAX = 65536;
|
|
|
|
const keyOf = (entry, completedAt) => JSON.stringify([entry, completedAt]);
|
|
const segment = (x) => typeof x === "string" && ID_PATTERN.test(x) && !x.includes("..");
|
|
|
|
// The store directory session.mjs sessionStore() makes, as a path only: the
|
|
// reader never creates it. Null for an id that isn't one.
|
|
export function storeDir(dataRoot, business, thread, session, { legacy = false } = {}) {
|
|
if (![business, thread, session].every(segment)) return null;
|
|
const dir = join(dataRoot, "threads", business, thread);
|
|
return legacy ? dir : join(dir, "sessions", session);
|
|
}
|
|
|
|
// The budget from the system config's compactionReadBytes: a whole number of
|
|
// bytes from 1 MiB to 1 GiB, or the default for anything else, missing included.
|
|
export function readBudget(value) {
|
|
return Number.isSafeInteger(value) && value >= READ_MIN && value <= READ_MAX ? value : READ_BYTES;
|
|
}
|
|
|
|
// The stream cap from the system config's compactionStreams: a whole number
|
|
// from 32 to 65536, or the default for anything else, missing included.
|
|
export function readStreams(value) {
|
|
return Number.isSafeInteger(value) && value >= STREAMS_MIN && value <= STREAMS_MAX ? value : STREAMS;
|
|
}
|
|
|
|
// The ledger as recorded. Throws on a ledger that can't be read. `bad` is set
|
|
// by any line that isn't a v 1 record, a torn last line included.
|
|
export function readLedger(dir) {
|
|
const events = new Map(), cursors = {};
|
|
let last = null, incomplete = false, unsupported = false, bad = false, torn = false, text;
|
|
const file = join(dir, LEDGER);
|
|
try {
|
|
if (!lstatSync(file).isFile()) throw Object.assign(new Error(`${file} isn't a regular file`), { code: "ledger-unsafe" });
|
|
text = readFileSync(file, { encoding: "utf8", flag: fsc.O_RDONLY | fsc.O_NOFOLLOW });
|
|
} catch (e) {
|
|
if (e.code === "ENOENT") return null;
|
|
throw e;
|
|
}
|
|
const lines = text.split("\n");
|
|
if (lines.pop() !== "") bad = torn = true;
|
|
for (const line of lines) {
|
|
let r;
|
|
try {
|
|
r = JSON.parse(line);
|
|
} catch {
|
|
r = null;
|
|
}
|
|
if (!r || typeof r !== "object" || r.v !== 1) {
|
|
bad = true;
|
|
continue;
|
|
}
|
|
if (r.kind === "compaction") {
|
|
const key = keyOf(r.entry, r.completedAt);
|
|
if (!events.has(key)) events.set(key, r);
|
|
} else if (r.kind === "cursor") cursors[r.stream] = { bytes: r.bytes, tail: r.tail };
|
|
else if (r.kind === "collected") {
|
|
last = r;
|
|
if (r.reason === "store-incomplete") incomplete = true;
|
|
if (r.reason === "harness-unsupported") unsupported = true;
|
|
} else bad = true;
|
|
}
|
|
return { events, cursors, last, incomplete, unsupported, bad, torn };
|
|
}
|
|
|
|
// What session.list shows: { count, known, reason, at }. Known only when the
|
|
// ledger is whole, no collection was harness-unsupported and none lost
|
|
// entries. Any other answer is unknown with a reason and a lower bound, or null
|
|
// for none: never 0.
|
|
export function compactionCount(dir) {
|
|
const unknown = (reason, count = null, at = null) => ({ count: count || null, known: false, reason, at });
|
|
let ledger;
|
|
try {
|
|
ledger = readLedger(dir);
|
|
} catch {
|
|
return unknown("store-unreadable");
|
|
}
|
|
if (!ledger) return unknown("not-collected");
|
|
const count = ledger.events.size;
|
|
const at = ledger.last?.at ?? null;
|
|
if (ledger.bad) return unknown("store-incomplete", count, at);
|
|
if (!ledger.last) return unknown("not-collected", count);
|
|
if (ledger.unsupported) return unknown("harness-unsupported", count, at);
|
|
if (ledger.incomplete) return unknown("store-incomplete", count, at);
|
|
if (!ledger.last.known) return unknown(ledger.last.reason, count, at);
|
|
return { count, known: true, reason: null, at };
|
|
}
|
|
|
|
const REASONS = ["store-incomplete", "store-unreadable", "not-collected", "harness-unsupported"];
|
|
const HEX = /^[0-9a-f]{64}$/;
|
|
|
|
// The harness module's answer, checked before anything is written: what the
|
|
// export's contract allows and nothing else. Returns what's wrong, or null.
|
|
function checkAnswer(a) {
|
|
const plain = (x) => x !== null && typeof x === "object" && !Array.isArray(x);
|
|
if (!plain(a)) return "not an object";
|
|
if (!Array.isArray(a.found)) return "found isn't a list";
|
|
for (const f of a.found) {
|
|
if (!plain(f) || typeof f.entry !== "string" || !f.entry) return "a found entry has no entry id";
|
|
if (f.completedAt !== null && typeof f.completedAt !== "string") return `found ${f.entry} has a completedAt that isn't a string or null`;
|
|
if (typeof f.stream !== "string" || !f.stream) return `found ${f.entry} has no stream`;
|
|
}
|
|
if (!plain(a.cursors)) return "cursors isn't an object";
|
|
for (const [stream, c] of Object.entries(a.cursors)) {
|
|
if (!plain(c) || !Number.isSafeInteger(c.bytes) || c.bytes < 0 || typeof c.tail !== "string" || !HEX.test(c.tail)) return `cursor ${JSON.stringify(stream)} isn't { bytes, tail }`;
|
|
}
|
|
if (a.reason !== null && !REASONS.includes(a.reason)) return `reason ${JSON.stringify(a.reason)} isn't one of ${REASONS.join(", ")}`;
|
|
if (a.known !== (a.reason === null)) return "known must be true exactly when reason is null";
|
|
if (a.version !== null && typeof a.version !== "string") return "version isn't a string or null";
|
|
return null;
|
|
}
|
|
|
|
// One collection for a session's store ({ dir, session, projects } from
|
|
// sessionStore), at run start (resume) and run end, under the session's
|
|
// workspace lock. `harness` is { name, compactions }: the route's harness name
|
|
// and its loaded module's export. Appends the new compactions, the moved
|
|
// cursors and one collected line in a single O_APPEND write, then answers the
|
|
// count. A key already in the ledger is never written again, and nothing
|
|
// rewrites the file. A module that throws or answers out of contract throws
|
|
// here, before anything is written.
|
|
export function collectCompactions(store, harness, { now = () => new Date().toISOString(), budget = READ_BYTES, streams = STREAMS } = {}) {
|
|
const prior = readLedger(store.dir);
|
|
const ledger = prior ?? { events: new Map(), cursors: {}, torn: false };
|
|
const at = now();
|
|
const answer = harness.compactions({ store: { session: store.session, projects: store.projects }, cursors: { ...ledger.cursors }, budget, streams, first: prior === null });
|
|
const wrong = checkAnswer(answer);
|
|
if (wrong) throw new Error(`harness ${harness.name} compactions: bad answer: ${wrong}`);
|
|
const out = [];
|
|
for (const f of answer.found) {
|
|
const key = keyOf(f.entry, f.completedAt);
|
|
if (ledger.events.has(key)) continue;
|
|
ledger.events.set(key, f);
|
|
out.push({ v: 1, kind: "compaction", entry: f.entry, completedAt: f.completedAt, stream: f.stream, at });
|
|
}
|
|
for (const [stream, c] of Object.entries(answer.cursors)) {
|
|
const old = ledger.cursors[stream];
|
|
if (!old || old.bytes !== c.bytes || old.tail !== c.tail) out.push({ v: 1, kind: "cursor", stream, bytes: c.bytes, tail: c.tail, at });
|
|
}
|
|
out.push({ v: 1, kind: "collected", at, known: answer.known, reason: answer.reason, harness: harness.name, version: answer.version });
|
|
const fd = openSync(join(store.dir, LEDGER), fsc.O_WRONLY | fsc.O_APPEND | fsc.O_CREAT | fsc.O_NOFOLLOW, 0o600);
|
|
try {
|
|
if (!fstatSync(fd).isFile()) throw Object.assign(new Error(`${join(store.dir, LEDGER)} isn't a regular file`), { code: "ledger-unsafe" });
|
|
// After a torn line, start on a fresh one: the torn line stays bad and
|
|
// keeps the count unknown, and the new lines stay whole.
|
|
const buf = Buffer.from((ledger.torn ? "\n" : "") + out.map((r) => JSON.stringify(r)).join("\n") + "\n");
|
|
let off = 0;
|
|
while (off < buf.length) off += writeSync(fd, buf, off, buf.length - off);
|
|
} finally {
|
|
closeSync(fd);
|
|
}
|
|
return compactionCount(store.dir);
|
|
}
|