Each turn's usage goes to <dataRoot>/triage/usage.jsonl, one line per usage.end, 0600 and owned by the running user; a link, wrong mode or owner, or a non-file refuses. The host holds the stored cap through daily-quota mayLaunch and report before each start and each queued message, lets a crossing turn finish, drains the queue with a host line, sums the file again after a restart (a torn line counts as an unknown turn), and GET /api/triage adds usageToday, cap and resetsAt. With no route stored the fields say "not set". cli.mjs keeps route null. (#1873, lane triage-cap)
221 lines
11 KiB
JavaScript
221 lines
11 KiB
JavaScript
// #1873 triage-cap: Triage's token usage and its daily cap.
|
|
//
|
|
// The Triage host appends one line per turn to <dataRoot>/triage/usage.jsonl:
|
|
// { at, run, turn, status, input, output, cacheRead, cacheWrite }
|
|
// A known turn carries four non-negative safe integers; an unknown turn carries four nulls. The file
|
|
// sits outside the run directory, so a new start never clears it. It is held to the route store's
|
|
// file checks: a regular file, one link, the running user's, mode 0600. A link, a non-file, a wrong
|
|
// mode or owner refuses the read and the append alike, and the append leaves the file as it was.
|
|
// Lines are only ever appended. Nothing here prunes the file.
|
|
//
|
|
// The cap is the daily-quota plugin's own rule (packages/plugins/limit-check/daily-quota), called
|
|
// with the organization limits shape and the stored cap as its one leaf, for one account named
|
|
// "triage". No counting rule lives here: the count, the window, the DST handling and the hit all
|
|
// come from mayLaunch and report. nextReset only asks report when the window turns over.
|
|
import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, openSync, readSync, writeSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { mayLaunch, report } from "../../plugins/limit-check/daily-quota/index.mjs";
|
|
|
|
export const USAGE_ACCOUNT = "triage";
|
|
export const USAGE_FIELDS = Object.freeze(["at", "run", "turn", "status", "input", "output", "cacheRead", "cacheWrite"]);
|
|
const KINDS = ["input", "output", "cacheWrite", "cacheRead"];
|
|
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
|
// A run id is the host's own triage-<14 digits>; a turn id is the runner's message id, maybe with
|
|
// ":retry". Both are held to one short, plain shape so a line never carries free text.
|
|
export const ID = /^[A-Za-z0-9:._-]{1,80}$/;
|
|
// 32 MiB holds well over a hundred thousand lines; a larger file refuses rather than stalls a read.
|
|
export const USAGE_MAX_BYTES = 32 * 1024 * 1024;
|
|
const HOUR = 3600000;
|
|
|
|
export class UsageError extends Error {
|
|
constructor(why) {
|
|
super(`the Triage usage file is unreadable: ${why}`);
|
|
this.name = "UsageError";
|
|
this.code = "triage-usage-unreadable";
|
|
}
|
|
}
|
|
const unreadable = (why) => new UsageError(why);
|
|
|
|
export const usagePath = (dataRoot) => join(dataRoot, "triage", "usage.jsonl");
|
|
|
|
const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
|
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
|
const count = (n) => Number.isSafeInteger(n) && n >= 0;
|
|
|
|
// One line's shape, exactly.
|
|
export function validLine(v) {
|
|
if (!plain(v) || Object.keys(v).length !== USAGE_FIELDS.length || !USAGE_FIELDS.every((k) => Object.hasOwn(v, k))) return false;
|
|
if (!canonicalTime(v.at) || typeof v.run !== "string" || !ID.test(v.run) || typeof v.turn !== "string" || !ID.test(v.turn)) return false;
|
|
if (v.status === "known") return KINDS.every((k) => count(v[k]));
|
|
if (v.status === "unknown") return KINDS.every((k) => v[k] === null);
|
|
return false;
|
|
}
|
|
|
|
// An opened file's checks, as the route store holds route.json to.
|
|
function checkFile(fd, uid) {
|
|
const st = fstatSync(fd);
|
|
if (!st.isFile()) throw unreadable("usage.jsonl is not a regular file");
|
|
if (st.nlink !== 1) throw unreadable("usage.jsonl has more than one link");
|
|
if (st.uid !== uid) throw unreadable("usage.jsonl is not owned by this user");
|
|
if ((st.mode & 0o777) !== 0o600) throw unreadable("usage.jsonl mode is not 0600");
|
|
return st;
|
|
}
|
|
|
|
const uidOf = (opts) => opts.uid ?? process.getuid();
|
|
|
|
// Append one line. A torn last line (a write that died) is closed off with a newline first, so the
|
|
// new line always stands alone and the torn one stays one bad line.
|
|
export function appendUsage(file, line, opts = {}) {
|
|
if (!validLine(line)) throw unreadable("the line does not match the usage shape");
|
|
const text = JSON.stringify(Object.fromEntries(USAGE_FIELDS.map((k) => [k, line[k]]))) + "\n";
|
|
const flags = constants.O_RDWR | constants.O_APPEND | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
|
let fd;
|
|
try { fd = openSync(file, flags); }
|
|
catch (e) {
|
|
if (e.code === "ELOOP") throw unreadable("usage.jsonl is a link");
|
|
if (e.code !== "ENOENT") throw unreadable("usage.jsonl cannot be opened");
|
|
try { fd = openSync(file, flags | constants.O_CREAT | constants.O_EXCL, 0o600); }
|
|
catch { throw unreadable("usage.jsonl cannot be made"); }
|
|
// The umask may have narrowed the mode; set it on the descriptor just made.
|
|
try { fchmodSync(fd, 0o600); } catch { closeSync(fd); throw unreadable("usage.jsonl cannot be made"); }
|
|
}
|
|
try {
|
|
const st = checkFile(fd, uidOf(opts));
|
|
let lead = "";
|
|
if (st.size > 0) {
|
|
const last = Buffer.alloc(1);
|
|
if (readSync(fd, last, 0, 1, st.size - 1) !== 1) throw unreadable("usage.jsonl cannot be read");
|
|
if (last[0] !== 0x0a) lead = "\n";
|
|
}
|
|
try {
|
|
writeSync(fd, lead + text);
|
|
fsyncSync(fd);
|
|
} catch { throw unreadable("usage.jsonl cannot be written"); }
|
|
} finally { closeSync(fd); }
|
|
}
|
|
|
|
const nulls = { input: null, output: null, cacheRead: null, cacheWrite: null };
|
|
|
|
// Every line as a daily-quota record for the triage account. A line that does not parse or does
|
|
// not match the shape (a torn last line, an edit by hand) is refused: it becomes one unknown turn,
|
|
// keyed by its line number, dated by the next good line (or the file's change time when none
|
|
// follows), and is listed in `bad` so the host can log it. No file reads as no turns.
|
|
export function readUsage(file, opts = {}) {
|
|
const maxBytes = opts.maxBytes ?? USAGE_MAX_BYTES;
|
|
let fd;
|
|
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
|
catch (e) {
|
|
if (e.code === "ENOENT") return { records: [], bad: [] };
|
|
if (e.code === "ELOOP") throw unreadable("usage.jsonl is a link");
|
|
throw unreadable("usage.jsonl cannot be opened");
|
|
}
|
|
let st;
|
|
let buf;
|
|
try {
|
|
st = checkFile(fd, uidOf(opts));
|
|
if (st.size > maxBytes) throw unreadable("usage.jsonl is too large");
|
|
buf = Buffer.alloc(st.size);
|
|
let got = 0;
|
|
while (got < st.size) {
|
|
const n = readSync(fd, buf, got, st.size - got, got);
|
|
if (n === 0) break;
|
|
got += n;
|
|
}
|
|
buf = buf.subarray(0, got);
|
|
} finally { closeSync(fd); }
|
|
const lines = buf.toString("utf8").split("\n");
|
|
if (lines.at(-1) === "") lines.pop();
|
|
const parsed = lines.map((text) => {
|
|
try {
|
|
const v = JSON.parse(text);
|
|
return validLine(v) ? v : null;
|
|
} catch { return null; }
|
|
});
|
|
const records = new Array(parsed.length);
|
|
const bad = [];
|
|
let nextAt = st.mtime.toISOString();
|
|
for (let i = parsed.length - 1; i >= 0; i--) {
|
|
const v = parsed[i];
|
|
if (v) {
|
|
records[i] = { account: USAGE_ACCOUNT, ...v };
|
|
nextAt = v.at;
|
|
} else {
|
|
records[i] = { account: USAGE_ACCOUNT, at: nextAt, run: null, turn: `line-${i + 1}`, status: "unknown", ...nulls };
|
|
bad.unshift({ line: i + 1 });
|
|
}
|
|
}
|
|
return { records, bad, ino: st.ino };
|
|
}
|
|
|
|
// The organization limits shape with the stored cap as its one leaf.
|
|
export const limitsFor = (cap) => ({ tokens: { daily: { [USAGE_ACCOUNT]: cap } } });
|
|
const SUBJECT = Object.freeze({ account: USAGE_ACCOUNT });
|
|
const iso = (ms) => new Date(ms).toISOString();
|
|
|
|
// The next reset after nowMs, or after the cap's `since` when that is later. A probe record at that
|
|
// instant counts while the window holding t still starts at or before it; the reset is the first
|
|
// millisecond at which report stops counting it. A day is at most 25 hours, so 49 hours always
|
|
// holds the next one.
|
|
export function nextReset(cap, nowMs) {
|
|
const from = Math.max(nowMs, Date.parse(cap.since));
|
|
const limits = limitsFor(cap);
|
|
const probe = [{ account: USAGE_ACCOUNT, at: iso(from), run: "probe", turn: "probe", status: "known", input: 1, output: 0, cacheRead: 0, cacheWrite: 0 }];
|
|
const counts = (t) => report({ limits, records: probe, now: iso(t) })[0].count > 0;
|
|
let lo = from;
|
|
let hi = from + 49 * HOUR;
|
|
if (counts(hi)) throw unreadable("the cap has no reset in the next two days");
|
|
while (hi - lo > 1) {
|
|
const mid = Math.floor((lo + hi) / 2);
|
|
if (counts(mid)) lo = mid; else hi = mid;
|
|
}
|
|
return hi;
|
|
}
|
|
|
|
// Today's count, the decision and the next reset for the stored cap.
|
|
export function capState({ cap, records, now }) {
|
|
const at = iso(now);
|
|
const limits = limitsFor(cap);
|
|
const decision = mayLaunch({ limits, subject: SUBJECT, records, now: at });
|
|
const [r] = report({ limits, records, now: at });
|
|
return {
|
|
allow: decision.allow,
|
|
hit: decision.allow ? null : decision.hit,
|
|
usageToday: { count: r.count, ...r.kinds, unknownTurns: r.unknownTurns },
|
|
resetsAt: iso(nextReset(cap, now)),
|
|
};
|
|
}
|
|
|
|
// The turn delta, copied from packages/bus/src/usage.mjs (snapshot, delta; not exported there and
|
|
// that package is not this lane's to change). Same codes, same rules: the turn's usage is the sum
|
|
// of the transcript entries that are new since the turn began; a changed or missing earlier entry
|
|
// is "transcript-rewritten", and no new entry is "no-new-usage".
|
|
const unknown = (code) => ({ status: "unknown", code });
|
|
export function snapshot(value) {
|
|
if (value?.status === "unknown") return unknown(/^transcript-(path|size|line|entries|records|changed|parse|usage|harness)$/.test(value.code) ? value.code : "source-unknown");
|
|
if (value?.status !== "known" || !value.entries || typeof value.entries !== "object" || Array.isArray(value.entries)) return unknown("source-invalid");
|
|
const entries = Object.entries(value.entries);
|
|
if (entries.length > 20000) return unknown("source-invalid");
|
|
for (const [id, v] of entries) {
|
|
if (id.length > 512 || !v || Object.keys(v).length !== 4 || KINDS.some((k) => !Number.isSafeInteger(v[k]) || v[k] < 0)) return unknown("source-invalid");
|
|
}
|
|
return { status: "known", entries: value.entries };
|
|
}
|
|
export function delta(before, after) {
|
|
if (!before || before.status !== "known") return unknown("baseline-unknown");
|
|
if (after.status !== "known") return after;
|
|
for (const [id, v] of Object.entries(before.entries)) {
|
|
if (!Object.hasOwn(after.entries, id) || KINDS.some((k) => after.entries[id][k] !== v[k])) return unknown("transcript-rewritten");
|
|
}
|
|
const sum = { input: 0, output: 0, cacheWrite: 0, cacheRead: 0 };
|
|
let added = false;
|
|
for (const [id, v] of Object.entries(after.entries)) {
|
|
if (Object.hasOwn(before.entries, id)) continue;
|
|
added = true;
|
|
for (const k of KINDS) {
|
|
sum[k] += v[k];
|
|
if (!Number.isSafeInteger(sum[k])) return unknown("usage-overflow");
|
|
}
|
|
}
|
|
return added ? { status: "known", ...sum } : unknown("no-new-usage");
|
|
}
|