Evidence for the STATE report Sage checked and filed in 88f3253d9 (sage-1151). The skills-enabled
tar stays out of git; it is the W0b restore point under $WIN only. No credential or Environment
values: unit reads record ExecStart only, and the P3 credential files were fake placeholders.
236 lines
12 KiB
JavaScript
236 lines
12 KiB
JavaScript
// Merged window B+C, S1, S2 and D3 (repin-merged-bc.md "S1 and S2"): a read-only snapshot of every
|
|
// table in a bus store, so the 3h to 3i migration and a restore can be checked without the bus.
|
|
// OLD's broker doesn't take --tester on usage, names or threads.
|
|
//
|
|
// usage: node snapshot.mjs --store /abs/bus.sqlite (--held | --stopped) [--after STAMP] [--backup /abs/bus.sqlite]
|
|
// --held the running bus holds the store: a plain readOnly open, which adds no file to a WAL
|
|
// store that already has -wal and -shm (as stack-release.sh does for a held store).
|
|
// Refuses store-not-held when either is absent
|
|
// --stopped the bus is stopped: refuses store-unclean when -wal or -shm is present, else opens
|
|
// with immutable=1 and readOnly, which creates no file and leaves the store's mtime
|
|
// and sha256 alone
|
|
// --after also counts each timed table's rows with `at` after STAMP (mos-0838 D3, before a
|
|
// --restore). STAMP is a backup's STAMP file, in now_ms form (stack-release.sh:43,
|
|
// 2026-10-20T00:11:14.123Z). The count is a text compare, as the tool's own heldcount
|
|
// does, so every `at` in every timed table must have that form: a table with any
|
|
// other value refuses at-format and names the table (sage-1125)
|
|
// --backup also compares every table with a stack-release backup's bus.sqlite, which opens as
|
|
// --stopped opens a store: immutable=1 and readOnly, refusing backup-unclean when -wal
|
|
// or -shm is present, so nothing writes into the backup set
|
|
//
|
|
// It prints one JSON object: the schema version, then for every table in sqlite_master (type
|
|
// table, less sqlite_*) the row count, the max seq and a sha256 of the rows (in seq order, else in
|
|
// column order). `untimed` names the tables with no `at` column. For usage_records it adds per-run
|
|
// input and output sums, counted as `mosaic usage` counts them (usage.mjs turnRecords). messages
|
|
// and message_threads hold bodies and names, so they print counts only, no sha256 (sage-1125). It
|
|
// prints counts, hashes and table and meta key names only: no message body, no instance, project
|
|
// or thread name, no credential and no file content. It reads no config: the store path is always
|
|
// given.
|
|
//
|
|
// With --backup, each table gets `vsBackup` and `stopTables` names every table other than events
|
|
// whose rows differ from the backup's (sage-1123): a row with `at` after STAMP, a row past the
|
|
// backup's max seq, a changed row at or below it, a changed untimed table, or a table only one
|
|
// side has. meta counts only when a key other than schema_version and schema_digest changed (the
|
|
// 3h to 3i migration sets those two). Events are left to the tool's own --events N.
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { createHash } from 'node:crypto';
|
|
import { existsSync, lstatSync } from 'node:fs';
|
|
import { isAbsolute } from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
|
|
// usage_records is hashed on these columns in this order, never SELECT * (mos-0888 F1), as
|
|
// 3h-3i.sql:9 copies it. Every other table is hashed on its own columns in name order, so a
|
|
// rebuild that reorders columns keeps the hash. usage_records with any other column set refuses.
|
|
const USAGE_COLUMNS = ['seq', 'at', 'business', 'run', 'instance', 'provider', 'model', 'account', 'turn', 'source',
|
|
'phase', 'status', 'code', 'cursor', 'input', 'output', 'cache_write', 'cache_read'];
|
|
const COUNTS_ONLY = new Set(['messages', 'message_threads']);
|
|
const SCHEMA_KEYS = ['schema_digest', 'schema_version'];
|
|
const STAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
|
|
|
function refuse(code, detail) {
|
|
process.stderr.write(`snapshot: ${code}: ${detail}\n`);
|
|
process.exit(1);
|
|
}
|
|
|
|
function args(argv) {
|
|
const o = {};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (a === '--store' || a === '--after' || a === '--backup') {
|
|
if (i + 1 >= argv.length) refuse('usage', `${a} needs a value`);
|
|
o[a.slice(2)] = argv[++i];
|
|
} else if (a === '--held' || a === '--stopped') {
|
|
if (o.mode) refuse('usage', 'give one of --held and --stopped');
|
|
o.mode = a.slice(2);
|
|
} else refuse('usage', `unknown argument ${a}`);
|
|
}
|
|
if (!o.store || !o.mode) refuse('usage', 'node snapshot.mjs --store /abs/bus.sqlite (--held | --stopped) [--after STAMP] [--backup /abs/bus.sqlite]');
|
|
if (!isAbsolute(o.store)) refuse('usage', '--store must be an absolute path');
|
|
if (o.backup !== undefined && !isAbsolute(o.backup)) refuse('usage', '--backup must be an absolute path');
|
|
if (o.after !== undefined && !(STAMP.test(o.after) && Number.isFinite(Date.parse(o.after))))
|
|
refuse('usage', '--after must be a STAMP like 2026-10-20T00:11:14.123Z');
|
|
return o;
|
|
}
|
|
|
|
function file(path, code) {
|
|
if (!existsSync(path) || !lstatSync(path).isFile()) refuse(code, `${path} is not a file`);
|
|
}
|
|
|
|
function immutable(path, code) {
|
|
for (const f of [path + '-wal', path + '-shm'])
|
|
if (existsSync(f)) refuse(code, `${f} is present; an immutable read would skip the WAL`);
|
|
return new DatabaseSync(pathToFileURL(path).href + '?immutable=1', { readOnly: true });
|
|
}
|
|
|
|
function open({ store, mode }) {
|
|
file(store, 'store-missing');
|
|
if (mode === 'stopped') return immutable(store, 'store-unclean');
|
|
// A plain readOnly open of a WAL store with no -wal and -shm creates them, so a store that
|
|
// nothing holds refuses here instead of gaining two files.
|
|
for (const f of [store + '-wal', store + '-shm'])
|
|
if (!existsSync(f)) refuse('store-not-held', `${f} is absent; use --stopped for a store the bus doesn't hold`);
|
|
return new DatabaseSync(store, { readOnly: true });
|
|
}
|
|
|
|
const q = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
const value = (v) => (typeof v === 'bigint' ? String(v) : v instanceof Uint8Array ? { hex: Buffer.from(v).toString('hex') } : v);
|
|
|
|
// The same records `mosaic usage` counts: per (run, turn, source) the latest row, an end row
|
|
// winning; a turn is known only when that row is an end row with status known.
|
|
function byRun(rows) {
|
|
const turns = new Map();
|
|
for (const row of rows) {
|
|
const key = JSON.stringify([row.business, row.run, row.turn, row.source]);
|
|
if (row.phase === 'end' || !turns.has(key)) turns.set(key, row);
|
|
}
|
|
const runs = new Map();
|
|
for (const row of turns.values()) {
|
|
const key = JSON.stringify([row.business, row.run]);
|
|
if (!runs.has(key)) runs.set(key, { business: row.business, run: row.run, input: 0n, output: 0n, unknown: new Set() });
|
|
const r = runs.get(key);
|
|
if (row.phase === 'end' && row.status === 'known') {
|
|
r.input += BigInt(row.input);
|
|
r.output += BigInt(row.output);
|
|
} else r.unknown.add(row.turn);
|
|
}
|
|
return [...runs.values()].map(({ unknown, input, output, ...r }) => ({
|
|
...r, input: String(input), output: String(output), unknownTurns: unknown.size,
|
|
}));
|
|
}
|
|
|
|
// meta is compared by key: the names of the keys whose value differs, never the values.
|
|
function metaKeys(db) {
|
|
return new Map(db.prepare('SELECT key, value FROM meta').all().map((r) => [String(r.key), String(r.value)]));
|
|
}
|
|
|
|
// One table: rows, max seq and sha256; rows after `after`; with `priorMax` (the backup's max
|
|
// seq) a second hash of the rows at or below it and the count of rows above it. The hashes stay
|
|
// inside: a COUNTS_ONLY table's `t` gets none.
|
|
function read(db, table, { after, priorMax } = {}) {
|
|
const cols = db.prepare('SELECT name FROM pragma_table_info(?)').all(table).map((c) => c.name);
|
|
if (!cols.length) refuse('table-missing', table);
|
|
let hashed = [...cols].sort();
|
|
if (table === 'usage_records') {
|
|
if (JSON.stringify(hashed) !== JSON.stringify([...USAGE_COLUMNS].sort()))
|
|
refuse('columns-changed', `usage_records has columns ${hashed.join(',')}`);
|
|
hashed = USAGE_COLUMNS;
|
|
}
|
|
const seq = cols.includes('seq');
|
|
const at = cols.includes('at');
|
|
const stmt = db.prepare(`SELECT ${hashed.map(q).join(',')} FROM ${q(table)} ORDER BY ${seq ? 'seq' : hashed.map(q).join(',')}`);
|
|
stmt.setReadBigInts(true);
|
|
const full = createHash('sha256');
|
|
const prior = createHash('sha256');
|
|
const limit = priorMax === undefined ? undefined : BigInt(priorMax ?? 0);
|
|
let rows = 0;
|
|
let rowsAfter = 0;
|
|
let badAt = 0;
|
|
let newRows = 0;
|
|
const kept = [];
|
|
for (const row of stmt.iterate()) {
|
|
rows++;
|
|
const line = JSON.stringify(hashed.map((c) => value(row[c]))) + '\n';
|
|
full.update(line);
|
|
if (seq && limit !== undefined) {
|
|
if (row.seq <= limit) prior.update(line);
|
|
else newRows++;
|
|
}
|
|
if (after !== undefined && at) {
|
|
if (typeof row.at !== 'string' || !STAMP.test(row.at)) badAt++;
|
|
else if (row.at > after) rowsAfter++;
|
|
}
|
|
if (table === 'usage_records') kept.push(row);
|
|
}
|
|
if (badAt) refuse('at-format', `${table} has ${badAt} row(s) whose at is not in the STAMP form, so a text compare with --after would miscount`);
|
|
const sha256 = full.digest('hex');
|
|
const t = { rows, maxSeq: seq ? (db.prepare(`SELECT max(seq) AS m FROM ${q(table)}`).get().m ?? null) : null };
|
|
if (!COUNTS_ONLY.has(table)) t.sha256 = sha256;
|
|
if (after !== undefined && at) t.rowsAfter = rowsAfter;
|
|
if (table === 'usage_records') t.byRun = byRun(kept);
|
|
return { t, sha256, cols: hashed, seq, at, newRows, priorSha: seq && limit !== undefined ? prior.digest('hex') : null };
|
|
}
|
|
|
|
// Tables only: no index, view or trigger, and none of SQLite's own sqlite_* tables.
|
|
const tablesOf = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name")
|
|
.all().map((r) => r.name);
|
|
|
|
const o = args(process.argv.slice(2));
|
|
if (o.backup !== undefined) file(o.backup, 'backup-missing');
|
|
const db = open(o);
|
|
const bdb = o.backup === undefined ? null : immutable(o.backup, 'backup-unclean');
|
|
const out = {};
|
|
try {
|
|
db.exec('BEGIN');
|
|
out.schema = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get()?.value ?? null;
|
|
const names = tablesOf(db);
|
|
const base = new Map();
|
|
if (bdb) {
|
|
out.backupSchema = bdb.prepare("SELECT value FROM meta WHERE key='schema_version'").get()?.value ?? null;
|
|
for (const name of tablesOf(bdb)) base.set(name, read(bdb, name));
|
|
}
|
|
out.tables = {};
|
|
out.untimed = [];
|
|
const stop = [];
|
|
for (const name of names) {
|
|
const b = base.get(name);
|
|
const r = read(db, name, { after: o.after, priorMax: b ? b.t.maxSeq : undefined });
|
|
if (!r.at) out.untimed.push(name);
|
|
if (bdb) {
|
|
const v = { backupRows: b ? b.t.rows : null };
|
|
let changed;
|
|
if (!b) changed = true;
|
|
else if (name === 'meta') {
|
|
const now = metaKeys(db);
|
|
const then = metaKeys(bdb);
|
|
v.keysChanged = [...new Set([...now.keys(), ...then.keys()])].filter((k) => now.get(k) !== then.get(k)).sort();
|
|
changed = v.keysChanged.some((k) => !SCHEMA_KEYS.includes(k));
|
|
} else if (r.seq) {
|
|
v.backupMaxSeq = b.t.maxSeq;
|
|
v.newRows = r.newRows;
|
|
v.priorSame = JSON.stringify(r.cols) === JSON.stringify(b.cols) && r.priorSha === b.sha256;
|
|
changed = r.newRows > 0 || !v.priorSame;
|
|
} else {
|
|
v.same = JSON.stringify(r.cols) === JSON.stringify(b.cols) && r.sha256 === b.sha256;
|
|
changed = !v.same;
|
|
}
|
|
if ((r.t.rowsAfter ?? 0) > 0) changed = true;
|
|
r.t.vsBackup = v;
|
|
if (changed && name !== 'events') stop.push(name);
|
|
}
|
|
out.tables[name] = r.t;
|
|
}
|
|
if (bdb) {
|
|
out.onlyInBackup = [...base.keys()].filter((n) => !names.includes(n));
|
|
out.stopTables = [...stop, ...out.onlyInBackup].sort();
|
|
}
|
|
if (o.after !== undefined) {
|
|
out.after = o.after;
|
|
out.eventsAfter = out.tables.events?.rowsAfter ?? null;
|
|
}
|
|
} finally {
|
|
try { db.exec('ROLLBACK'); } catch {}
|
|
db.close();
|
|
bdb?.close();
|
|
}
|
|
process.stdout.write(JSON.stringify(out, null, 2) + '\n');
|