Files
jason.woltje c5ae4d7ac4 darkwing: merged B+C window packet, STATE.md (sha256 7be42b6bcab5657d), W0-W11 transcripts, S1/S2/S2b snapshots, P3 rehearsals and snapshot.mjs checks (#1673, #1674; P19 386, 382)
Evidence for the STATE report Sage checked and filed in 88f3253d9 (sage-1151). The skills-enabled
tar stays out of git; it is the W0b restore point under $WIN only. No credential or Environment
values: unit reads record ExecStart only, and the P3 credential files were fake placeholders.
2026-10-19 21:26:49 -05:00

236 lines
12 KiB
JavaScript

// Merged window B+C, S1, S2 and D3 (repin-merged-bc.md "S1 and S2"): a read-only snapshot of every
// table in a bus store, so the 3h to 3i migration and a restore can be checked without the bus.
// OLD's broker doesn't take --tester on usage, names or threads.
//
// usage: node snapshot.mjs --store /abs/bus.sqlite (--held | --stopped) [--after STAMP] [--backup /abs/bus.sqlite]
// --held the running bus holds the store: a plain readOnly open, which adds no file to a WAL
// store that already has -wal and -shm (as stack-release.sh does for a held store).
// Refuses store-not-held when either is absent
// --stopped the bus is stopped: refuses store-unclean when -wal or -shm is present, else opens
// with immutable=1 and readOnly, which creates no file and leaves the store's mtime
// and sha256 alone
// --after also counts each timed table's rows with `at` after STAMP (mos-0838 D3, before a
// --restore). STAMP is a backup's STAMP file, in now_ms form (stack-release.sh:43,
// 2026-10-20T00:11:14.123Z). The count is a text compare, as the tool's own heldcount
// does, so every `at` in every timed table must have that form: a table with any
// other value refuses at-format and names the table (sage-1125)
// --backup also compares every table with a stack-release backup's bus.sqlite, which opens as
// --stopped opens a store: immutable=1 and readOnly, refusing backup-unclean when -wal
// or -shm is present, so nothing writes into the backup set
//
// It prints one JSON object: the schema version, then for every table in sqlite_master (type
// table, less sqlite_*) the row count, the max seq and a sha256 of the rows (in seq order, else in
// column order). `untimed` names the tables with no `at` column. For usage_records it adds per-run
// input and output sums, counted as `mosaic usage` counts them (usage.mjs turnRecords). messages
// and message_threads hold bodies and names, so they print counts only, no sha256 (sage-1125). It
// prints counts, hashes and table and meta key names only: no message body, no instance, project
// or thread name, no credential and no file content. It reads no config: the store path is always
// given.
//
// With --backup, each table gets `vsBackup` and `stopTables` names every table other than events
// whose rows differ from the backup's (sage-1123): a row with `at` after STAMP, a row past the
// backup's max seq, a changed row at or below it, a changed untimed table, or a table only one
// side has. meta counts only when a key other than schema_version and schema_digest changed (the
// 3h to 3i migration sets those two). Events are left to the tool's own --events N.
import { DatabaseSync } from 'node:sqlite';
import { createHash } from 'node:crypto';
import { existsSync, lstatSync } from 'node:fs';
import { isAbsolute } from 'node:path';
import { pathToFileURL } from 'node:url';
// usage_records is hashed on these columns in this order, never SELECT * (mos-0888 F1), as
// 3h-3i.sql:9 copies it. Every other table is hashed on its own columns in name order, so a
// rebuild that reorders columns keeps the hash. usage_records with any other column set refuses.
const USAGE_COLUMNS = ['seq', 'at', 'business', 'run', 'instance', 'provider', 'model', 'account', 'turn', 'source',
'phase', 'status', 'code', 'cursor', 'input', 'output', 'cache_write', 'cache_read'];
const COUNTS_ONLY = new Set(['messages', 'message_threads']);
const SCHEMA_KEYS = ['schema_digest', 'schema_version'];
const STAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
function refuse(code, detail) {
process.stderr.write(`snapshot: ${code}: ${detail}\n`);
process.exit(1);
}
function args(argv) {
const o = {};
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === '--store' || a === '--after' || a === '--backup') {
if (i + 1 >= argv.length) refuse('usage', `${a} needs a value`);
o[a.slice(2)] = argv[++i];
} else if (a === '--held' || a === '--stopped') {
if (o.mode) refuse('usage', 'give one of --held and --stopped');
o.mode = a.slice(2);
} else refuse('usage', `unknown argument ${a}`);
}
if (!o.store || !o.mode) refuse('usage', 'node snapshot.mjs --store /abs/bus.sqlite (--held | --stopped) [--after STAMP] [--backup /abs/bus.sqlite]');
if (!isAbsolute(o.store)) refuse('usage', '--store must be an absolute path');
if (o.backup !== undefined && !isAbsolute(o.backup)) refuse('usage', '--backup must be an absolute path');
if (o.after !== undefined && !(STAMP.test(o.after) && Number.isFinite(Date.parse(o.after))))
refuse('usage', '--after must be a STAMP like 2026-10-20T00:11:14.123Z');
return o;
}
function file(path, code) {
if (!existsSync(path) || !lstatSync(path).isFile()) refuse(code, `${path} is not a file`);
}
function immutable(path, code) {
for (const f of [path + '-wal', path + '-shm'])
if (existsSync(f)) refuse(code, `${f} is present; an immutable read would skip the WAL`);
return new DatabaseSync(pathToFileURL(path).href + '?immutable=1', { readOnly: true });
}
function open({ store, mode }) {
file(store, 'store-missing');
if (mode === 'stopped') return immutable(store, 'store-unclean');
// A plain readOnly open of a WAL store with no -wal and -shm creates them, so a store that
// nothing holds refuses here instead of gaining two files.
for (const f of [store + '-wal', store + '-shm'])
if (!existsSync(f)) refuse('store-not-held', `${f} is absent; use --stopped for a store the bus doesn't hold`);
return new DatabaseSync(store, { readOnly: true });
}
const q = (name) => `"${name.replaceAll('"', '""')}"`;
const value = (v) => (typeof v === 'bigint' ? String(v) : v instanceof Uint8Array ? { hex: Buffer.from(v).toString('hex') } : v);
// The same records `mosaic usage` counts: per (run, turn, source) the latest row, an end row
// winning; a turn is known only when that row is an end row with status known.
function byRun(rows) {
const turns = new Map();
for (const row of rows) {
const key = JSON.stringify([row.business, row.run, row.turn, row.source]);
if (row.phase === 'end' || !turns.has(key)) turns.set(key, row);
}
const runs = new Map();
for (const row of turns.values()) {
const key = JSON.stringify([row.business, row.run]);
if (!runs.has(key)) runs.set(key, { business: row.business, run: row.run, input: 0n, output: 0n, unknown: new Set() });
const r = runs.get(key);
if (row.phase === 'end' && row.status === 'known') {
r.input += BigInt(row.input);
r.output += BigInt(row.output);
} else r.unknown.add(row.turn);
}
return [...runs.values()].map(({ unknown, input, output, ...r }) => ({
...r, input: String(input), output: String(output), unknownTurns: unknown.size,
}));
}
// meta is compared by key: the names of the keys whose value differs, never the values.
function metaKeys(db) {
return new Map(db.prepare('SELECT key, value FROM meta').all().map((r) => [String(r.key), String(r.value)]));
}
// One table: rows, max seq and sha256; rows after `after`; with `priorMax` (the backup's max
// seq) a second hash of the rows at or below it and the count of rows above it. The hashes stay
// inside: a COUNTS_ONLY table's `t` gets none.
function read(db, table, { after, priorMax } = {}) {
const cols = db.prepare('SELECT name FROM pragma_table_info(?)').all(table).map((c) => c.name);
if (!cols.length) refuse('table-missing', table);
let hashed = [...cols].sort();
if (table === 'usage_records') {
if (JSON.stringify(hashed) !== JSON.stringify([...USAGE_COLUMNS].sort()))
refuse('columns-changed', `usage_records has columns ${hashed.join(',')}`);
hashed = USAGE_COLUMNS;
}
const seq = cols.includes('seq');
const at = cols.includes('at');
const stmt = db.prepare(`SELECT ${hashed.map(q).join(',')} FROM ${q(table)} ORDER BY ${seq ? 'seq' : hashed.map(q).join(',')}`);
stmt.setReadBigInts(true);
const full = createHash('sha256');
const prior = createHash('sha256');
const limit = priorMax === undefined ? undefined : BigInt(priorMax ?? 0);
let rows = 0;
let rowsAfter = 0;
let badAt = 0;
let newRows = 0;
const kept = [];
for (const row of stmt.iterate()) {
rows++;
const line = JSON.stringify(hashed.map((c) => value(row[c]))) + '\n';
full.update(line);
if (seq && limit !== undefined) {
if (row.seq <= limit) prior.update(line);
else newRows++;
}
if (after !== undefined && at) {
if (typeof row.at !== 'string' || !STAMP.test(row.at)) badAt++;
else if (row.at > after) rowsAfter++;
}
if (table === 'usage_records') kept.push(row);
}
if (badAt) refuse('at-format', `${table} has ${badAt} row(s) whose at is not in the STAMP form, so a text compare with --after would miscount`);
const sha256 = full.digest('hex');
const t = { rows, maxSeq: seq ? (db.prepare(`SELECT max(seq) AS m FROM ${q(table)}`).get().m ?? null) : null };
if (!COUNTS_ONLY.has(table)) t.sha256 = sha256;
if (after !== undefined && at) t.rowsAfter = rowsAfter;
if (table === 'usage_records') t.byRun = byRun(kept);
return { t, sha256, cols: hashed, seq, at, newRows, priorSha: seq && limit !== undefined ? prior.digest('hex') : null };
}
// Tables only: no index, view or trigger, and none of SQLite's own sqlite_* tables.
const tablesOf = (db) => db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name")
.all().map((r) => r.name);
const o = args(process.argv.slice(2));
if (o.backup !== undefined) file(o.backup, 'backup-missing');
const db = open(o);
const bdb = o.backup === undefined ? null : immutable(o.backup, 'backup-unclean');
const out = {};
try {
db.exec('BEGIN');
out.schema = db.prepare("SELECT value FROM meta WHERE key='schema_version'").get()?.value ?? null;
const names = tablesOf(db);
const base = new Map();
if (bdb) {
out.backupSchema = bdb.prepare("SELECT value FROM meta WHERE key='schema_version'").get()?.value ?? null;
for (const name of tablesOf(bdb)) base.set(name, read(bdb, name));
}
out.tables = {};
out.untimed = [];
const stop = [];
for (const name of names) {
const b = base.get(name);
const r = read(db, name, { after: o.after, priorMax: b ? b.t.maxSeq : undefined });
if (!r.at) out.untimed.push(name);
if (bdb) {
const v = { backupRows: b ? b.t.rows : null };
let changed;
if (!b) changed = true;
else if (name === 'meta') {
const now = metaKeys(db);
const then = metaKeys(bdb);
v.keysChanged = [...new Set([...now.keys(), ...then.keys()])].filter((k) => now.get(k) !== then.get(k)).sort();
changed = v.keysChanged.some((k) => !SCHEMA_KEYS.includes(k));
} else if (r.seq) {
v.backupMaxSeq = b.t.maxSeq;
v.newRows = r.newRows;
v.priorSame = JSON.stringify(r.cols) === JSON.stringify(b.cols) && r.priorSha === b.sha256;
changed = r.newRows > 0 || !v.priorSame;
} else {
v.same = JSON.stringify(r.cols) === JSON.stringify(b.cols) && r.sha256 === b.sha256;
changed = !v.same;
}
if ((r.t.rowsAfter ?? 0) > 0) changed = true;
r.t.vsBackup = v;
if (changed && name !== 'events') stop.push(name);
}
out.tables[name] = r.t;
}
if (bdb) {
out.onlyInBackup = [...base.keys()].filter((n) => !names.includes(n));
out.stopTables = [...stop, ...out.onlyInBackup].sort();
}
if (o.after !== undefined) {
out.after = o.after;
out.eventsAfter = out.tables.events?.rowsAfter ?? null;
}
} finally {
try { db.exec('ROLLBACK'); } catch {}
db.close();
bdb?.close();
}
process.stdout.write(JSON.stringify(out, null, 2) + '\n');