fix(cli): isolate browser network and halt on ledger loss (#797)
This commit is contained in:
1 parent
19e6cb0f1e
commit
905d313cdf
9 files changed
+334
-49
No files matched your search
@@ -6198,3 +6198,5 @@ After: personally authored three-file skills/ms-ux-improvement coordinator packa
|
|||||||
2026-11-05 | #797 ui797 support author candidate verified | Added local single-subject HTTPS OIDC and fixed TLS forwarder, persistent real-input Chromium CLI with private NSS trust, and external durable per-model metered-stop ledgers. Final focused run: 34 pass / 0 fail / 0 skip (identity 6, browser 5, budget 23); dependency regressions: 111 pass / 0 fail / 0 skip; supported processing CLI regressions: 4 pass / 0 fail / 0 skip. Initial configuration-selection, Basic credential decoding, browser readiness/count and shutdown issues were corrected; all final gates pass. Source remains confined to packages/cli/live plus append-only protocol records. Local commit authorized; independent exact-candidate review and supervised-service acceptance remain with lead. No .53/live activity, model spend, push or product-runtime/role/contract changes. Evidence: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-r1-evidence/; report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-r1-report.md.
|
2026-11-05 | #797 ui797 support author candidate verified | Added local single-subject HTTPS OIDC and fixed TLS forwarder, persistent real-input Chromium CLI with private NSS trust, and external durable per-model metered-stop ledgers. Final focused run: 34 pass / 0 fail / 0 skip (identity 6, browser 5, budget 23); dependency regressions: 111 pass / 0 fail / 0 skip; supported processing CLI regressions: 4 pass / 0 fail / 0 skip. Initial configuration-selection, Basic credential decoding, browser readiness/count and shutdown issues were corrected; all final gates pass. Source remains confined to packages/cli/live plus append-only protocol records. Local commit authorized; independent exact-candidate review and supervised-service acceptance remain with lead. No .53/live activity, model spend, push or product-runtime/role/contract changes. Evidence: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-r1-evidence/; report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-r1-report.md.
|
||||||
|
|
||||||
2026-11-06 | #797 ui797 support R2 review corrections verified | Before: local bd8340b9b accepted unrelated named instances, allowed Chromium WebSocket escape, trusted journal token totals, bypassed halt on lock failure, split model allowances across routes, omitted the absolute cutoff/100M GLM exception, and leaked browser fixtures. After: literal ui797/config/marker validation and persisted campaign revalidation; forced origin-only HTTP proxy (loopback bypass disabled, upgrades refused, HTTPS CONNECT restricted) with real HTTP/HTTPS Chromium negatives and independent zero-TCP/zero-upgrade forbidden-listener assertions; authoritative entry-token comparison/recomputed totals; preserved stale locks with durable halt on interruption/restart/storage failure; chartered proxy GLM/Claude/Codex bindings and one allowance per provider/model; inclusive 2026-11-07T06:00:00.000Z cutoff; exact GLM 100M/other models 10M maxima; awaited reverse-order fixture shutdown with owned-process checks and an intentional-failure whole-file regression. Initial CDP/response-policy browser containment and daemon close-event attempts failed and were corrected; terminal individual whole-file runs are identity 8, browser 8, budget 33 = 49 pass / 0 fail / 0 skip / 0 cancelled. All retained review probes rerun: wrong instance/deadline/split-route refuse, numeric corruption latches/halts, locked checks halt without reclaim, forbidden WebSocket hits 0, SIGTERM/SIGKILL survivors 0. The original general probe now exits 1 at its uncaught, correctly refused split-route initialization; a logging-only companion records every outcome. Final owned-process and 17440/17442/17443 listener sweeps are empty; unrelated existing sessions untouched. Source scope packages/cli/live only plus append-only logs. Queue refused linked worktree; no queue/forge write, push, stash, model call, .53/live contact or product-runtime/role/contract/dependency change. Local author candidate only; independent exact-head approval remains with lead. Evidence/report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r2-evidence/ and ui797-support-fix-r2-report.md.
|
2026-11-06 | #797 ui797 support R2 review corrections verified | Before: local bd8340b9b accepted unrelated named instances, allowed Chromium WebSocket escape, trusted journal token totals, bypassed halt on lock failure, split model allowances across routes, omitted the absolute cutoff/100M GLM exception, and leaked browser fixtures. After: literal ui797/config/marker validation and persisted campaign revalidation; forced origin-only HTTP proxy (loopback bypass disabled, upgrades refused, HTTPS CONNECT restricted) with real HTTP/HTTPS Chromium negatives and independent zero-TCP/zero-upgrade forbidden-listener assertions; authoritative entry-token comparison/recomputed totals; preserved stale locks with durable halt on interruption/restart/storage failure; chartered proxy GLM/Claude/Codex bindings and one allowance per provider/model; inclusive 2026-11-07T06:00:00.000Z cutoff; exact GLM 100M/other models 10M maxima; awaited reverse-order fixture shutdown with owned-process checks and an intentional-failure whole-file regression. Initial CDP/response-policy browser containment and daemon close-event attempts failed and were corrected; terminal individual whole-file runs are identity 8, browser 8, budget 33 = 49 pass / 0 fail / 0 skip / 0 cancelled. All retained review probes rerun: wrong instance/deadline/split-route refuse, numeric corruption latches/halts, locked checks halt without reclaim, forbidden WebSocket hits 0, SIGTERM/SIGKILL survivors 0. The original general probe now exits 1 at its uncaught, correctly refused split-route initialization; a logging-only companion records every outcome. Final owned-process and 17440/17442/17443 listener sweeps are empty; unrelated existing sessions untouched. Source scope packages/cli/live only plus append-only logs. Queue refused linked worktree; no queue/forge write, push, stash, model call, .53/live contact or product-runtime/role/contract/dependency change. Local author candidate only; independent exact-head approval remains with lead. Evidence/report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r2-evidence/ and ui797-support-fix-r2-report.md.
|
||||||
|
|
||||||
|
2026-11-06 | #797 ui797 support R3 P1 corrections verified | Before: 19e6cb0f1 still emitted WebRTC STUN/TURN UDP despite its flag, and unavailable ledger directories bypassed halt and terminated serve. After: mandatory Bubblewrap network/PID namespaces with an isolated, up loopback; a single namespace TCP relay over a 0600 Unix socket to the existing origin-only proxy, preserving private NSS/TLS validation, control socket permissions and awaited browser-before-proxy cleanup. Real Chromium regressions from both origins create nine ICE agents against six independent IPv4/IPv6/127.0.0.2 STUN/TURN UDP listeners and three TCP listeners; zero datagrams/connections, existing WebSocket/HTTP/SSO/CA negatives retained. Campaign initialization now exclusively pins owner/config/binding/ledger evidence beside the campaign, independently of the ledger volume; missing/renamed/0500 storage still attempts supported CLI halt, explicitly reports nondurable latch/receipt, preserves evidence and never recreates an epoch. Serve survives failures and reasserts each tick/Resume, retaining its stop across storage recovery; invalid/foreign/changed targets refuse. Older intact ledger format preserved. Final sequential whole files: identity 8, browser 10, budget 43 = 61 pass / 0 fail / 0 skip / 0 cancelled. All R1/R2 probes rerun: zero forbidden TCP/UDP, all four storage variants count one halt, original latch preserved, stale-lock/corruption/cap/cutoff/identity negatives retained, SIGTERM/SIGKILL/startup-failure survivors zero. Original R1 general probe exits 1 at its correctly refused split-route setup. Independent final sweep: zero owned processes/listeners at fixed and 43 recorded endpoints; scratch sweep parser corrected before its terminal run. Exactly packages/cli/live plus this single entry and one SESSIONS line; linked-worktree queue refused, no queue/forge writes, push, stash, model call, .53 contact, host network/DNS/trust changes or unrelated-session cleanup. Local implementation ready for independent exact-head review; owner-service successful quiescence remains a later acceptance gate. Evidence/report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r3-evidence/ and ui797-support-fix-r3-report.md.
|
||||||
@@ -2067,3 +2067,4 @@ are never rewritten or removed; corrections are new entries.
|
|||||||
2026-11-05 | Codex ui797 implementation child | #797 isolated OIDC/TLS, browser CLI and campaign budget support from 05fdfef9e on feat/ui797-support | Author implementation/tests underway in lead-chartered worktree; queue refused linked worktree, no canonical queue writes or protected actions.
|
2026-11-05 | Codex ui797 implementation child | #797 isolated OIDC/TLS, browser CLI and campaign budget support from 05fdfef9e on feat/ui797-support | Author implementation/tests underway in lead-chartered worktree; queue refused linked worktree, no canonical queue writes or protected actions.
|
||||||
2026-11-05 | Codex ui797 implementation child | #797 feat/ui797-support author handoff | Three isolated test tools and focused tests ready for lead review; final unique gates 149 pass / 0 fail / 0 skip, real production-CLI browser OIDC sign-in included, metered-stop/ancestry limits documented. Local commit only; no remote/live activity, spend, role/contract or product runtime edits.
|
2026-11-05 | Codex ui797 implementation child | #797 feat/ui797-support author handoff | Three isolated test tools and focused tests ready for lead review; final unique gates 149 pass / 0 fail / 0 skip, real production-CLI browser OIDC sign-in included, metered-stop/ancestry limits documented. Local commit only; no remote/live activity, spend, role/contract or product runtime edits.
|
||||||
2026-11-06 | ui797-support-fix-r2 implementation sub-agent | #797 corrections on feat/ui797-support above bd8340b9b, confined to packages/cli/live plus one appended build entry and this line | All eight review findings addressed with regression tests; sequential whole files identity 8/browser 8/budget 33 (49 pass, 0 fail/skip/cancel), all review probes rerun, owned-process/listener sweeps empty; forced proxy chosen after browser-only containment failed. Local commit authorized; independent review pending. Queue refused linked worktree; no canonical queue/forge writes, push, stash, live/.53 contact, model spend, product-runtime/role/contract/dependency changes or attribution trailers. Report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r2-report.md.
|
2026-11-06 | ui797-support-fix-r2 implementation sub-agent | #797 corrections on feat/ui797-support above bd8340b9b, confined to packages/cli/live plus one appended build entry and this line | All eight review findings addressed with regression tests; sequential whole files identity 8/browser 8/budget 33 (49 pass, 0 fail/skip/cancel), all review probes rerun, owned-process/listener sweeps empty; forced proxy chosen after browser-only containment failed. Local commit authorized; independent review pending. Queue refused linked worktree; no canonical queue/forge writes, push, stash, live/.53 contact, model spend, product-runtime/role/contract/dependency changes or attribution trailers. Report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r2-report.md.
|
||||||
|
2026-11-06 | ui797-support-fix-r3 implementation sub-agent | #797 two P1 corrections above 19e6cb0f1 on feat/ui797-support | Mandatory browser network namespace/private proxy relay and independently bound unavailable-ledger halt with persistent serve reassertion; final sequential whole-file suites 61 pass, 0 fail/skip/cancel; all R1/R2 probes rerun, zero forbidden TCP/UDP, storage halts counted and latch preserved, zero owned processes/listeners. One BUILD-LOG entry and packages/cli/live changes only; local commit authorized, independent review pending. Queue refused linked worktree; no push/stash/forge/model/.53/host network-DNS-trust or unrelated-session action. Report: /mnt/storage/scratch/mosaic-lead/mvp3/ui797-support-fix-r3-report.md.
|
||||||
@@ -136,9 +136,14 @@ page/subresource redirect must stay on the selected origin. Other targets and
|
|||||||
WebSocket connections outside that origin are blocked before the handshake by a
|
WebSocket connections outside that origin are blocked before the handshake by a
|
||||||
forced local HTTP proxy with Chromium's loopback bypass disabled. It opens sockets
|
forced local HTTP proxy with Chromium's loopback bypass disabled. It opens sockets
|
||||||
only to the selected origin, refuses HTTP upgrades, and permits CONNECT only to
|
only to the selected origin, refuses HTTP upgrades, and permits CONNECT only to
|
||||||
the selected HTTPS host/port. QUIC and non-proxied WebRTC UDP are disabled. The
|
the selected HTTPS host/port. Chromium and its TCP relay run in a mandatory Bubblewrap
|
||||||
installed Chromium's CDP WebSocket blocklist did not contain handshakes in the
|
network namespace, with its own loopback up and no external interfaces or routes.
|
||||||
negative probe; this external boundary does. Request interception also guards
|
The relay accepts only the forced proxy's TCP port and carries bytes over a 0600
|
||||||
|
Unix socket to that origin-checking host proxy. All other TCP and UDP, including
|
||||||
|
IPv4/IPv6 STUN/TURN on other loopback addresses, stay inside the empty namespace.
|
||||||
|
Bubblewrap's PID namespace and parent-death handling also contain its children.
|
||||||
|
The WebRTC flag alone does not prevent UDP on Chromium 156; it remains defense
|
||||||
|
in depth, alongside disabling QUIC. Request interception also guards
|
||||||
redirects and subresources. A detected
|
redirects and subresources. A detected
|
||||||
blocked network attempt makes the session fail until
|
blocked network attempt makes the session fail until
|
||||||
`stop`. The driver never contacts 7332 or mosaic.woltje.com.
|
`stop`. The driver never contacts 7332 or mosaic.woltje.com.
|
||||||
@@ -255,10 +260,17 @@ messages and ended runs detect missing completed usage. An in-flight begin waits
|
|||||||
for its outcome; unknown/malformed completed usage stops the campaign.
|
for its outcome; unknown/malformed completed usage stops the campaign.
|
||||||
|
|
||||||
Each provider/route/model has a separate append-only ledger, durable across
|
Each provider/route/model has a separate append-only ledger, durable across
|
||||||
rounds/restarts. Campaign bytes, root identity and store identity are pinned;
|
rounds/restarts. Campaign bytes, config bytes, root identity and store identity are pinned;
|
||||||
changed caps/epoch, rewritten/disappeared usage and damaged ledger files stop.
|
changed caps/epoch, rewritten/disappeared usage and damaged ledger files stop.
|
||||||
Checks recompute each journal entry's tokens and totals from authoritative usage;
|
Checks recompute each journal entry's tokens and totals from authoritative usage;
|
||||||
a valid-JSON token mismatch durably latches and halts.
|
a valid-JSON token mismatch durably latches and halts.
|
||||||
|
Initialization exclusively writes `campaign.json.binding.json` beside the owner
|
||||||
|
campaign, outside the ledger volume, before creating the ledger. Preserve this
|
||||||
|
0600 file and the campaign: they independently pin the target and ledger path
|
||||||
|
when accounting storage is unavailable. Checks never recreate that file or an
|
||||||
|
accounting epoch. A campaign initialized by an older implementation without this
|
||||||
|
pin requires intact ledger evidence to identify its target; do not reinitialize
|
||||||
|
or reset an existing campaign to work around missing evidence.
|
||||||
Initialization is exclusive. There is no reset, raise or resume command. A stale
|
Initialization is exclusive. There is no reset, raise or resume command. A stale
|
||||||
`check.lock` after interruption requires owner diagnosis; never auto-reclaim it.
|
`check.lock` after interruption requires owner diagnosis; never auto-reclaim it.
|
||||||
An existing or uncreatable lock prevents accounting and still asserts the sticky
|
An existing or uncreatable lock prevents accounting and still asserts the sticky
|
||||||
@@ -270,12 +282,18 @@ At any cap, deadline or accounting refusal, the supervisor fsyncs `stop.json`
|
|||||||
before invoking the existing `mosaic halt --business ... --json` production CLI
|
before invoking the existing `mosaic halt --business ... --json` production CLI
|
||||||
with the selected test `MOSAIC_CONFIG`. The CLI's human-shell/ancestry proof stays
|
with the selected test `MOSAIC_CONFIG`. The CLI's human-shell/ancestry proof stays
|
||||||
intact. A refusal/unknown stop is retained as a failed `halt-N.json` receipt and
|
intact. A refusal/unknown stop is retained as a failed `halt-N.json` receipt and
|
||||||
exits nonzero; the latch remains. If storage cannot persist a latch or receipt, it
|
`check` exits nonzero; the latch remains. Missing, renamed or nonprivate ledger
|
||||||
still attempts halt and reports the storage failure; durable evidence then needs
|
directories still trigger halt using the independently validated owner binding.
|
||||||
|
If storage cannot persist a latch or receipt, it explicitly reports
|
||||||
|
`stop-latch-not-durable` and/or `stop-receipt-not-durable` after attempting halt;
|
||||||
|
durable evidence then needs
|
||||||
owner repair before any further campaign work. Do not strip markers, direct-bind a principal,
|
owner repair before any further campaign work. Do not strip markers, direct-bind a principal,
|
||||||
write processing state, or treat that receipt as quiescence. A properly installed
|
write processing state, or treat that receipt as quiescence. A properly installed
|
||||||
owner service must perform/confirm halt. `serve` reasserts a latched halt each tick;
|
owner service must perform/confirm halt. `serve` stays running on check/halt/storage
|
||||||
it never resumes. Manual Resume is outside the campaign protocol.
|
errors, reports each failed tick and reasserts halt on every tick, including after
|
||||||
|
a modelled Resume. It retains an in-memory stop if storage subsequently recovers.
|
||||||
|
Every attempt revalidates the target; invalid, changed or foreign bindings refuse
|
||||||
|
targeting. It never resumes. Manual Resume is outside the campaign protocol.
|
||||||
|
|
||||||
This is a **metered stop**, not strict inference admission or a hard provider
|
This is a **metered stop**, not strict inference admission or a hard provider
|
||||||
ceiling. Concurrent/in-flight work, transcript accounting and the 0.1–10-second
|
ceiling. Concurrent/in-flight work, transcript accounting and the 0.1–10-second
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
// Runs only inside Bubblewrap's network/PID namespaces. No host TCP/UDP handle
|
||||||
|
// is inherited: the single TCP listener relays bytes to the fixed Unix proxy.
|
||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { createServer, createConnection } from 'node:net';
|
||||||
|
import { closeSync, readlinkSync } from 'node:fs';
|
||||||
|
import { networkInterfaces } from 'node:os';
|
||||||
|
|
||||||
|
const [hostNamespace, socket, port, executable, ...args] = process.argv.slice(2);
|
||||||
|
if (readlinkSync('/proc/self/ns/net') === hostNamespace ||
|
||||||
|
Object.keys(networkInterfaces()).some(name => name !== 'lo')) {
|
||||||
|
throw new Error('isolated browser network required');
|
||||||
|
}
|
||||||
|
const sockets = new Set();
|
||||||
|
const track = s => { sockets.add(s); s.once('close', () => sockets.delete(s)); return s; };
|
||||||
|
const server = createServer(peer => {
|
||||||
|
track(peer);
|
||||||
|
const upstream = track(createConnection(socket));
|
||||||
|
peer.on('error', () => upstream.destroy()); upstream.on('error', () => peer.destroy());
|
||||||
|
peer.once('close', () => upstream.destroy()); upstream.once('close', () => peer.destroy());
|
||||||
|
peer.pipe(upstream); upstream.pipe(peer);
|
||||||
|
});
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
server.once('error', reject); server.listen(Number(port), '127.0.0.1', resolve);
|
||||||
|
});
|
||||||
|
const child = spawn(executable, args, { stdio:['ignore','ignore','ignore',3,4] });
|
||||||
|
// Chromium alone owns the CDP pipes. Do not hold its EOF open on startup failure.
|
||||||
|
closeSync(3); closeSync(4);
|
||||||
|
let closing;
|
||||||
|
const close = () => closing ??= (async () => {
|
||||||
|
const closed = new Promise(resolve => server.close(resolve));
|
||||||
|
for (const s of sockets) s.destroy();
|
||||||
|
await closed;
|
||||||
|
})();
|
||||||
|
child.once('error', () => close().then(() => { process.exitCode = 1; }));
|
||||||
|
child.once('exit', (code, signal) => close().then(() => { process.exitCode = signal ? 1 : code; }));
|
||||||
|
for (const signal of ['SIGTERM','SIGINT']) process.once(signal, () => child.kill(signal));
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
// Metered stop, with accounting/in-flight delay. Never a provider-side hard ceiling.
|
// Metered stop, with accounting/in-flight delay. Never a provider-side hard ceiling.
|
||||||
import { DatabaseSync } from 'node:sqlite';
|
import { DatabaseSync } from 'node:sqlite';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import { closeSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, rmdirSync, writeFileSync } from 'node:fs';
|
import { closeSync, existsSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, rmdirSync, writeFileSync } from 'node:fs';
|
||||||
import { join, dirname } from 'node:path';
|
import { join, dirname } from 'node:path';
|
||||||
import { spawnSync } from 'node:child_process';
|
import { spawnSync } from 'node:child_process';
|
||||||
import { turnRecords } from '../../bus/src/usage.mjs';
|
import { turnRecords } from '../../bus/src/usage.mjs';
|
||||||
@@ -58,7 +58,12 @@ function flushDir(dir) { const fd = openSync(dir,'r'); try { fsyncSync(fd); } fi
|
|||||||
export function initialize({ campaign, ledger }) {
|
export function initialize({ campaign, ledger }) {
|
||||||
const { c,hash } = campaignAt(campaign), b = binding(c);
|
const { c,hash } = campaignAt(campaign), b = binding(c);
|
||||||
outsideRuntime(campaign,{dataRoot:b.dataRoot},c.config);
|
outsideRuntime(campaign,{dataRoot:b.dataRoot},c.config);
|
||||||
outsideRuntime(ledger,{dataRoot:b.dataRoot},c.config); privateDir(ledger,true);
|
outsideRuntime(ledger,{dataRoot:b.dataRoot},c.config);
|
||||||
|
// Independent owner evidence survives loss of the accounting volume. Created
|
||||||
|
// only at initialization; a check must never reconstruct an epoch or this pin.
|
||||||
|
exclusive(campaign+'.binding.json',JSON.stringify({version:1,hash,configHash:digest(readFileSync(c.config,'utf8')),binding:b,ledger})+'\n');
|
||||||
|
flushDir(dirname(campaign));
|
||||||
|
privateDir(ledger,true);
|
||||||
for (const cap of c.caps) exclusive(join(ledger,filename(cap)),JSON.stringify({ version:1,cap })+'\n');
|
for (const cap of c.caps) exclusive(join(ledger,filename(cap)),JSON.stringify({ version:1,cap })+'\n');
|
||||||
exclusive(join(ledger,'campaign.json'),JSON.stringify({ version:1,hash,binding:b,campaign:c })+'\n');
|
exclusive(join(ledger,'campaign.json'),JSON.stringify({ version:1,hash,binding:b,campaign:c })+'\n');
|
||||||
flushDir(ledger);
|
flushDir(ledger);
|
||||||
@@ -130,26 +135,38 @@ function loadLedger(ledger) {
|
|||||||
if (p.version !== 1 || !p.binding || !p.campaign) fail('ledger-manifest-invalid');
|
if (p.version !== 1 || !p.binding || !p.campaign) fail('ledger-manifest-invalid');
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
export async function check({ campaign,ledger }, { now=Date.now, read=readAccounting, halt=supportedHalt, acquireLock=mkdirSync }={}) {
|
function checkTarget({campaign,ledger}) {
|
||||||
let p, integrityFailure=false;
|
let p, pin, reason=null, writable=true;
|
||||||
try { p=loadLedger(ledger); } catch {
|
try { privateDir(ledger); } catch { writable=false; reason='ledger-storage-unavailable'; }
|
||||||
// With a damaged manifest, only the owner campaign and validated TEST marker may identify
|
try { if (!writable) throw new Error(reason); p=loadLedger(ledger); } catch {
|
||||||
// the target. Never silently rebuild the accounting epoch or continue with a zero ledger.
|
const {c,hash}=campaignAt(campaign);pin=JSON.parse(readPrivate(campaign+'.binding.json'));
|
||||||
privateDir(ledger); const {c,hash}=campaignAt(campaign);
|
if(pin.version!==1 || pin.hash!==hash || pin.ledger!==ledger || !pin.binding) fail('owner-binding-invalid-or-changed');
|
||||||
p={campaign:c,hash,binding:binding(c)}; integrityFailure=true;
|
p={campaign:c,hash,binding:pin.binding}; reason??='ledger-manifest-invalid';
|
||||||
}
|
}
|
||||||
const c=p.campaign;
|
const c=p.campaign;
|
||||||
// Pin the target before any halt. A changed config cannot redirect the stop into live data.
|
// Pin the target before any halt. A changed config cannot redirect the stop into live data.
|
||||||
checkBinding(p.binding,c); outsideRuntime(ledger,{dataRoot:p.binding.dataRoot},c.config);
|
checkBinding(p.binding,c); outsideRuntime(ledger,{dataRoot:p.binding.dataRoot},c.config);
|
||||||
outsideRuntime(campaign,{dataRoot:p.binding.dataRoot},c.config);
|
outsideRuntime(campaign,{dataRoot:p.binding.dataRoot},c.config);
|
||||||
|
// Older intact ledgers retain their original binding format. New campaigns
|
||||||
|
// additionally pin config bytes in independent owner evidence, without rewriting them.
|
||||||
|
if(pin || existsSync(campaign+'.binding.json')) {
|
||||||
|
pin??=JSON.parse(readPrivate(campaign+'.binding.json'));
|
||||||
|
if(pin.version!==1 || pin.ledger!==ledger || JSON.stringify(pin.binding)!==JSON.stringify(p.binding) ||
|
||||||
|
pin.configHash!==digest(readFileSync(c.config,'utf8')))fail('campaign-instance-changed');
|
||||||
|
}
|
||||||
|
return {p,c,reason,writable};
|
||||||
|
}
|
||||||
|
export async function check(opts, { now=Date.now, read=readAccounting, halt=supportedHalt, acquireLock=mkdirSync, stopReason=null }={}) {
|
||||||
|
const {ledger,campaign}=opts;
|
||||||
|
const {p,c,reason:targetReason,writable}=checkTarget(opts);
|
||||||
const lock=join(ledger,'check.lock'); let locked=false, lockFailure=false;
|
const lock=join(ledger,'check.lock'); let locked=false, lockFailure=false;
|
||||||
try { acquireLock(lock,{mode:0o700}); locked=true; } catch { lockFailure=true; }
|
if(writable) try { acquireLock(lock,{mode:0o700}); locked=true; } catch { lockFailure=true; }
|
||||||
// An interrupted/active/unwritable lock prevents accounting, never the verified halt.
|
// An interrupted/active/unwritable lock prevents accounting, never the verified halt.
|
||||||
// Only the check that created a lock may remove it on normal completion.
|
// Only the check that created a lock may remove it on normal completion.
|
||||||
try {
|
try {
|
||||||
let reason=integrityFailure?'ledger-manifest-invalid':lockFailure?'check-lock-unavailable':null, totals=[];
|
let reason=targetReason ?? stopReason ?? (lockFailure?'check-lock-unavailable':null), totals=[];
|
||||||
const stopFile=join(ledger,'stop.json');
|
const stopFile=join(ledger,'stop.json');
|
||||||
if (existsSync(stopFile)) {
|
if (writable && existsSync(stopFile)) {
|
||||||
reason='stop-already-latched';
|
reason='stop-already-latched';
|
||||||
try { JSON.parse(readPrivate(stopFile)); } catch { reason='stop-latch-invalid'; }
|
try { JSON.parse(readPrivate(stopFile)); } catch { reason='stop-latch-invalid'; }
|
||||||
}
|
}
|
||||||
@@ -190,23 +207,27 @@ export async function check({ campaign,ledger }, { now=Date.now, read=readAccoun
|
|||||||
}
|
}
|
||||||
} catch(e) { reason=/^[a-z-]{1,80}$/.test(e.message)?e.message:'accounting-failed'; }
|
} catch(e) { reason=/^[a-z-]{1,80}$/.test(e.message)?e.message:'accounting-failed'; }
|
||||||
if (reason) {
|
if (reason) {
|
||||||
let storageError=null;
|
const storageErrors=[];
|
||||||
try {
|
try {
|
||||||
|
if(!writable) throw new Error('unavailable');
|
||||||
if (!existsSync(stopFile)) { exclusive(stopFile,JSON.stringify({version:1,at:new Date(now()).toISOString(),reason,binding:p.binding,totals})+'\n'); flushDir(ledger); }
|
if (!existsSync(stopFile)) { exclusive(stopFile,JSON.stringify({version:1,at:new Date(now()).toISOString(),reason,binding:p.binding,totals})+'\n'); flushDir(ledger); }
|
||||||
} catch { storageError='stop-latch-not-durable'; }
|
} catch { storageErrors.push('stop-latch-not-durable'); }
|
||||||
// Always attempt halt, even when the evidence volume cannot persist the latch.
|
// Always attempt halt, even when the evidence volume cannot persist the latch.
|
||||||
let result,error;
|
let result,error;
|
||||||
try { checkBinding(p.binding,c); result=await halt(p.binding); } catch { error='supported-halt-refused-or-unknown'; }
|
try { checkBinding(p.binding,c); result=await halt(p.binding); } catch { error='supported-halt-refused-or-unknown'; }
|
||||||
try {
|
try {
|
||||||
|
if(!writable) throw new Error('unavailable');
|
||||||
const attempt=readdirSync(ledger).filter(n=>/^halt-[0-9]+\.json$/.test(n)).length+1;
|
const attempt=readdirSync(ledger).filter(n=>/^halt-[0-9]+\.json$/.test(n)).length+1;
|
||||||
exclusive(join(ledger,`halt-${attempt}.json`),JSON.stringify({at:new Date(now()).toISOString(),ok:!error,result:result??null,error:error??null,storageError})+'\n'); flushDir(ledger);
|
exclusive(join(ledger,`halt-${attempt}.json`),JSON.stringify({at:new Date(now()).toISOString(),ok:!error,result:result??null,error:error??null,storageError:storageErrors[0]??null})+'\n'); flushDir(ledger);
|
||||||
} catch { storageError='stop-receipt-not-durable'; }
|
} catch { storageErrors.push('stop-receipt-not-durable'); }
|
||||||
if (storageError) fail(storageError);
|
if (storageErrors.length) throw Object.assign(new Error(storageErrors.join('; ')),{
|
||||||
|
result:{stopped:true,reason,totals,halt:result??null,haltError:error??null,storageErrors},
|
||||||
|
});
|
||||||
if (error) fail(error);
|
if (error) fail(error);
|
||||||
return {stopped:true,reason,totals,halt:result};
|
return {stopped:true,reason,totals,halt:result};
|
||||||
}
|
}
|
||||||
return {stopped:false,totals};
|
return {stopped:false,totals};
|
||||||
} finally { if (locked) rmdirSync(lock); }
|
} finally { if (locked) try { rmdirSync(lock); } catch { /* Preserve an unavailable lock; never recreate its parent. */ } }
|
||||||
}
|
}
|
||||||
export function status({ campaign,ledger }) {
|
export function status({ campaign,ledger }) {
|
||||||
const p=loadLedger(ledger); validateCampaign(p.campaign); checkBinding(p.binding,p.campaign);
|
const p=loadLedger(ledger); validateCampaign(p.campaign); checkBinding(p.binding,p.campaign);
|
||||||
@@ -216,6 +237,26 @@ export function status({ campaign,ledger }) {
|
|||||||
const totals=p.campaign.caps.map(cap=>({...cap,total:journal(ledger,cap).total}));
|
const totals=p.campaign.caps.map(cap=>({...cap,total:journal(ledger,cap).total}));
|
||||||
return { stopped:existsSync(join(ledger,'stop.json')),end:p.campaign.end,totals };
|
return { stopped:existsSync(join(ledger,'stop.json')),end:p.campaign.end,totals };
|
||||||
}
|
}
|
||||||
|
export async function serveBudget(opts, {interval=1000,signal,report=value=>console.log(JSON.stringify(value)),...deps}={}) {
|
||||||
|
checkTarget(opts); // Refuse an invalid initial target before entering the service loop.
|
||||||
|
let stopReason=null;
|
||||||
|
while(!signal?.aborted) {
|
||||||
|
try {
|
||||||
|
const result=await check(opts,{...deps,stopReason});
|
||||||
|
if(result.stopped) stopReason=result.reason;
|
||||||
|
report(result);
|
||||||
|
} catch(e) {
|
||||||
|
// Keep the owner service alive even if evidence storage or halt fails. A
|
||||||
|
// changed/foreign target is revalidated and refused on every later tick.
|
||||||
|
stopReason??=e.result?.reason ?? 'supervisor-check-failed';
|
||||||
|
report({...e.result,error:e.message});
|
||||||
|
}
|
||||||
|
if(!signal?.aborted) await new Promise(resolve=>{
|
||||||
|
const done=()=>{clearTimeout(timer);signal?.removeEventListener('abort',done);resolve();};
|
||||||
|
const timer=setTimeout(done,interval);signal?.addEventListener('abort',done,{once:true});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
export async function main(argv) {
|
export async function main(argv) {
|
||||||
const command=argv.shift(),o=options(argv,['--campaign','--ledger','--interval']);
|
const command=argv.shift(),o=options(argv,['--campaign','--ledger','--interval']);
|
||||||
const opts={campaign:o['--campaign'],ledger:o['--ledger']};
|
const opts={campaign:o['--campaign'],ledger:o['--ledger']};
|
||||||
@@ -224,11 +265,10 @@ export async function main(argv) {
|
|||||||
if (command==='check') return check(opts);
|
if (command==='check') return check(opts);
|
||||||
if (command!=='serve') fail('use init, check, serve or status');
|
if (command!=='serve') fail('use init, check, serve or status');
|
||||||
const interval=Number(o['--interval']??1); if (!Number.isFinite(interval) || interval<0.1 || interval>10) fail('interval must be 0.1..10 seconds');
|
const interval=Number(o['--interval']??1); if (!Number.isFinite(interval) || interval<0.1 || interval>10) fail('interval must be 0.1..10 seconds');
|
||||||
let running=true; for(const signal of ['SIGTERM','SIGINT']) process.once(signal,()=>{running=false;});
|
const controller=new AbortController();
|
||||||
while(running) {
|
const stop=()=>controller.abort();
|
||||||
// Reassert a latched halt on EVERY tick, so a later manual Resume cannot clear this campaign.
|
for(const signal of ['SIGTERM','SIGINT']) process.once(signal,stop);
|
||||||
const result=await check(opts); console.log(JSON.stringify(result));
|
try { await serveBudget(opts,{interval:interval*1000,signal:controller.signal}); }
|
||||||
await new Promise(r=>setTimeout(r,interval*1000));
|
finally { for(const signal of ['SIGTERM','SIGINT']) process.removeListener(signal,stop); }
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (isMain(import.meta.url)) main(process.argv.slice(2)).then(r=>{if(r!==undefined)console.log(JSON.stringify(r,null,2));}).catch(e=>{console.error(e.message);process.exitCode=1;});
|
if (isMain(import.meta.url)) main(process.argv.slice(2)).then(r=>{if(r!==undefined)console.log(JSON.stringify(r,null,2));}).catch(e=>{console.error(e.message);process.exitCode=1;});
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { mkdirSync,writeFileSync,readFileSync,readdirSync,unlinkSync,symlinkSync,renameSync,existsSync,lstatSync } from 'node:fs';
|
import { mkdirSync,writeFileSync,readFileSync,readdirSync,unlinkSync,symlinkSync,renameSync,existsSync,lstatSync,chmodSync,rmSync } from 'node:fs';
|
||||||
import { spawn } from 'node:child_process';
|
import { spawn } from 'node:child_process';
|
||||||
import { once } from 'node:events';
|
import { once } from 'node:events';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { initialize,check,status,readAccounting,supportedHalt,main,CUTOFF } from './budget-supervisor.mjs';
|
import { initialize,check,status,readAccounting,supportedHalt,main,serveBudget,CUTOFF } from './budget-supervisor.mjs';
|
||||||
import { fixture } from './test-support.mjs';
|
import { fixture,closeChild } from './test-support.mjs';
|
||||||
import { Store } from '../../bus/src/store.mjs';
|
import { Store } from '../../bus/src/store.mjs';
|
||||||
const NOW=Date.parse('2026-11-06T02:30:00.000Z');
|
const NOW=Date.parse('2026-11-06T02:30:00.000Z');
|
||||||
const row=(extra={})=>({at:'2026-11-06T02:29:00.000Z',run:'r123456789abc',turn:'t1',source:'transcript',instance:'assistant',provider:'proxy',route:'glm',model:'glm-5.3-flash',status:'known',input:1,output:1,cacheWrite:1,cacheRead:1,...extra});
|
const row=(extra={})=>({at:'2026-11-06T02:29:00.000Z',run:'r123456789abc',turn:'t1',source:'transcript',instance:'assistant',provider:'proxy',route:'glm',model:'glm-5.3-flash',status:'known',input:1,output:1,cacheWrite:1,cacheRead:1,...extra});
|
||||||
@@ -228,3 +228,115 @@ test('the authorized GLM and other-model maxima halt at the exact charged ceilin
|
|||||||
assert.equal(result.reason,'model-cap-reached');assert.equal(result.totals[0].total,cap.tokens);assert.equal(f.halts.length,1);
|
assert.equal(result.reason,'model-cap-reached');assert.equal(result.totals[0].total,cap.tokens);assert.equal(f.halts.length,1);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
for(const variant of ['renamed','missing','mode-0500','cutoff','latched-restart'])test(`unavailable ledger (${variant}) still counts a halt without recreating accounting`,async t=>{
|
||||||
|
const f=setup(t,1),offline=f.ledger+'-offline',deps={now:()=>NOW,halt:f.halt,read:()=>[row()]};
|
||||||
|
let priorLatch;
|
||||||
|
if(variant==='latched-restart') {
|
||||||
|
await check(f.opts,deps);priorLatch=readFileSync(join(f.ledger,'stop.json'),'utf8');
|
||||||
|
f.halts.length=0; // Model Resume and a fresh check, without any cached binding.
|
||||||
|
}
|
||||||
|
const before=readdirSync(f.ledger).map(n=>[n,readFileSync(join(f.ledger,n),'utf8')]);
|
||||||
|
if(variant==='mode-0500'){chmodSync(f.ledger,0o500);f.after(()=>chmodSync(f.ledger,0o700));}
|
||||||
|
else if(variant==='missing')rmSync(f.ledger,{recursive:true});
|
||||||
|
else renameSync(f.ledger,offline);
|
||||||
|
let running=true;
|
||||||
|
const attempts=variant==='latched-restart'?2:1;
|
||||||
|
for(let n=0;n<attempts;n++) {
|
||||||
|
await assert.rejects(check(f.opts,{...deps,now:()=>variant==='cutoff'?CUTOFF:NOW,halt:b=>{
|
||||||
|
assert.equal(running,true);running=false;return f.halt(b);
|
||||||
|
}}),e=>{
|
||||||
|
assert.deepEqual(e.result.storageErrors,['stop-latch-not-durable','stop-receipt-not-durable']);
|
||||||
|
assert.equal(e.result.halt.halted,true);return true;
|
||||||
|
});
|
||||||
|
assert.equal(running,false);running=true; // Model another Resume after restarted halt.
|
||||||
|
}
|
||||||
|
assert.equal(f.halts.length,attempts);assert.equal(f.halts[0].dataRoot,f.dataRoot);
|
||||||
|
if(variant==='mode-0500')assert.deepEqual(readdirSync(f.ledger).map(n=>[n,readFileSync(join(f.ledger,n),'utf8')]),before);
|
||||||
|
else assert.equal(existsSync(f.ledger),false);
|
||||||
|
if(priorLatch)assert.equal(readFileSync(join(offline,'stop.json'),'utf8'),priorLatch);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an unavailable ledger refuses invalid, foreign, changed or unpinned targets before halt',async t=>{
|
||||||
|
for(const variant of ['invalid','foreign','changed-root','changed-config','changed-campaign','missing-pin']) {
|
||||||
|
const f=setup(t,1);renameSync(f.ledger,f.ledger+'-offline');
|
||||||
|
const c=JSON.parse(readFileSync(f.config));
|
||||||
|
if(variant==='invalid')writeFileSync(f.config,'{');
|
||||||
|
if(variant==='foreign')writeFileSync(join(f.dataRoot,'instance.json'),JSON.stringify({instanceVersion:1,name:'unrelated'}));
|
||||||
|
if(variant==='changed-root') {
|
||||||
|
const other=join(f.root,'other');mkdirSync(other);writeFileSync(join(other,'instance.json'),JSON.stringify({instanceVersion:1,name:'ui797'}));
|
||||||
|
const store=new Store(other);f.after(()=>store.close());c.dataRoot=other;writeFileSync(f.config,JSON.stringify(c));
|
||||||
|
}
|
||||||
|
if(variant==='changed-config'){c.execution.model='other';writeFileSync(f.config,JSON.stringify(c));}
|
||||||
|
if(variant==='changed-campaign'){f.c.business='other-business';writeFileSync(f.campaign,JSON.stringify(f.c));}
|
||||||
|
if(variant==='missing-pin')unlinkSync(f.campaign+'.binding.json');
|
||||||
|
await assert.rejects(check(f.opts,{now:()=>NOW,halt:f.halt}));assert.equal(f.halts.length,0,variant);
|
||||||
|
assert.equal(existsSync(f.ledger),false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('an older intact ledger keeps its epoch without creating an owner binding during checks',async t=>{
|
||||||
|
const f=setup(t,100),manifest=readFileSync(join(f.ledger,'campaign.json'),'utf8');
|
||||||
|
unlinkSync(f.campaign+'.binding.json');
|
||||||
|
assert.equal((await check(f.opts,{now:()=>NOW,read:()=>[row()],halt:f.halt})).stopped,false);
|
||||||
|
assert.equal(readFileSync(join(f.ledger,'campaign.json'),'utf8'),manifest);
|
||||||
|
assert.equal(existsSync(f.campaign+'.binding.json'),false);assert.equal(f.halts.length,0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('serve survives unavailable storage, reasserts after Resume each tick and preserves its in-memory stop on recovery',async t=>{
|
||||||
|
const f=setup(t,100),offline=f.ledger+'-offline',controller=new AbortController(),reports=[];
|
||||||
|
await check(f.opts,{now:()=>NOW,halt:f.halt,read:()=>[]});
|
||||||
|
const epoch=readFileSync(join(f.ledger,'campaign.json'),'utf8');renameSync(f.ledger,offline);
|
||||||
|
let running=true;
|
||||||
|
await serveBudget(f.opts,{interval:1,signal:controller.signal,now:()=>NOW,read:()=>[],halt:b=>{
|
||||||
|
assert.equal(running,true);running=false;return f.halt(b);
|
||||||
|
},report:r=>{
|
||||||
|
reports.push(r);assert.equal(r.stopped,true);assert.equal(r.halt.halted,true);
|
||||||
|
running=true; // Model Resume between every tick.
|
||||||
|
if(reports.length===2)renameSync(offline,f.ledger);
|
||||||
|
if(reports.length===3)controller.abort();
|
||||||
|
}});
|
||||||
|
assert.equal(f.halts.length,3);assert.equal(reports.length,3);
|
||||||
|
for(const r of reports.slice(0,2))assert.deepEqual(r.storageErrors,['stop-latch-not-durable','stop-receipt-not-durable']);
|
||||||
|
assert.equal(readFileSync(join(f.ledger,'campaign.json'),'utf8'),epoch); // Never initialize a new epoch.
|
||||||
|
assert.equal(JSON.parse(readFileSync(join(f.ledger,'stop.json'))).reason,'ledger-storage-unavailable');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('serve keeps retrying a refused halt and refuses targeting if the config changes between ticks',async t=>{
|
||||||
|
const f=setup(t,1),controller=new AbortController(),reports=[];
|
||||||
|
await serveBudget(f.opts,{interval:1,signal:controller.signal,now:()=>NOW,read:()=>[row()],halt:b=>{
|
||||||
|
f.halts.push(b);throw new Error('modelled halt refusal');
|
||||||
|
},report:r=>{
|
||||||
|
reports.push(r);
|
||||||
|
if(reports.length===2){const c=JSON.parse(readFileSync(f.config));c.execution.model='changed';writeFileSync(f.config,JSON.stringify(c));}
|
||||||
|
if(reports.length===4)controller.abort();
|
||||||
|
}});
|
||||||
|
assert.equal(f.halts.length,2);assert.equal(reports.length,4);
|
||||||
|
for(const r of reports.slice(0,2))assert.match(r.error,/supported-halt-refused/);
|
||||||
|
for(const r of reports.slice(2))assert.match(r.error,/campaign-instance-changed/);
|
||||||
|
assert.equal(existsSync(join(f.ledger,'stop.json')),true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('real serve CLI stays alive and attempts supported halt every tick with a missing ledger',async t=>{
|
||||||
|
const f=setup(t,1);renameSync(f.ledger,f.ledger+'-offline');
|
||||||
|
const child=spawn(process.execPath,['packages/cli/live/budget-supervisor.mjs','serve','--campaign',f.campaign,'--ledger',f.ledger,'--interval','0.1'],{
|
||||||
|
env:{...process.env,MOSAIC_AGENT_RUN:'test-unavailable-ledger'},stdio:['ignore','pipe','pipe'],
|
||||||
|
});
|
||||||
|
f.after(()=>closeChild(child));
|
||||||
|
let text='',errors='';
|
||||||
|
child.stderr.on('data',chunk=>errors+=chunk);
|
||||||
|
const reports=[];
|
||||||
|
child.stdout.on('data',chunk=>{
|
||||||
|
text+=chunk;let end;
|
||||||
|
while((end=text.indexOf('\n'))!==-1){reports.push(JSON.parse(text.slice(0,end)));text=text.slice(end+1);}
|
||||||
|
if(reports.length>=3)child.kill('SIGTERM');
|
||||||
|
});
|
||||||
|
const timer=setTimeout(()=>child.kill('SIGKILL'),20000);
|
||||||
|
const [code,signal]=await once(child,'close');clearTimeout(timer);
|
||||||
|
assert.equal(signal,null,errors);assert.equal(code,0,errors);assert.ok(reports.length>=3,errors);
|
||||||
|
for(const r of reports) {
|
||||||
|
assert.equal(r.stopped,true);assert.equal(r.haltError,'supported-halt-refused-or-unknown');
|
||||||
|
assert.deepEqual(r.storageErrors,['stop-latch-not-durable','stop-receipt-not-durable']);
|
||||||
|
}
|
||||||
|
assert.equal(existsSync(f.ledger),false);
|
||||||
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
// Focused test fixtures only; runtime CLIs never import this file.
|
// Focused test fixtures only; runtime CLIs never import this file.
|
||||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync,readFileSync,readdirSync } from 'node:fs';
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync,readFileSync,readdirSync,readlinkSync } from 'node:fs';
|
||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
export function fixture(t) {
|
export function fixture(t) {
|
||||||
@@ -46,7 +46,15 @@ export function ownedProcesses(root) {
|
|||||||
} catch { return []; }
|
} catch { return []; }
|
||||||
});
|
});
|
||||||
const selected=new Set(processes.filter(p=>p.args.some(a=>a===root || a.startsWith(root+'/') || a.startsWith('--user-data-dir='+root+'/'))).map(p=>p.pid));
|
const selected=new Set(processes.filter(p=>p.args.some(a=>a===root || a.startsWith(root+'/') || a.startsWith('--user-data-dir='+root+'/'))).map(p=>p.pid));
|
||||||
for(const p of processes) if(selected.has(p.pid)) for(const a of p.args) if(a.startsWith('--crashpad-handler-pid=')) selected.add(Number(a.split('=')[1]));
|
// Chromium reports PIDs in its own namespace. Never mistake one for a host
|
||||||
|
// PID (which could belong to another seat) when tracking orphan crash handlers.
|
||||||
|
for(const p of processes) if(selected.has(p.pid)) for(const a of p.args) if(a.startsWith('--crashpad-handler-pid=')) {
|
||||||
|
for(const candidate of processes) try {
|
||||||
|
if(readlinkSync(`/proc/${candidate.pid}/ns/pid`)!==readlinkSync(`/proc/${p.pid}/ns/pid`))continue;
|
||||||
|
const ids=readFileSync(`/proc/${candidate.pid}/status`,'utf8').match(/^NSpid:\s+(.+)$/m)?.[1].trim().split(/\s+/);
|
||||||
|
if(Number(ids?.at(-1))===Number(a.split('=')[1]))selected.add(candidate.pid);
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
let size;
|
let size;
|
||||||
do {size=selected.size;for(const p of processes) if(selected.has(p.ppid))selected.add(p.pid);} while(size!==selected.size);
|
do {size=selected.size;for(const p of processes) if(selected.has(p.ppid))selected.add(p.pid);} while(size!==selected.size);
|
||||||
return processes.filter(p=>selected.has(p.pid));
|
return processes.filter(p=>selected.has(p.pid));
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { createServer, createConnection } from 'node:net';
|
|||||||
import {createServer as httpServer,request as httpRequest} from 'node:http';
|
import {createServer as httpServer,request as httpRequest} from 'node:http';
|
||||||
import { once } from 'node:events';
|
import { once } from 'node:events';
|
||||||
import { createHash, X509Certificate } from 'node:crypto';
|
import { createHash, X509Certificate } from 'node:crypto';
|
||||||
import { appendFileSync, chmodSync, existsSync, readdirSync, readFileSync, unlinkSync } from 'node:fs';
|
import { appendFileSync, chmodSync, existsSync, readdirSync, readFileSync, readlinkSync, unlinkSync } from 'node:fs';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { homedir } from 'node:os';
|
import { homedir } from 'node:os';
|
||||||
@@ -21,7 +21,7 @@ export function testURL(value, origin = null) {
|
|||||||
if (origin && u.origin !== origin) fail('navigation must stay on the selected test origin');
|
if (origin && u.origin !== origin) fail('navigation must stay on the selected test origin');
|
||||||
return u;
|
return u;
|
||||||
}
|
}
|
||||||
async function chromium(evidence, ca, proxyPort) {
|
async function chromium(evidence, ca, proxy) {
|
||||||
const cert = new X509Certificate(readPrivate(ca));
|
const cert = new X509Certificate(readPrivate(ca));
|
||||||
if (!cert.ca || Date.parse(cert.validTo) <= Date.now() || Date.parse(cert.validFrom) > Date.now()) fail('valid private test CA required');
|
if (!cert.ca || Date.parse(cert.validTo) <= Date.now() || Date.parse(cert.validFrom) > Date.now()) fail('valid private test CA required');
|
||||||
const xdg = join(evidence, 'xdg'); privateDir(xdg, true);
|
const xdg = join(evidence, 'xdg'); privateDir(xdg, true);
|
||||||
@@ -42,10 +42,16 @@ async function chromium(evidence, ca, proxyPort) {
|
|||||||
const executable = process.env.CHROMIUM || '/usr/bin/chromium';
|
const executable = process.env.CHROMIUM || '/usr/bin/chromium';
|
||||||
const chromeArgs = ['--headless','--no-first-run','--no-default-browser-check','--disable-dev-shm-usage',
|
const chromeArgs = ['--headless','--no-first-run','--no-default-browser-check','--disable-dev-shm-usage',
|
||||||
'--disable-background-networking','--disable-component-update','--disable-sync','--disable-quic',
|
'--disable-background-networking','--disable-component-update','--disable-sync','--disable-quic',
|
||||||
`--proxy-server=http://127.0.0.1:${proxyPort}`,'--proxy-bypass-list=<-loopback>',
|
`--proxy-server=http://127.0.0.1:${proxy.port}`,'--proxy-bypass-list=<-loopback>',
|
||||||
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp','--remote-debugging-pipe',
|
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp','--remote-debugging-pipe',
|
||||||
`--user-data-dir=${profile}`];
|
`--user-data-dir=${profile}`];
|
||||||
const child = spawn(legacy ? '/usr/bin/bwrap' : executable, legacy ? ['--die-with-parent','--bind','/','/','--dev-bind','/dev','/dev','--proc','/proc','--bind',nss,legacyNss,'--',executable,...chromeArgs] : chromeArgs,
|
// Bubblewrap brings up the new namespace's own loopback. There is no veth,
|
||||||
|
// route, DNS forwarder or host-network FD. Only the fixed proxy has a relay.
|
||||||
|
const child = spawn('/usr/bin/bwrap', ['--die-with-parent','--unshare-net','--unshare-pid',
|
||||||
|
'--bind','/','/','--dev-bind','/dev','/dev','--proc','/proc',
|
||||||
|
...(legacy ? ['--bind',nss,legacyNss] : []),'--',process.execPath,
|
||||||
|
fileURLToPath(new URL('./browser-network.mjs',import.meta.url)),readlinkSync('/proc/self/ns/net'),
|
||||||
|
proxy.socket,String(proxy.port),executable,...chromeArgs],
|
||||||
{ env:{...process.env,XDG_DATA_HOME:xdg}, stdio: ['ignore','ignore','ignore','pipe','pipe'] });
|
{ env:{...process.env,XDG_DATA_HOME:xdg}, stdio: ['ignore','ignore','ignore','pipe','pipe'] });
|
||||||
let id = 0, buffer = '';
|
let id = 0, buffer = '';
|
||||||
const pending = new Map(), events = new Map();
|
const pending = new Map(), events = new Map();
|
||||||
@@ -67,8 +73,15 @@ async function chromium(evidence, ca, proxyPort) {
|
|||||||
child.stdio[3].write(JSON.stringify({ id:key, method, params, ...(sessionId ? { sessionId } : {}) }) + '\0');
|
child.stdio[3].write(JSON.stringify({ id:key, method, params, ...(sessionId ? { sessionId } : {}) }) + '\0');
|
||||||
});
|
});
|
||||||
const on = (method, f) => { if (!events.has(method)) events.set(method,new Set()); events.get(method).add(f); };
|
const on = (method, f) => { if (!events.has(method)) events.set(method,new Set()); events.get(method).add(f); };
|
||||||
const { targetId } = await send('Target.createTarget', { url:'about:blank' });
|
let targetId, sessionId;
|
||||||
const { sessionId } = await send('Target.attachToTarget', { targetId, flatten:true });
|
try {
|
||||||
|
({ targetId } = await send('Target.createTarget', { url:'about:blank' }));
|
||||||
|
({ sessionId } = await send('Target.attachToTarget', { targetId, flatten:true }));
|
||||||
|
} catch (e) {
|
||||||
|
const closed=once(child,'close');
|
||||||
|
if (child.exitCode===null && child.signalCode===null) { child.kill('SIGKILL'); await closed; }
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
const call = (method, params) => send(method, params, sessionId);
|
const call = (method, params) => send(method, params, sessionId);
|
||||||
let closing;
|
let closing;
|
||||||
return { call, send, on, child, close() {
|
return { call, send, on, child, close() {
|
||||||
@@ -81,7 +94,7 @@ async function chromium(evidence, ca, proxyPort) {
|
|||||||
}
|
}
|
||||||
// External transport boundary: Chromium must use this proxy even for loopback.
|
// External transport boundary: Chromium must use this proxy even for loopback.
|
||||||
// It never resolves/opens a socket for an unapproved destination. CDP is defense in depth.
|
// It never resolves/opens a socket for an unapproved destination. CDP is defense in depth.
|
||||||
async function originProxy(origin) {
|
async function originProxy(origin, evidence) {
|
||||||
const allowed=new URL(origin),sockets=new Set();
|
const allowed=new URL(origin),sockets=new Set();
|
||||||
const track=socket=>{sockets.add(socket);socket.once('close',()=>sockets.delete(socket));return socket;};
|
const track=socket=>{sockets.add(socket);socket.once('close',()=>sockets.delete(socket));return socket;};
|
||||||
const server=httpServer((req,res)=>{
|
const server=httpServer((req,res)=>{
|
||||||
@@ -106,19 +119,36 @@ async function originProxy(origin) {
|
|||||||
peer.once('close',()=>upstream.destroy());upstream.once('close',()=>peer.destroy());
|
peer.once('close',()=>upstream.destroy());upstream.once('close',()=>peer.destroy());
|
||||||
});
|
});
|
||||||
await new Promise((resolve,reject)=>{server.once('error',reject);server.listen(0,'127.0.0.1',resolve);});
|
await new Promise((resolve,reject)=>{server.once('error',reject);server.listen(0,'127.0.0.1',resolve);});
|
||||||
|
const socket=join(evidence,'proxy.sock');
|
||||||
|
const bridge=createServer(peer=>{
|
||||||
|
track(peer);
|
||||||
|
const upstream=track(createConnection({host:'127.0.0.1',port:server.address().port}));
|
||||||
|
peer.on('error',()=>upstream.destroy());upstream.on('error',()=>peer.destroy());
|
||||||
|
peer.once('close',()=>upstream.destroy());upstream.once('close',()=>peer.destroy());
|
||||||
|
peer.pipe(upstream);upstream.pipe(peer);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
if(Buffer.byteLength(socket)>100) fail('evidence path too long for Unix socket');
|
||||||
|
await new Promise((resolve,reject)=>{bridge.once('error',reject);bridge.listen(socket,resolve);});
|
||||||
|
chmodSync(socket,0o600);
|
||||||
|
} catch(e) {
|
||||||
|
const closed=Promise.all([new Promise(resolve=>bridge.close(resolve)),new Promise(resolve=>server.close(resolve))]);
|
||||||
|
for(const peer of sockets)peer.destroy();await closed;throw e;
|
||||||
|
}
|
||||||
let closing;
|
let closing;
|
||||||
return {port:server.address().port,close(){return closing??=(async()=>{
|
return {port:server.address().port,socket,close(){return closing??=(async()=>{
|
||||||
const closed=new Promise(resolve=>server.close(resolve));for(const socket of sockets)socket.destroy();await closed;
|
const closed=Promise.all([new Promise(resolve=>bridge.close(resolve)),new Promise(resolve=>server.close(resolve))]);
|
||||||
|
for(const socket of sockets)socket.destroy();await closed;
|
||||||
})();}};
|
})();}};
|
||||||
}
|
}
|
||||||
export async function daemon(evidence) {
|
export async function daemon(evidence) {
|
||||||
privateDir(evidence);
|
privateDir(evidence);
|
||||||
const state = JSON.parse(readPrivate(join(evidence, 'browser.json'))), origin = testURL(state.origin).origin;
|
const state = JSON.parse(readPrivate(join(evidence, 'browser.json'))), origin = testURL(state.origin).origin;
|
||||||
let blocked=false;
|
let blocked=false;
|
||||||
const proxy=await originProxy(origin);
|
const proxy=await originProxy(origin,evidence);
|
||||||
let b;
|
let b;
|
||||||
try {
|
try {
|
||||||
b = await chromium(evidence, state.ca,proxy.port);
|
b = await chromium(evidence, state.ca,proxy);
|
||||||
const closeChrome=b.close;
|
const closeChrome=b.close;
|
||||||
b.close=async()=>{try{await closeChrome();}finally{await proxy.close();}};
|
b.close=async()=>{try{await closeChrome();}finally{await proxy.close();}};
|
||||||
let sequence = 0, generation = 0, refs = new Map(), detail = {};
|
let sequence = 0, generation = 0, refs = new Map(), detail = {};
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import test,{after} from 'node:test';
|
import test,{after} from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { createServer } from 'node:http';
|
import { createServer } from 'node:http';
|
||||||
import { readFileSync, readdirSync, existsSync,writeFileSync } from 'node:fs';
|
import { createServer as tcpServer } from 'node:net';
|
||||||
|
import { createSocket } from 'node:dgram';
|
||||||
|
import { readFileSync, readdirSync, existsSync,writeFileSync,lstatSync,readlinkSync } from 'node:fs';
|
||||||
import { join } from 'node:path';
|
import { join } from 'node:path';
|
||||||
import { spawn } from 'node:child_process';
|
import { spawn } from 'node:child_process';
|
||||||
import { start,callBrowser,testURL } from './ui-browser.mjs';
|
import { start,callBrowser,testURL } from './ui-browser.mjs';
|
||||||
@@ -135,6 +137,42 @@ test('intentional failure after browser startup',async t=>{
|
|||||||
assert.equal(existsSync(root),false,output);assert.deepEqual(ownedProcesses(root),[]);
|
assert.equal(existsSync(root),false,output);assert.deepEqual(ownedProcesses(root),[]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
for(const origin of ['http://127.0.0.1:17440','https://127.0.0.1:17443'])test(`Chromium ${origin} emits no STUN/TURN UDP or TCP to host loopback listeners`,async t=>{
|
||||||
|
const targets=[],counters=[];
|
||||||
|
// Each protocol/address has its own independent host listener, outside bwrap.
|
||||||
|
for(const host of ['127.0.0.1','::1','127.0.0.2']) {
|
||||||
|
const address=host.includes(':')?`[${host}]`:host;
|
||||||
|
for(const scheme of ['stun','turn']) {
|
||||||
|
const udp=createSocket(host.includes(':')?'udp6':'udp4'),count={host,scheme,datagrams:0};counters.push(count);
|
||||||
|
udp.on('message',()=>count.datagrams++);
|
||||||
|
await new Promise((resolve,reject)=>{udp.once('error',reject);udp.bind(0,host,resolve);});
|
||||||
|
count.port=udp.address().port;
|
||||||
|
t.after(()=>new Promise(resolve=>udp.close(resolve)));
|
||||||
|
targets.push({urls:`${scheme}:${address}:${udp.address().port}${scheme==='turn'?'?transport=udp':''}`,username:'test',credential:'test'});
|
||||||
|
}
|
||||||
|
const tcp=tcpServer(socket=>{count.connections++;socket.destroy();}),count={host,connections:0};counters.push(count);
|
||||||
|
await new Promise((resolve,reject)=>{tcp.once('error',reject);tcp.listen(host==='127.0.0.1'?17442:0,host,resolve);});
|
||||||
|
count.port=tcp.address().port;
|
||||||
|
t.after(()=>new Promise(resolve=>tcp.close(resolve)));
|
||||||
|
targets.push({urls:`turn:${address}:${tcp.address().port}?transport=tcp`,username:'test',credential:'test'});
|
||||||
|
}
|
||||||
|
const f=await setup(t,`<!doctype html><h1>ICE boundary</h1><button onclick="go()">Send ICE</button><output>idle</output><script>
|
||||||
|
const pcs=[];async function go(){for(const server of ${JSON.stringify(targets)}){const pc=new RTCPeerConnection({iceServers:[server]});pcs.push(pc);pc.createDataChannel('probe');await pc.setLocalDescription(await pc.createOffer());}document.querySelector('output').textContent='9 ICE agents started';}</script>`);
|
||||||
|
if(origin.startsWith('https:')){const id=await serve(f.identity);f.after(()=>closeServer(id));}
|
||||||
|
await start({evidence:f.evidence,origin,ca:join(f.identity,'ca.pem')});
|
||||||
|
assert.equal(lstatSync(join(f.evidence,'browser.sock')).mode&0o777,0o600);
|
||||||
|
assert.equal(lstatSync(join(f.evidence,'proxy.sock')).mode&0o777,0o600);
|
||||||
|
const processes=ownedProcesses(f.root).filter(p=>p.args.includes('--user-data-dir='+f.evidence+'/profile'));
|
||||||
|
assert.ok(processes.length);
|
||||||
|
for(const p of processes.filter(p=>!p.args.includes('--unshare-net')))assert.notEqual(readlinkSync(`/proc/${p.pid}/ns/net`),readlinkSync('/proc/self/ns/net'));
|
||||||
|
await callBrowser(f.evidence,'click',['--text','Send ICE']);
|
||||||
|
await callBrowser(f.evidence,'wait',['--text','9 ICE agents started','--timeout','5']);
|
||||||
|
await pause(2500);
|
||||||
|
await callBrowser(f.evidence,'stop');
|
||||||
|
t.diagnostic(JSON.stringify({origin,listeners:counters}));
|
||||||
|
for(const count of counters)assert.equal(count.datagrams??count.connections,0,JSON.stringify(count));
|
||||||
|
});
|
||||||
|
|
||||||
after(async()=>{
|
after(async()=>{
|
||||||
const {createConnection}=await import('node:net');
|
const {createConnection}=await import('node:net');
|
||||||
for(const port of [17440,17442,17443]) await new Promise((resolve,reject)=>{
|
for(const port of [17440,17442,17443]) await new Promise((resolve,reject)=>{
|
||||||
|
|||||||
Reference in new issue
Block a user