/** * tools/matrix-presence-harness/validate.ts * * *** DEV harness — LOCAL SANDBOX ONLY. Never point at live infra. *** * * End-to-end proof of RFC-001 P1 acceptance criteria against the local dev * Synapse (infra/matrix): * * A2 provision >=3 agents, they heartbeat, all show ONLINE in the fleet * presence room. * A3 hard-kill (SIGKILL) one agent's process; assert it flips to OFFLINE * within dark_threshold, DETERMINISTICALLY (heartbeat-age based, not * native-presence-timeout), while the survivors stay ONLINE. * A4 dump the human-readable fleet liveness board. * * Exits 0 on success, 1 on any failed assertion. * ESM / NodeNext: .js import extensions. */ import { spawn, type ChildProcess } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { dirname, resolve } from 'node:path'; import { FleetLivenessReader, MinimalMatrixClient, type AgentLiveness, type LivenessPolicy, } from '@mosaicstack/comms'; import { provision } from './provision.js'; const HERE = dirname(fileURLToPath(import.meta.url)); const env = (k: string, fallback?: string): string => { const v = process.env[k] ?? fallback; if (v === undefined) throw new Error(`missing env ${k}`); return v; }; const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); const board = (rows: AgentLiveness[]): Record => Object.fromEntries(rows.map((r) => [r.slug, r.status])); async function main(): Promise { const homeserverUrl = env('MATRIX_CS_URL'); const asToken = env('MOSAIC_AS_TOKEN'); const hsToken = env('MOSAIC_HS_TOKEN', 'unused-in-p1'); const serverName = env('MATRIX_SERVER_NAME'); const intervalMs = Number(env('HEARTBEAT_INTERVAL_MS', '1000')); const missTolerance = Number(env('MISS_TOLERANCE', '2')); const darkThresholdMs = Number(env('DARK_THRESHOLD_MS', '6000')); const slugs = env('AGENT_SLUGS', 'alpha,bravo,charlie').split(','); const victim = env('VICTIM_SLUG', 'charlie'); const policy: LivenessPolicy = { heartbeatIntervalMs: intervalMs, missTolerance, darkThresholdMs, }; const failures: string[] = []; const assert = (ok: boolean, msg: string): void => { console.log(` ${ok ? 'PASS' : 'FAIL'} ${msg}`); if (!ok) failures.push(msg); }; // ── Provision (register agents, create fleet room, join) ────────────────── console.log(`\n[validate] provisioning ${slugs.length} agents + fleet presence room...`); const { roomId, senderUserId } = await provision({ homeserverUrl, serverName, asToken, hsToken, agentSlugs: slugs, }); console.log(`[validate] fleet room ${roomId} (sender ${senderUserId})`); // Reader masquerades as the AS sender (a room member) to read heartbeats. const reader = new FleetLivenessReader({ client: new MinimalMatrixClient({ homeserverUrl, accessToken: asToken, actAsUserId: senderUserId, }), roomId, policy, }); // ── Spawn each agent as its own OS process ──────────────────────────────── const procs = new Map(); for (const slug of slugs) { // `--import tsx` runs agent-proc.ts IN-PROCESS (no tsx grandchild), so the // spawned pid IS the agent — a SIGKILL to it is a true hard-kill (A3). // cwd=HERE so the `tsx` specifier resolves against the harness node_modules. const child = spawn('node', ['--import', 'tsx', resolve(HERE, 'agent-proc.ts')], { cwd: HERE, env: { ...process.env, AGENT_SLUG: slug, FLEET_ROOM_ID: roomId, MATRIX_CS_URL: homeserverUrl, MOSAIC_AS_TOKEN: asToken, MATRIX_SERVER_NAME: serverName, HEARTBEAT_INTERVAL_MS: String(intervalMs), MISS_TOLERANCE: String(missTolerance), DARK_THRESHOLD_MS: String(darkThresholdMs), }, stdio: ['ignore', 'inherit', 'inherit'], }); procs.set(slug, child); } const hardKill = (pid: number): void => { try { process.kill(pid, 'SIGKILL'); // pid is the in-process agent — a true kill } catch { /* already gone */ } }; const cleanup = (): void => { for (const [, c] of procs) { if (c.pid) hardKill(c.pid); } }; process.on('exit', cleanup); try { // ── A2: let a few beats flow, assert all online ───────────────────────── console.log(`\n[validate] A2 — waiting for heartbeats (${intervalMs}ms interval)...`); await sleep(intervalMs * 4 + 1000); const a2 = await reader.read(); console.log('[validate] board:', board(a2)); for (const slug of slugs) { assert(a2.find((r) => r.slug === slug)?.status === 'online', `A2 ${slug} is online`); } assert(a2.length >= 3, 'A2 >=3 agents present in fleet room'); // ── A4: human-readable board ──────────────────────────────────────────── console.log('\n[validate] A4 — fleet liveness board (human view):'); console.log(await reader.formatBoard()); // ── A3: hard-kill the victim, measure flip-to-offline latency ─────────── const victimProc = procs.get(victim); if (!victimProc?.pid) throw new Error(`no pid for victim ${victim}`); console.log( `\n[validate] A3 — SIGKILL ${victim} (pid ${victimProc.pid}); dark_threshold=${darkThresholdMs}ms`, ); const killTs = Date.now(); hardKill(victimProc.pid); let victimOffline: AgentLiveness | undefined; let detectMs = 0; const deadline = killTs + darkThresholdMs + 6000; while (Date.now() < deadline) { await sleep(400); const rows = await reader.read(); const v = rows.find((r) => r.slug === victim); const elapsed = Date.now() - killTs; if (v) { console.log( ` t+${String(elapsed).padStart(5)}ms ${victim}=${v.status} (age=${v.ageMs}ms) ` + `survivors=${slugs .filter((s) => s !== victim) .map((s) => `${s}:${rows.find((r) => r.slug === s)?.status}`) .join(',')}`, ); if (v.status === 'offline') { victimOffline = v; detectMs = elapsed; break; } } } assert(!!victimOffline, `A3 ${victim} flipped to offline after kill`); if (victimOffline) { // Deterministic: the flip is driven by heartbeat age reaching dark_threshold. assert( victimOffline.ageMs >= darkThresholdMs, `A3 flip is heartbeat-deterministic (age ${victimOffline.ageMs}ms >= dark_threshold ${darkThresholdMs}ms)`, ); assert( detectMs <= darkThresholdMs + 2000, `A3 detected within dark_threshold + poll slack (detected at t+${detectMs}ms)`, ); } // Survivors unaffected. const finalRows = await reader.read(); for (const slug of slugs.filter((s) => s !== victim)) { assert( finalRows.find((r) => r.slug === slug)?.status === 'online', `A3 survivor ${slug} still online`, ); } console.log('\n[validate] final board after kill:'); console.log(await reader.formatBoard()); } finally { cleanup(); } console.log( `\n[validate] ${failures.length === 0 ? 'ALL PASSED ✅' : `FAILED ❌ (${failures.length})`}`, ); process.exit(failures.length === 0 ? 0 : 1); } main().catch((err) => { console.error('[validate] fatal:', err); process.exit(1); });