stack/tools/matrix-presence-harness/validate.ts

/**
 * tools/matrix-presence-harness/validate.ts
 *
 *   *** DEV harness — LOCAL SANDBOX ONLY. Never point at live infra. ***
 *
 * End-to-end proof of RFC-001 P1 acceptance criteria against the local dev
 * Synapse (infra/matrix):
 *
 *   A2  provision >=3 agents, they heartbeat, all show ONLINE in the fleet
 *       presence room.
 *   A3  hard-kill (SIGKILL) one agent's process; assert it flips to OFFLINE
 *       within dark_threshold, DETERMINISTICALLY (heartbeat-age based, not
 *       native-presence-timeout), while the survivors stay ONLINE.
 *   A4  dump the human-readable fleet liveness board.
 *
 * Exits 0 on success, 1 on any failed assertion.
 * ESM / NodeNext: .js import extensions.
 */
import { spawn, type ChildProcess } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';

import {
  FleetLivenessReader,
  MinimalMatrixClient,
  type AgentLiveness,
  type LivenessPolicy,
} from '@mosaicstack/comms';

import { provision } from './provision.js';

const HERE = dirname(fileURLToPath(import.meta.url));

const env = (k: string, fallback?: string): string => {
  const v = process.env[k] ?? fallback;
  if (v === undefined) throw new Error(`missing env ${k}`);
  return v;
};

const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
const board = (rows: AgentLiveness[]): Record<string, string> =>
  Object.fromEntries(rows.map((r) => [r.slug, r.status]));

async function main(): Promise<void> {
  const homeserverUrl = env('MATRIX_CS_URL');
  const asToken = env('MOSAIC_AS_TOKEN');
  const hsToken = env('MOSAIC_HS_TOKEN', 'unused-in-p1');
  const serverName = env('MATRIX_SERVER_NAME');
  const intervalMs = Number(env('HEARTBEAT_INTERVAL_MS', '1000'));
  const missTolerance = Number(env('MISS_TOLERANCE', '2'));
  const darkThresholdMs = Number(env('DARK_THRESHOLD_MS', '6000'));
  const slugs = env('AGENT_SLUGS', 'alpha,bravo,charlie').split(',');
  const victim = env('VICTIM_SLUG', 'charlie');

  const policy: LivenessPolicy = {
    heartbeatIntervalMs: intervalMs,
    missTolerance,
    darkThresholdMs,
  };
  const failures: string[] = [];
  const assert = (ok: boolean, msg: string): void => {
    console.log(`  ${ok ? 'PASS' : 'FAIL'}  ${msg}`);
    if (!ok) failures.push(msg);
  };

  // ── Provision (register agents, create fleet room, join) ──────────────────
  console.log(`\n[validate] provisioning ${slugs.length} agents + fleet presence room...`);
  const { roomId, senderUserId } = await provision({
    homeserverUrl,
    serverName,
    asToken,
    hsToken,
    agentSlugs: slugs,
  });
  console.log(`[validate] fleet room ${roomId} (sender ${senderUserId})`);

  // Reader masquerades as the AS sender (a room member) to read heartbeats.
  const reader = new FleetLivenessReader({
    client: new MinimalMatrixClient({
      homeserverUrl,
      accessToken: asToken,
      actAsUserId: senderUserId,
    }),
    roomId,
    policy,
  });

  // ── Spawn each agent as its own OS process ────────────────────────────────
  const procs = new Map<string, ChildProcess>();
  for (const slug of slugs) {
    // `--import tsx` runs agent-proc.ts IN-PROCESS (no tsx grandchild), so the
    // spawned pid IS the agent — a SIGKILL to it is a true hard-kill (A3).
    // cwd=HERE so the `tsx` specifier resolves against the harness node_modules.
    const child = spawn('node', ['--import', 'tsx', resolve(HERE, 'agent-proc.ts')], {
      cwd: HERE,
      env: {
        ...process.env,
        AGENT_SLUG: slug,
        FLEET_ROOM_ID: roomId,
        MATRIX_CS_URL: homeserverUrl,
        MOSAIC_AS_TOKEN: asToken,
        MATRIX_SERVER_NAME: serverName,
        HEARTBEAT_INTERVAL_MS: String(intervalMs),
        MISS_TOLERANCE: String(missTolerance),
        DARK_THRESHOLD_MS: String(darkThresholdMs),
      },
      stdio: ['ignore', 'inherit', 'inherit'],
    });
    procs.set(slug, child);
  }

  const hardKill = (pid: number): void => {
    try {
      process.kill(pid, 'SIGKILL'); // pid is the in-process agent — a true kill
    } catch {
      /* already gone */
    }
  };
  const cleanup = (): void => {
    for (const [, c] of procs) {
      if (c.pid) hardKill(c.pid);
    }
  };
  process.on('exit', cleanup);

  try {
    // ── A2: let a few beats flow, assert all online ─────────────────────────
    console.log(`\n[validate] A2 — waiting for heartbeats (${intervalMs}ms interval)...`);
    await sleep(intervalMs * 4 + 1000);
    const a2 = await reader.read();
    console.log('[validate] board:', board(a2));
    for (const slug of slugs) {
      assert(a2.find((r) => r.slug === slug)?.status === 'online', `A2 ${slug} is online`);
    }
    assert(a2.length >= 3, 'A2 >=3 agents present in fleet room');

    // ── A4: human-readable board ────────────────────────────────────────────
    console.log('\n[validate] A4 — fleet liveness board (human view):');
    console.log(await reader.formatBoard());

    // ── A3: hard-kill the victim, measure flip-to-offline latency ───────────
    const victimProc = procs.get(victim);
    if (!victimProc?.pid) throw new Error(`no pid for victim ${victim}`);
    console.log(
      `\n[validate] A3 — SIGKILL ${victim} (pid ${victimProc.pid}); dark_threshold=${darkThresholdMs}ms`,
    );
    const killTs = Date.now();
    hardKill(victimProc.pid);

    let victimOffline: AgentLiveness | undefined;
    let detectMs = 0;
    const deadline = killTs + darkThresholdMs + 6000;
    while (Date.now() < deadline) {
      await sleep(400);
      const rows = await reader.read();
      const v = rows.find((r) => r.slug === victim);
      const elapsed = Date.now() - killTs;
      if (v) {
        console.log(
          `  t+${String(elapsed).padStart(5)}ms  ${victim}=${v.status} (age=${v.ageMs}ms)  ` +
            `survivors=${slugs
              .filter((s) => s !== victim)
              .map((s) => `${s}:${rows.find((r) => r.slug === s)?.status}`)
              .join(',')}`,
        );
        if (v.status === 'offline') {
          victimOffline = v;
          detectMs = elapsed;
          break;
        }
      }
    }

    assert(!!victimOffline, `A3 ${victim} flipped to offline after kill`);
    if (victimOffline) {
      // Deterministic: the flip is driven by heartbeat age reaching dark_threshold.
      assert(
        victimOffline.ageMs >= darkThresholdMs,
        `A3 flip is heartbeat-deterministic (age ${victimOffline.ageMs}ms >= dark_threshold ${darkThresholdMs}ms)`,
      );
      assert(
        detectMs <= darkThresholdMs + 2000,
        `A3 detected within dark_threshold + poll slack (detected at t+${detectMs}ms)`,
      );
    }
    // Survivors unaffected.
    const finalRows = await reader.read();
    for (const slug of slugs.filter((s) => s !== victim)) {
      assert(
        finalRows.find((r) => r.slug === slug)?.status === 'online',
        `A3 survivor ${slug} still online`,
      );
    }

    console.log('\n[validate] final board after kill:');
    console.log(await reader.formatBoard());
  } finally {
    cleanup();
  }

  console.log(
    `\n[validate] ${failures.length === 0 ? 'ALL PASSED ✅' : `FAILED ❌ (${failures.length})`}`,
  );
  process.exit(failures.length === 0 ? 0 : 1);
}

main().catch((err) => {
  console.error('[validate] fatal:', err);
  process.exit(1);
});