fix(#838): bound broker reply deadlines

packages/mosaic/src/lease-broker/broker-test-client.ts

@@ -0,0 +1,157 @@
+import { createConnection, type Socket } from 'node:net';
+
+const DEFAULT_TIMEOUT_MS = 3_000;
+const MAX_REPLY_BYTES = 64 * 1024;
+
+export interface BrokerTestClientOptions {
+  timeoutMs?: number;
+}
+
+export type BrokerTransportFailureKind =
+  | 'early-close'
+  | 'malformed-reply'
+  | 'socket-error'
+  | 'timeout';
+
+export class BrokerTransportError extends Error {
+  public readonly responseLength: number;
+  public readonly responseBytes: string;
+  public readonly responseHex: string;
+
+  public constructor(
+    public readonly kind: BrokerTransportFailureKind,
+    description: string,
+    response: Buffer,
+    public readonly attempts = 1,
+  ) {
+    super(`${description}; attempts=${attempts}; ${responseContext(response)}`);
+    this.name = 'BrokerTransportError';
+    this.responseLength = response.length;
+    this.responseBytes = response.toString('utf8');
+    this.responseHex = response.toString('hex');
+  }
+}
+
+function responseContext(response: Buffer): string {
+  const text = JSON.stringify(response.toString('utf8'));
+  const hex = response.length === 0 ? '<empty>' : response.toString('hex');
+  return `length=${response.length}; bytes=${text}; hex=${hex}`;
+}
+
+function malformedReply(response: Buffer, reason: string): BrokerTransportError {
+  return new BrokerTransportError('malformed-reply', `Malformed broker reply: ${reason}`, response);
+}
+
+function readBrokerReplyAttempt<T extends object>(
+  socketPath: string,
+  write: (socket: Socket) => void,
+  timeoutMs: number,
+): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const socket = createConnection(socketPath);
+    let response = Buffer.alloc(0);
+    let settled = false;
+
+    const settle = (callback: () => void): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      callback();
+      socket.destroy();
+    };
+    const fail = (error: Error): void => settle(() => reject(error));
+    const timer = setTimeout(
+      () =>
+        fail(
+          new BrokerTransportError(
+            'timeout',
+            `Broker reply timed out before newline after ${timeoutMs}ms`,
+            response,
+          ),
+        ),
+      timeoutMs,
+    );
+
+    socket.once('error', (error) =>
+      fail(
+        new BrokerTransportError(
+          'socket-error',
+          `Broker reply socket error before newline: ${error.message}`,
+          response,
+        ),
+      ),
+    );
+    socket.on('data', (chunk: Buffer) => {
+      if (settled) return;
+      response = Buffer.concat([response, chunk]);
+      if (response.length > MAX_REPLY_BYTES) {
+        fail(malformedReply(response, `exceeds ${MAX_REPLY_BYTES} bytes`));
+        return;
+      }
+
+      const newline = response.indexOf(0x0a);
+      if (newline < 0) return;
+      if (newline !== response.length - 1) {
+        fail(malformedReply(response, 'contains bytes after the newline terminator'));
+        return;
+      }
+
+      try {
+        const parsed: unknown = JSON.parse(response.subarray(0, newline).toString('utf8'));
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+          fail(malformedReply(response, 'JSON value is not an object'));
+          return;
+        }
+        settle(() => resolve(parsed as T));
+      } catch (error: unknown) {
+        fail(
+          malformedReply(response, error instanceof Error ? error.message : 'JSON parsing failed'),
+        );
+      }
+    });
+    socket.once('end', () => {
+      if (!settled) {
+        fail(
+          new BrokerTransportError(
+            'early-close',
+            'Broker reply socket closed before newline',
+            response,
+          ),
+        );
+      }
+    });
+    socket.once('connect', () => {
+      try {
+        write(socket);
+      } catch (error: unknown) {
+        fail(error instanceof Error ? error : new Error(String(error)));
+      }
+    });
+  });
+}
+
+/** Read exactly one complete newline-framed JSON object; transport failures reject. */
+export async function readBrokerReply<T extends object>(
+  socketPath: string,
+  write: (socket: Socket) => void,
+  options: BrokerTestClientOptions = {},
+): Promise<T> {
+  return await readBrokerReplyAttempt<T>(
+    socketPath,
+    write,
+    options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+  );
+}
+
+/** Send one newline-framed request and read its complete broker reply. */
+export async function requestBrokerReply<T extends object>(
+  socketPath: string,
+  requestValue: object,
+  options?: BrokerTestClientOptions,
+): Promise<T> {
+  return await readBrokerReply<T>(
+    socketPath,
+    (socket) => socket.end(`${JSON.stringify(requestValue)}\n`),
+    options,
+  );
+}

packages/mosaic/src/lease-broker/daemon_deadline_unittest.py

@@ -39,6 +39,37 @@ class SlowBroker:
 
 
 class BrokerDeadlineTest(unittest.TestCase):
+    def test_lock_queue_timeout_returns_explicit_fail_closed_reply_without_handling(self) -> None:
+        broker = SlowBroker(0)
+        broker_lock = threading.Lock()
+        broker_lock.acquire()
+        server, client = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
+        client.settimeout(DAEMON.HANDLE_QUEUE_TIMEOUT_SECONDS + 2.0)
+        worker = threading.Thread(
+            target=DAEMON.handle_connection,
+            args=(server, broker, broker_lock),
+            daemon=True,
+        )
+        worker.start()
+        client.sendall(b'{"action":"probe"}\n')
+        client.shutdown(socket.SHUT_WR)
+
+        reply = bytearray()
+        try:
+            while True:
+                chunk = client.recv(4096)
+                if not chunk:
+                    break
+                reply.extend(chunk)
+        finally:
+            broker_lock.release()
+            worker.join(timeout=2.0)
+            client.close()
+
+        self.assertFalse(worker.is_alive())
+        self.assertEqual(broker.calls, 0)
+        self.assertEqual(json.loads(reply), {"ok": False, "code": "BROKER_BUSY"})
+
     def test_completed_slow_handle_gets_a_complete_framed_reply(self) -> None:
         broker = SlowBroker(original_connection_budget() + 0.1)
         broker_lock = threading.Lock()

packages/mosaic/src/lease-broker/lease-broker.acceptance.spec.ts

@@ -7,6 +7,8 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
 
 import { afterEach, describe, expect, test, vi } from 'vitest';
 
+import { readBrokerReply, requestBrokerReply } from './broker-test-client.js';
+
 interface BrokerReply {
   ok: boolean;
   code?: string;
@@ -41,34 +43,11 @@ async function rawRequest(
   socketPath: string,
   write: (socket: Socket) => void,
 ): Promise<BrokerReply> {
-  return await withTimeout(
-    new Promise<BrokerReply>((resolve, reject) => {
-      const socket = createConnection(socketPath);
-      let response = '';
-      socket.setEncoding('utf8');
-      socket.once('error', reject);
-      socket.on('data', (chunk: string) => {
-        response += chunk;
-      });
-      socket.once('end', () => resolve(JSON.parse(response) as BrokerReply));
-      socket.once('connect', () => write(socket));
-    }),
-    'raw broker request',
-  );
+  return await readBrokerReply<BrokerReply>(socketPath, write);
 }
 
 async function request(socketPath: string, requestValue: object): Promise<BrokerReply> {
-  return await new Promise<BrokerReply>((resolve, reject) => {
-    const socket = createConnection(socketPath);
-    let response = '';
-    socket.setEncoding('utf8');
-    socket.once('error', reject);
-    socket.on('data', (chunk: string) => {
-      response += chunk;
-    });
-    socket.once('end', () => resolve(JSON.parse(response) as BrokerReply));
-    socket.once('connect', () => socket.end(`${JSON.stringify(requestValue)}\n`));
-  });
+  return await requestBrokerReply<BrokerReply>(socketPath, requestValue);
 }
 
 async function startBroker(

packages/mosaic/src/mutator-gate/mutator-gate.acceptance.spec.ts

@@ -1,5 +1,4 @@
 import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { createConnection } from 'node:net';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
@@ -7,6 +6,7 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
 import { afterEach, describe, expect, test } from 'vitest';
 
 import { launchClaudex, type ClaudexHarnessAdapter } from '../commands/claudex.js';
+import { requestBrokerReply } from '../lease-broker/broker-test-client.js';
 
 interface BrokerReply {
   ok: boolean;
@@ -43,17 +43,7 @@ const binding = (compaction_epoch = 1) => ({
 });
 
 async function request(socketPath: string, requestValue: object): Promise<BrokerReply> {
-  return await new Promise<BrokerReply>((resolve, reject) => {
-    const socket = createConnection(socketPath);
-    let response = '';
-    socket.setEncoding('utf8');
-    socket.once('error', reject);
-    socket.on('data', (chunk: string) => {
-      response += chunk;
-    });
-    socket.once('end', () => resolve(JSON.parse(response) as BrokerReply));
-    socket.once('connect', () => socket.end(`${JSON.stringify(requestValue)}\n`));
-  });
+  return await requestBrokerReply<BrokerReply>(socketPath, requestValue);
 }
 
 async function startBroker(): Promise<BrokerPaths> {

packages/mosaic/src/mutator-gate/runtime_tools_unittest.py

@@ -9,7 +9,10 @@ import json
 import os
 import runpy
 import socket
+import subprocess
 import sys
+import tempfile
+import threading
 import unittest
 from contextlib import redirect_stderr
 from pathlib import Path
@@ -225,6 +228,45 @@ class LaunchRuntimeTest(unittest.TestCase):
 
 
 class ExecutableEntrypointTest(unittest.TestCase):
+    def test_real_claude_and_pi_gates_fail_closed_on_empty_or_truncated_reply(self) -> None:
+        for runtime in ("claude", "pi"):
+            for wire_reply in (b"", b'{"ok":true'):
+                with self.subTest(runtime=runtime, wire_reply=wire_reply), tempfile.TemporaryDirectory() as root:
+                    socket_path = Path(root) / "broker.sock"
+                    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                    server.bind(str(socket_path))
+                    server.listen(1)
+
+                    def serve_reply() -> None:
+                        with server:
+                            connection, _ = server.accept()
+                            with connection:
+                                while connection.recv(4096):
+                                    pass
+                                if wire_reply:
+                                    connection.sendall(wire_reply)
+
+                    thread = threading.Thread(target=serve_reply, daemon=True)
+                    thread.start()
+                    environment = {
+                        **os.environ,
+                        "MOSAIC_LEASE_BROKER_SOCKET": str(socket_path),
+                        "MOSAIC_LEASE_SESSION_ID": "d" * 64,
+                        "MOSAIC_RUNTIME_GENERATION": "1",
+                    }
+                    result = subprocess.run(
+                        [sys.executable, str(TOOLS_DIR / "mutator-gate.py"), "--runtime", runtime],
+                        input=b'{"tool_name":"Read"}\n',
+                        capture_output=True,
+                        env=environment,
+                        check=False,
+                    )
+                    thread.join(timeout=2)
+
+                    self.assertFalse(thread.is_alive())
+                    self.assertEqual(result.returncode, 2)
+                    self.assertIn(b"GATE_UNAVAILABLE", result.stderr)
+
     def test_launcher_entrypoint_returns_usage_without_a_command(self) -> None:
         with patch.object(
             sys,