fix(#838): bound broker reply deadlines

packages/mosaic/framework/tools/lease-broker/daemon.py

@@ -26,7 +26,9 @@ MAX_PENDING_TOKENS: Final = 256
 MAX_IN_FLIGHT_CONNECTIONS: Final = 16
 MAX_LEASE_TTL_SECONDS: Final = 300
 STATE_VERSION: Final = 1
-CONNECTION_DEADLINE_SECONDS: Final = 1.0
+READ_DEADLINE_SECONDS: Final = 1.0
+HANDLE_QUEUE_TIMEOUT_SECONDS: Final = 1.0
+SEND_TIMEOUT_SECONDS: Final = 1.0
 HEX_256_LENGTH: Final = 64
 LEASE_UNVERIFIED: Final = "UNVERIFIED"
 LEASE_PENDING: Final = "PENDING_VERIFICATION"
@@ -570,26 +572,33 @@ def handle_connection(
     broker_lock: threading.Lock,
 ) -> None:
     with connection:
-        deadline = time.monotonic() + CONNECTION_DEADLINE_SECONDS
+        read_deadline = time.monotonic() + READ_DEADLINE_SECONDS
         try:
             raw = connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)
             peer = struct.unpack("3i", raw)
-            request = read_frame(connection, deadline)
+            request = read_frame(connection, read_deadline)
         except BrokerFailure as exc:
             reply = {"ok": False, "code": exc.code}
         except OSError:
             return
         else:
-            try:
-                with broker_lock:
-                    reply = broker.handle(peer, request)
-            except BrokerFailure as exc:
-                reply = {"ok": False, "code": exc.code}
+            # Queueing is bounded and fails closed before broker.handle mutates
+            # state. Once handling starts it must finish atomically; its reply
+            # then receives an independent send budget so a slow fsync cannot
+            # consume the write opportunity and create client/broker ambiguity.
+            acquired = broker_lock.acquire(timeout=HANDLE_QUEUE_TIMEOUT_SECONDS)
+            if not acquired:
+                reply = {"ok": False, "code": "BROKER_BUSY"}
+            else:
+                try:
+                    try:
+                        reply = broker.handle(peer, request)
+                    except BrokerFailure as exc:
+                        reply = {"ok": False, "code": exc.code}
+                finally:
+                    broker_lock.release()
         try:
-            remaining = deadline - time.monotonic()
-            if remaining <= 0:
-                return
-            connection.settimeout(remaining)
+            connection.settimeout(SEND_TIMEOUT_SECONDS)
             connection.sendall((json.dumps(reply, separators=(",", ":")) + "\n").encode())
         except OSError:
             return