The activation-capability probe spawns the whole Node CLI rather than exec'ing a binary. Measured 3.0-3.7s on an idle 4-core VM and 3.55-3.61s on web1, against `node -e 0` at 0.05s. The budget was 2.0s, so the probe timed out on every call on both hosts. The gate is fail-closed, and an expiry is indistinguishable from "no capability", so every fleet seat launch was denied with a version-skew message telling the operator to "upgrade both as one unit" -- advice that cannot fix a timeout. This is why web1 shows roster seats with no live sessions. Raise the budget to 20s, well clear of the measured range, and add MOSAIC_LEASE_VERSION_PROBE_TIMEOUT_SECONDS for slower hosts. Unusable override values fall back to the default rather than removing the bound. Also fix _resolve_probe_command ignoring the environ it is handed: shutil.which was called without path=, so it read the ambient PATH. That made test_returns_none_when_mosaic_is_not_resolvable_on_path pass only because the 2.0s budget expired first -- right answer, wrong reason, and it masked the timeout defect. The suite's runtime drops from 2.0s to 0.002s, which is that accidental timeout leaving. Verified: 18/18 version_coupling_unittest (new tests red against the old gate: 2 failures + 1 error), tsc build clean, test-start-agent-session.sh and test-fleet-units.sh rc=0. invariant_r_unittest fails identically with and without this change (pinned pi 0.84.1 vs installed 0.84.2).
210 lines
9.0 KiB
Python
210 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Enforcement-side version-coupling gate (issue #869, Point-1 card C4).
|
|
|
|
Root cause this exists to guard against (#828 version skew, restated from
|
|
the C1 activation probe in ``lease-activation-probe.ts``): the lease
|
|
broker's ENFORCEMENT half (this toolkit — ``launch-runtime.py``,
|
|
``mutator-gate.py``, ``revoke-lease.py``) and its ACTIVATION half
|
|
(``execLeaseGatedRuntime()`` in ``launch.ts``, which chains the gated
|
|
runtime through ``launch-runtime.py`` and injects ``MOSAIC_LEASE_*``) ship
|
|
on different channels — an npm package and a framework/CLI reseed. C1 gave
|
|
the activation half a versioned, machine-checkable identity
|
|
(``LEASE_ACTIVATION_CAPABILITY``, printed by the CLI's hidden
|
|
``mosaic __lease-capability`` subcommand). That identity is inert on its
|
|
own: nothing yet asserted that ENFORCEMENT actually requires the version
|
|
ACTIVATION advertises. This module is that assertion, owned by the
|
|
enforcement side.
|
|
|
|
``EXPECTED_ACTIVATION_CAPABILITY`` below is this toolkit's own contract
|
|
declaration — bump it only when this toolkit's launch/gate seam starts
|
|
requiring a different activation contract (new env vars it depends on,
|
|
changed chaining behavior, etc.), independent of any package semver, for
|
|
the same reason C1's constant is: #828 happened precisely because a
|
|
version number that should have moved did not.
|
|
|
|
This module never talks to a real broker or a real installed CLI in its
|
|
own tests — both the probe's command resolution and its ``run`` transport
|
|
are injectable so tests can drive every branch with fakes/stubs (see
|
|
``version_coupling_unittest.py``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
from collections.abc import Callable, Mapping
|
|
from typing import Final, TypedDict
|
|
|
|
|
|
class ActivationCapability(TypedDict):
|
|
name: str
|
|
version: int
|
|
|
|
|
|
# ENFORCEMENT-side expected activation contract. OWNED by this toolkit (the
|
|
# enforcement half). Mirrors — but is deliberately a SEPARATE constant from
|
|
# — `LEASE_ACTIVATION_CAPABILITY` in
|
|
# `packages/mosaic/src/commands/lease-activation-probe.ts` (the activation
|
|
# half's own declaration of what it implements). The two are compared at
|
|
# runtime by `assert_activation_capability_matches()`; drift between them is
|
|
# exactly the version-skew failure mode #828/#869 exist to catch, and must
|
|
# FAIL LOUD, never a silent pass and never a dead (always-true) gate.
|
|
EXPECTED_ACTIVATION_CAPABILITY: Final[ActivationCapability] = {
|
|
"name": "lease-runtime-activation",
|
|
"version": 1,
|
|
}
|
|
|
|
# Matches `LEASE_CAPABILITY_PROBE_COMMAND` in lease-activation-probe.ts —
|
|
# the hidden CLI subcommand that prints the activation half's advertised
|
|
# capability as compact JSON.
|
|
LEASE_CAPABILITY_PROBE_COMMAND: Final = "__lease-capability"
|
|
|
|
# Running the probe boots the whole Node CLI; it does not merely exec a binary.
|
|
# Measured: 3.0-3.7 s on an idle 4-core VM and 3.55-3.61 s on web1, against
|
|
# `node -e 0` at 0.05 s. The former 2.0 s budget therefore expired on every
|
|
# call on both hosts. Because the probe is fail-closed, an expiry is
|
|
# indistinguishable from "no capability", so every seat launch was denied with
|
|
# a version-skew message that no upgrade could fix. Sized well above the
|
|
# measured range: the gate still fails closed, it just no longer fails closed
|
|
# on a stopwatch.
|
|
PROBE_TIMEOUT_SECONDS: Final = 20.0
|
|
|
|
# Override hook: seconds to wait for the probe, for hosts slow or loaded enough
|
|
# that even the default is tight. Non-numeric or non-positive values are
|
|
# ignored in favour of the default rather than disabling the bound.
|
|
PROBE_TIMEOUT_OVERRIDE_VAR: Final = "MOSAIC_LEASE_VERSION_PROBE_TIMEOUT_SECONDS"
|
|
|
|
# Override hook: a full shell-style command line (parsed with `shlex.split`)
|
|
# to run INSTEAD of resolving `mosaic` on PATH and appending the probe
|
|
# subcommand. Real deployments should never need this — `mosaic` is on PATH
|
|
# whenever a runtime was launched via `mosaic <cmd>` in the first place, the
|
|
# only real caller of this seam. It exists for integration tests that spawn
|
|
# `launch-runtime.py` directly (never through the real CLI) to supply a
|
|
# fake/stub CLI probe, matching the existing convention of those tests
|
|
# supplying a fake broker and a fake runtime binary rather than depending on
|
|
# host state.
|
|
MOSAIC_COMMAND_OVERRIDE_VAR: Final = "MOSAIC_LEASE_VERSION_PROBE_COMMAND"
|
|
|
|
|
|
class VersionCouplingError(Exception):
|
|
"""Raised when the activation capability is absent, unreadable, or does
|
|
not match what enforcement expects. Callers MUST fail loud on this
|
|
(non-zero exit, clear actionable stderr) — never swallow it into a
|
|
silent pass, and never let its absence be treated as compatible."""
|
|
|
|
|
|
def _resolve_probe_timeout(environ: Mapping[str, str]) -> float:
|
|
raw = environ.get(PROBE_TIMEOUT_OVERRIDE_VAR)
|
|
if not raw:
|
|
return PROBE_TIMEOUT_SECONDS
|
|
try:
|
|
seconds = float(raw)
|
|
except ValueError:
|
|
return PROBE_TIMEOUT_SECONDS
|
|
if seconds <= 0 or seconds != seconds or seconds == float("inf"):
|
|
return PROBE_TIMEOUT_SECONDS
|
|
return seconds
|
|
|
|
|
|
def _resolve_probe_command(environ: Mapping[str, str]) -> list[str] | None:
|
|
override = environ.get(MOSAIC_COMMAND_OVERRIDE_VAR)
|
|
if override:
|
|
parsed = shlex.split(override)
|
|
return parsed or None
|
|
# Resolve against the caller's PATH, not the ambient process one. The
|
|
# function is handed an `environ` and honoured it only for the override
|
|
# var, so a caller passing an explicit PATH was silently ignored here.
|
|
resolved = shutil.which("mosaic", path=environ.get("PATH"))
|
|
if resolved is None:
|
|
return None
|
|
return [resolved, LEASE_CAPABILITY_PROBE_COMMAND]
|
|
|
|
|
|
def default_probe_activation_capability(
|
|
environ: Mapping[str, str] | None = None,
|
|
*,
|
|
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
|
|
) -> ActivationCapability | None:
|
|
"""Real capability lookup: resolves and executes the CLI's hidden
|
|
``__lease-capability`` probe subcommand out-of-process (the same
|
|
mechanism `defaultCapabilityProbe()` in lease-activation-probe.ts uses
|
|
from the activation side) and parses its JSON stdout. Any failure to
|
|
resolve a command, spawn it, have it exit zero, or produce a well-shaped
|
|
``{name, version}`` JSON object is treated as NO capability (``None``)
|
|
— fail-closed, never a fabricated/guessed capability.
|
|
"""
|
|
source_environment = os.environ if environ is None else environ
|
|
command = _resolve_probe_command(source_environment)
|
|
if command is None:
|
|
return None
|
|
try:
|
|
completed = run(
|
|
command,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=_resolve_probe_timeout(source_environment),
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired, ValueError):
|
|
return None
|
|
if completed.returncode != 0:
|
|
return None
|
|
try:
|
|
parsed = json.loads(completed.stdout)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
if (
|
|
not isinstance(parsed, dict)
|
|
or not isinstance(parsed.get("name"), str)
|
|
or not isinstance(parsed.get("version"), int)
|
|
or isinstance(parsed.get("version"), bool)
|
|
):
|
|
return None
|
|
return {"name": parsed["name"], "version": parsed["version"]}
|
|
|
|
|
|
def format_mismatch_message(
|
|
activation: ActivationCapability | None,
|
|
expected: ActivationCapability,
|
|
) -> str:
|
|
"""Actionable, non-silent remediation message for either failure shape:
|
|
absent/unreadable capability, or a present-but-incompatible one."""
|
|
if activation is None:
|
|
return (
|
|
"Mosaic lease activation capability unreadable: enforcement "
|
|
f"expects '{expected['name']}' v{expected['version']} but the "
|
|
f"CLI's `mosaic {LEASE_CAPABILITY_PROBE_COMMAND}` probe produced "
|
|
"no usable result (mosaic not on PATH, non-zero exit, or "
|
|
"malformed output) — framework/CLI version skew; upgrade both "
|
|
"as one unit; see #869."
|
|
)
|
|
if activation["name"] != expected["name"]:
|
|
return (
|
|
f"activation capability name '{activation['name']}' != "
|
|
f"enforcement expects '{expected['name']}' — framework/CLI "
|
|
"version skew; upgrade both as one unit; see #869"
|
|
)
|
|
return (
|
|
f"activation capability v{activation['version']} != enforcement "
|
|
f"expects v{expected['version']} — framework/CLI version skew; "
|
|
"upgrade both as one unit; see #869"
|
|
)
|
|
|
|
|
|
def assert_activation_capability_matches(
|
|
activation: ActivationCapability | None,
|
|
expected: ActivationCapability = EXPECTED_ACTIVATION_CAPABILITY,
|
|
) -> None:
|
|
"""Raise `VersionCouplingError` unless `activation` is present AND its
|
|
`name`/`version` exactly match `expected`. Absence is treated the same
|
|
as a mismatch — never a silent pass."""
|
|
if (
|
|
activation is None
|
|
or activation.get("name") != expected["name"]
|
|
or activation.get("version") != expected["version"]
|
|
):
|
|
raise VersionCouplingError(format_mismatch_message(activation, expected))
|