From a3d9bd890cbe081b3867f0f90121545b4edef4c9 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Fri, 14 Aug 2026 23:49:15 -0500 Subject: [PATCH] lease probe: fix the same 2s budget on the activation half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enforcement half was fixed in 9efd903c. The activation half (`defaultCapabilityProbe`) had the identical hardcoded 2000 ms budget against the identical CLI boot, so half the defect was still shipping. Measured cost of the exact call this makes: 1.04-1.11 s on an idle developer host against `node -e 0` at 0.055 s, and 3.0-3.7 s on a 4-core VM / 3.55-3.61 s on web1 when reached through the `mosaic` shim. The budget was below the real cost on two production hosts and inside the noise band on a third. Because the probe is fail-closed, an expiry is indistinguishable from "this build has no activation capability", so it surfaced as a framework/CLI version-skew error that no upgrade could satisfy. This was not theoretical: install-ordering-guard.spec.ts failed intermittently in the full suite (3819 ms) while passing alone (2199 ms) — the budget expiring under parallel load. That failure is gone. - named constant + env override, mirroring the enforcement half - override rejects non-finite/non-positive values rather than unbounding the probe, so a bad value cannot hang a launch - a test asserts the two halves stay numerically equal, so the tighter one can never silently become the real budget again Verified: build rc=0; new tests red against the old constant (2 fail), green after; full vitest down to the 4 pre-existing mutator-gate.acceptance failures that are also red on origin/main. --- .../commands/lease-activation-probe.spec.ts | 57 ++++++++++++++++++- .../src/commands/lease-activation-probe.ts | 42 +++++++++++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/packages/mosaic/src/commands/lease-activation-probe.spec.ts b/packages/mosaic/src/commands/lease-activation-probe.spec.ts index 8839c6a3..125cdd19 100644 --- a/packages/mosaic/src/commands/lease-activation-probe.spec.ts +++ b/packages/mosaic/src/commands/lease-activation-probe.spec.ts @@ -1,13 +1,16 @@ import { describe, it, expect } from 'vitest'; import { Command } from 'commander'; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { + CAPABILITY_PROBE_TIMEOUT_MS, + CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR, LEASE_ACTIVATION_CAPABILITY, LEASE_CAPABILITY_PROBE_COMMAND, defaultCapabilityProbe, + resolveCapabilityProbeTimeoutMs, defaultResolveCliEntry, defaultSupervisorProbe, leaseEnforcementActivatable, @@ -169,6 +172,58 @@ describe('defaultCapabilityProbe', () => { }); }); +describe('capability probe timeout budget', () => { + /** + * Regression for the defect that denied every seat launch: the probe boots + * the whole CLI (measured 1.04-1.11 s idle here, 3.0-3.7 s on a 4-core VM + * and 3.55-3.61 s on web1 through the `mosaic` shim) but was budgeted + * 2000 ms. Because the probe is fail-closed, the expiry surfaced as + * "framework/CLI version skew" — a diagnosis no upgrade could satisfy. + * + * The bound is asserted as a floor, not an equality, so raising the default + * further never fails this test; only dropping it back under the measured + * host cost does. + */ + it('is budgeted above the measured cost of booting the CLI on a real host', () => { + expect(CAPABILITY_PROBE_TIMEOUT_MS).toBeGreaterThan(10_000); + }); + + it('mirrors the enforcement half, which must not be budgeted tighter than this', () => { + // The two halves probe the same CLI boot. If they drift, the tighter one + // silently becomes the real budget and reintroduces the same failure. + const enforcementGate = join( + dirname(fileURLToPath(import.meta.url)), + '..', + '..', + 'framework', + 'tools', + 'lease-broker', + 'activation_version_gate.py', + ); + const source = readFileSync(enforcementGate, 'utf-8'); + const match = /^PROBE_TIMEOUT_SECONDS: Final = ([0-9.]+)$/m.exec(source); + expect(match, 'enforcement-half timeout constant not found').not.toBeNull(); + const enforcementMs = Number(match?.[1]) * 1000; + expect(enforcementMs).toBeGreaterThan(10_000); + expect(CAPABILITY_PROBE_TIMEOUT_MS).toBe(enforcementMs); + }); + + it('honours a valid override and ignores unusable ones rather than unbounding the probe', () => { + expect( + resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: '45000' }), + ).toBe(45_000); + + for (const unusable of ['', ' ', 'abc', '0', '-5', 'NaN', 'Infinity']) { + expect( + resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: unusable }), + `"${unusable}" must fall back to the default, never disable the bound`, + ).toBe(CAPABILITY_PROBE_TIMEOUT_MS); + } + + expect(resolveCapabilityProbeTimeoutMs({})).toBe(CAPABILITY_PROBE_TIMEOUT_MS); + }); +}); + describe('defaultResolveCliEntry', () => { it('resolves the bare "@mosaicstack/mosaic" specifier (the exported "." entry), never the non-exported "./package.json" subpath', () => { // Fully isolated from the real filesystem/package state (no dependency diff --git a/packages/mosaic/src/commands/lease-activation-probe.ts b/packages/mosaic/src/commands/lease-activation-probe.ts index 7c634d22..c7a5fddf 100644 --- a/packages/mosaic/src/commands/lease-activation-probe.ts +++ b/packages/mosaic/src/commands/lease-activation-probe.ts @@ -55,6 +55,46 @@ export const LEASE_ACTIVATION_CAPABILITY: LeaseActivationCapability = { /** Hidden CLI probe subcommand name — wired via {@link registerLeaseCapabilityProbe}. */ export const LEASE_CAPABILITY_PROBE_COMMAND = '__lease-capability'; +/** + * Milliseconds to wait for the out-of-process capability probe. + * + * The probe boots the entire CLI; it does not merely exec a binary. Measured + * cost of the exact call {@link defaultCapabilityProbe} makes: 1.04-1.11 s on + * an idle developer host, against `node -e 0` at 0.055 s — and 3.0-3.7 s on a + * 4-core VM and 3.55-3.61 s on web1 when reached through the `mosaic` shim, + * which is what the enforcement half in + * `framework/tools/lease-broker/activation_version_gate.py` runs. The former + * 2000 ms budget was therefore below the real cost on two production hosts and + * inside the noise band on a third: this same call was observed exceeding it + * under parallel test load while taking 2.2 s when run alone. + * + * That matters more than a slow probe normally would, because the probe is + * fail-closed: an expiry is indistinguishable from "this build has no + * activation capability". A budget set below the measured cost does not make + * the gate stricter, it makes it report a version skew that no upgrade can + * fix. Sized well above every measurement — the gate still fails closed, it + * just no longer fails closed on a stopwatch. + */ +export const CAPABILITY_PROBE_TIMEOUT_MS = 20_000; + +/** + * Override hook: milliseconds to wait for the probe, for hosts slow or loaded + * enough that even the default is tight. Non-numeric, non-positive, and + * non-finite values are ignored in favour of the default rather than + * disabling the bound — an unbounded probe would hang a launch instead of + * denying it. + */ +export const CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR = 'MOSAIC_LEASE_VERSION_PROBE_TIMEOUT_MS'; + +/** Resolve the probe budget from the environment, falling back to the default. */ +export function resolveCapabilityProbeTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]; + if (raw === undefined || raw.trim() === '') return CAPABILITY_PROBE_TIMEOUT_MS; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return CAPABILITY_PROBE_TIMEOUT_MS; + return parsed; +} + function capabilityMatches(candidate: LeaseActivationCapability | null): boolean { return ( candidate !== null && @@ -141,7 +181,7 @@ export function defaultCapabilityProbe( const output = execFileSync(process.execPath, [cliEntry, LEASE_CAPABILITY_PROBE_COMMAND], { encoding: 'utf-8', - timeout: 2000, + timeout: resolveCapabilityProbeTimeoutMs(), stdio: ['ignore', 'pipe', 'ignore'], });