lease probe: fix the same 2s budget on the activation half

The enforcement half was fixed in 9efd903c. The activation half
(`defaultCapabilityProbe`) had the identical hardcoded 2000 ms budget
against the identical CLI boot, so half the defect was still shipping.

Measured cost of the exact call this makes: 1.04-1.11 s on an idle
developer host against `node -e 0` at 0.055 s, and 3.0-3.7 s on a 4-core
VM / 3.55-3.61 s on web1 when reached through the `mosaic` shim. The
budget was below the real cost on two production hosts and inside the
noise band on a third.

Because the probe is fail-closed, an expiry is indistinguishable from
"this build has no activation capability", so it surfaced as a
framework/CLI version-skew error that no upgrade could satisfy.

This was not theoretical: install-ordering-guard.spec.ts failed
intermittently in the full suite (3819 ms) while passing alone (2199 ms)
— the budget expiring under parallel load. That failure is gone.

- named constant + env override, mirroring the enforcement half
- override rejects non-finite/non-positive values rather than unbounding
  the probe, so a bad value cannot hang a launch
- a test asserts the two halves stay numerically equal, so the tighter
  one can never silently become the real budget again

Verified: build rc=0; new tests red against the old constant (2 fail),
green after; full vitest down to the 4 pre-existing
mutator-gate.acceptance failures that are also red on origin/main.
This commit is contained in:
Jason Woltje
2026-08-14 23:49:15 -05:00
parent 9efd903c16
commit a3d9bd890c
2 changed files with 97 additions and 2 deletions
@@ -1,13 +1,16 @@
import { describe, it, expect } from 'vitest';
import { Command } from 'commander';
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';
import {
CAPABILITY_PROBE_TIMEOUT_MS,
CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR,
LEASE_ACTIVATION_CAPABILITY,
LEASE_CAPABILITY_PROBE_COMMAND,
defaultCapabilityProbe,
resolveCapabilityProbeTimeoutMs,
defaultResolveCliEntry,
defaultSupervisorProbe,
leaseEnforcementActivatable,
@@ -169,6 +172,58 @@ describe('defaultCapabilityProbe', () => {
});
});
describe('capability probe timeout budget', () => {
/**
* Regression for the defect that denied every seat launch: the probe boots
* the whole CLI (measured 1.04-1.11 s idle here, 3.0-3.7 s on a 4-core VM
* and 3.55-3.61 s on web1 through the `mosaic` shim) but was budgeted
* 2000 ms. Because the probe is fail-closed, the expiry surfaced as
* "framework/CLI version skew" — a diagnosis no upgrade could satisfy.
*
* The bound is asserted as a floor, not an equality, so raising the default
* further never fails this test; only dropping it back under the measured
* host cost does.
*/
it('is budgeted above the measured cost of booting the CLI on a real host', () => {
expect(CAPABILITY_PROBE_TIMEOUT_MS).toBeGreaterThan(10_000);
});
it('mirrors the enforcement half, which must not be budgeted tighter than this', () => {
// The two halves probe the same CLI boot. If they drift, the tighter one
// silently becomes the real budget and reintroduces the same failure.
const enforcementGate = join(
dirname(fileURLToPath(import.meta.url)),
'..',
'..',
'framework',
'tools',
'lease-broker',
'activation_version_gate.py',
);
const source = readFileSync(enforcementGate, 'utf-8');
const match = /^PROBE_TIMEOUT_SECONDS: Final = ([0-9.]+)$/m.exec(source);
expect(match, 'enforcement-half timeout constant not found').not.toBeNull();
const enforcementMs = Number(match?.[1]) * 1000;
expect(enforcementMs).toBeGreaterThan(10_000);
expect(CAPABILITY_PROBE_TIMEOUT_MS).toBe(enforcementMs);
});
it('honours a valid override and ignores unusable ones rather than unbounding the probe', () => {
expect(
resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: '45000' }),
).toBe(45_000);
for (const unusable of ['', ' ', 'abc', '0', '-5', 'NaN', 'Infinity']) {
expect(
resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: unusable }),
`"${unusable}" must fall back to the default, never disable the bound`,
).toBe(CAPABILITY_PROBE_TIMEOUT_MS);
}
expect(resolveCapabilityProbeTimeoutMs({})).toBe(CAPABILITY_PROBE_TIMEOUT_MS);
});
});
describe('defaultResolveCliEntry', () => {
it('resolves the bare "@mosaicstack/mosaic" specifier (the exported "." entry), never the non-exported "./package.json" subpath', () => {
// Fully isolated from the real filesystem/package state (no dependency
@@ -55,6 +55,46 @@ export const LEASE_ACTIVATION_CAPABILITY: LeaseActivationCapability = {
/** Hidden CLI probe subcommand name — wired via {@link registerLeaseCapabilityProbe}. */
export const LEASE_CAPABILITY_PROBE_COMMAND = '__lease-capability';
/**
* Milliseconds to wait for the out-of-process capability probe.
*
* The probe boots the entire CLI; it does not merely exec a binary. Measured
* cost of the exact call {@link defaultCapabilityProbe} makes: 1.04-1.11 s on
* an idle developer host, against `node -e 0` at 0.055 s — and 3.0-3.7 s on a
* 4-core VM and 3.55-3.61 s on web1 when reached through the `mosaic` shim,
* which is what the enforcement half in
* `framework/tools/lease-broker/activation_version_gate.py` runs. The former
* 2000 ms budget was therefore below the real cost on two production hosts and
* inside the noise band on a third: this same call was observed exceeding it
* under parallel test load while taking 2.2 s when run alone.
*
* That matters more than a slow probe normally would, because the probe is
* fail-closed: an expiry is indistinguishable from "this build has no
* activation capability". A budget set below the measured cost does not make
* the gate stricter, it makes it report a version skew that no upgrade can
* fix. Sized well above every measurement — the gate still fails closed, it
* just no longer fails closed on a stopwatch.
*/
export const CAPABILITY_PROBE_TIMEOUT_MS = 20_000;
/**
* Override hook: milliseconds to wait for the probe, for hosts slow or loaded
* enough that even the default is tight. Non-numeric, non-positive, and
* non-finite values are ignored in favour of the default rather than
* disabling the bound — an unbounded probe would hang a launch instead of
* denying it.
*/
export const CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR = 'MOSAIC_LEASE_VERSION_PROBE_TIMEOUT_MS';
/** Resolve the probe budget from the environment, falling back to the default. */
export function resolveCapabilityProbeTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
const raw = env[CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR];
if (raw === undefined || raw.trim() === '') return CAPABILITY_PROBE_TIMEOUT_MS;
const parsed = Number(raw);
if (!Number.isFinite(parsed) || parsed <= 0) return CAPABILITY_PROBE_TIMEOUT_MS;
return parsed;
}
function capabilityMatches(candidate: LeaseActivationCapability | null): boolean {
return (
candidate !== null &&
@@ -141,7 +181,7 @@ export function defaultCapabilityProbe(
const output = execFileSync(process.execPath, [cliEntry, LEASE_CAPABILITY_PROBE_COMMAND], {
encoding: 'utf-8',
timeout: 2000,
timeout: resolveCapabilityProbeTimeoutMs(),
stdio: ['ignore', 'pipe', 'ignore'],
});