lease probe: fix the same 2s budget on the activation half
The enforcement half was fixed in 9efd903c. The activation half
(`defaultCapabilityProbe`) had the identical hardcoded 2000 ms budget
against the identical CLI boot, so half the defect was still shipping.
Measured cost of the exact call this makes: 1.04-1.11 s on an idle
developer host against `node -e 0` at 0.055 s, and 3.0-3.7 s on a 4-core
VM / 3.55-3.61 s on web1 when reached through the `mosaic` shim. The
budget was below the real cost on two production hosts and inside the
noise band on a third.
Because the probe is fail-closed, an expiry is indistinguishable from
"this build has no activation capability", so it surfaced as a
framework/CLI version-skew error that no upgrade could satisfy.
This was not theoretical: install-ordering-guard.spec.ts failed
intermittently in the full suite (3819 ms) while passing alone (2199 ms)
— the budget expiring under parallel load. That failure is gone.
- named constant + env override, mirroring the enforcement half
- override rejects non-finite/non-positive values rather than unbounding
the probe, so a bad value cannot hang a launch
- a test asserts the two halves stay numerically equal, so the tighter
one can never silently become the real budget again
Verified: build rc=0; new tests red against the old constant (2 fail),
green after; full vitest down to the 4 pre-existing
mutator-gate.acceptance failures that are also red on origin/main.
This commit is contained in:
@@ -1,13 +1,16 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Command } from 'commander';
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
CAPABILITY_PROBE_TIMEOUT_MS,
|
||||
CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR,
|
||||
LEASE_ACTIVATION_CAPABILITY,
|
||||
LEASE_CAPABILITY_PROBE_COMMAND,
|
||||
defaultCapabilityProbe,
|
||||
resolveCapabilityProbeTimeoutMs,
|
||||
defaultResolveCliEntry,
|
||||
defaultSupervisorProbe,
|
||||
leaseEnforcementActivatable,
|
||||
@@ -169,6 +172,58 @@ describe('defaultCapabilityProbe', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('capability probe timeout budget', () => {
|
||||
/**
|
||||
* Regression for the defect that denied every seat launch: the probe boots
|
||||
* the whole CLI (measured 1.04-1.11 s idle here, 3.0-3.7 s on a 4-core VM
|
||||
* and 3.55-3.61 s on web1 through the `mosaic` shim) but was budgeted
|
||||
* 2000 ms. Because the probe is fail-closed, the expiry surfaced as
|
||||
* "framework/CLI version skew" — a diagnosis no upgrade could satisfy.
|
||||
*
|
||||
* The bound is asserted as a floor, not an equality, so raising the default
|
||||
* further never fails this test; only dropping it back under the measured
|
||||
* host cost does.
|
||||
*/
|
||||
it('is budgeted above the measured cost of booting the CLI on a real host', () => {
|
||||
expect(CAPABILITY_PROBE_TIMEOUT_MS).toBeGreaterThan(10_000);
|
||||
});
|
||||
|
||||
it('mirrors the enforcement half, which must not be budgeted tighter than this', () => {
|
||||
// The two halves probe the same CLI boot. If they drift, the tighter one
|
||||
// silently becomes the real budget and reintroduces the same failure.
|
||||
const enforcementGate = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
'..',
|
||||
'..',
|
||||
'framework',
|
||||
'tools',
|
||||
'lease-broker',
|
||||
'activation_version_gate.py',
|
||||
);
|
||||
const source = readFileSync(enforcementGate, 'utf-8');
|
||||
const match = /^PROBE_TIMEOUT_SECONDS: Final = ([0-9.]+)$/m.exec(source);
|
||||
expect(match, 'enforcement-half timeout constant not found').not.toBeNull();
|
||||
const enforcementMs = Number(match?.[1]) * 1000;
|
||||
expect(enforcementMs).toBeGreaterThan(10_000);
|
||||
expect(CAPABILITY_PROBE_TIMEOUT_MS).toBe(enforcementMs);
|
||||
});
|
||||
|
||||
it('honours a valid override and ignores unusable ones rather than unbounding the probe', () => {
|
||||
expect(
|
||||
resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: '45000' }),
|
||||
).toBe(45_000);
|
||||
|
||||
for (const unusable of ['', ' ', 'abc', '0', '-5', 'NaN', 'Infinity']) {
|
||||
expect(
|
||||
resolveCapabilityProbeTimeoutMs({ [CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR]: unusable }),
|
||||
`"${unusable}" must fall back to the default, never disable the bound`,
|
||||
).toBe(CAPABILITY_PROBE_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
expect(resolveCapabilityProbeTimeoutMs({})).toBe(CAPABILITY_PROBE_TIMEOUT_MS);
|
||||
});
|
||||
});
|
||||
|
||||
describe('defaultResolveCliEntry', () => {
|
||||
it('resolves the bare "@mosaicstack/mosaic" specifier (the exported "." entry), never the non-exported "./package.json" subpath', () => {
|
||||
// Fully isolated from the real filesystem/package state (no dependency
|
||||
|
||||
@@ -55,6 +55,46 @@ export const LEASE_ACTIVATION_CAPABILITY: LeaseActivationCapability = {
|
||||
/** Hidden CLI probe subcommand name — wired via {@link registerLeaseCapabilityProbe}. */
|
||||
export const LEASE_CAPABILITY_PROBE_COMMAND = '__lease-capability';
|
||||
|
||||
/**
|
||||
* Milliseconds to wait for the out-of-process capability probe.
|
||||
*
|
||||
* The probe boots the entire CLI; it does not merely exec a binary. Measured
|
||||
* cost of the exact call {@link defaultCapabilityProbe} makes: 1.04-1.11 s on
|
||||
* an idle developer host, against `node -e 0` at 0.055 s — and 3.0-3.7 s on a
|
||||
* 4-core VM and 3.55-3.61 s on web1 when reached through the `mosaic` shim,
|
||||
* which is what the enforcement half in
|
||||
* `framework/tools/lease-broker/activation_version_gate.py` runs. The former
|
||||
* 2000 ms budget was therefore below the real cost on two production hosts and
|
||||
* inside the noise band on a third: this same call was observed exceeding it
|
||||
* under parallel test load while taking 2.2 s when run alone.
|
||||
*
|
||||
* That matters more than a slow probe normally would, because the probe is
|
||||
* fail-closed: an expiry is indistinguishable from "this build has no
|
||||
* activation capability". A budget set below the measured cost does not make
|
||||
* the gate stricter, it makes it report a version skew that no upgrade can
|
||||
* fix. Sized well above every measurement — the gate still fails closed, it
|
||||
* just no longer fails closed on a stopwatch.
|
||||
*/
|
||||
export const CAPABILITY_PROBE_TIMEOUT_MS = 20_000;
|
||||
|
||||
/**
|
||||
* Override hook: milliseconds to wait for the probe, for hosts slow or loaded
|
||||
* enough that even the default is tight. Non-numeric, non-positive, and
|
||||
* non-finite values are ignored in favour of the default rather than
|
||||
* disabling the bound — an unbounded probe would hang a launch instead of
|
||||
* denying it.
|
||||
*/
|
||||
export const CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR = 'MOSAIC_LEASE_VERSION_PROBE_TIMEOUT_MS';
|
||||
|
||||
/** Resolve the probe budget from the environment, falling back to the default. */
|
||||
export function resolveCapabilityProbeTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
|
||||
const raw = env[CAPABILITY_PROBE_TIMEOUT_OVERRIDE_VAR];
|
||||
if (raw === undefined || raw.trim() === '') return CAPABILITY_PROBE_TIMEOUT_MS;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return CAPABILITY_PROBE_TIMEOUT_MS;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function capabilityMatches(candidate: LeaseActivationCapability | null): boolean {
|
||||
return (
|
||||
candidate !== null &&
|
||||
@@ -141,7 +181,7 @@ export function defaultCapabilityProbe(
|
||||
|
||||
const output = execFileSync(process.execPath, [cliEntry, LEASE_CAPABILITY_PROBE_COMMAND], {
|
||||
encoding: 'utf-8',
|
||||
timeout: 2000,
|
||||
timeout: resolveCapabilityProbeTimeoutMs(),
|
||||
stdio: ['ignore', 'pipe', 'ignore'],
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user