diff --git a/packages/bus/src/model-keys.mjs b/packages/bus/src/model-keys.mjs index f7f9d90ca..e25428bb2 100644 --- a/packages/bus/src/model-keys.mjs +++ b/packages/bus/src/model-keys.mjs @@ -7,6 +7,7 @@ import { BusError } from './broker.mjs'; export const REVOKE_TIMEOUT_MS = 5000; export const REVOKE_POLL_MS = 25; export const ISSUE_PROBE_TIMEOUT_MS = 2000; +export const ISSUE_REPROBE_MIN_MS = 250; export const MODEL_STOP_TIMEOUT_MS = 5000; const keyId = key => createHash('sha256').update(key).digest('hex').slice(0, 12); const fail = code => { throw new BusError(code); }; @@ -224,13 +225,17 @@ export class ModelKeys { this.#sentinel(keys); return this.#issue(instance, false, keys); }); - // CLIProxyAPI applies a key-list PUT by an async reload after it answers. From 8.0.79 a - // just-issued key answers 401 for a few milliseconds, so only 401 is probed again until a - // short deadline. Any other non-2xx refuses on the first answer, as before. + // Every v8 CLIProxyAPI applies a key-list PUT by an async reload after it answers, so a + // just-issued key can answer 401 for a moment. On 8.0.79 that moment outlasts the confirming + // GET (measured, #1873). Only 401 is probed again, until a short deadline; a re-probe waits + // about the time left, at least ISSUE_REPROBE_MIN_MS. Any other non-2xx refuses on the first + // answer, as before. const deadline = performance.now() + ISSUE_PROBE_TIMEOUT_MS; - let status; + let status, first = true; for (;;) { - try { status = await this.#client.probeKey(proxy.key); } + const options = first ? undefined : { timeout: Math.max(ISSUE_REPROBE_MIN_MS, Math.ceil(deadline - performance.now())) }; + first = false; + try { status = await this.#client.probeKey(proxy.key, options); } catch { fail('proxy-down'); } if (status !== 401 || performance.now() >= deadline) break; await new Promise(resolve => setTimeout(resolve, REVOKE_POLL_MS)); diff --git a/packages/bus/tests/model-keys.test.mjs b/packages/bus/tests/model-keys.test.mjs index 4782bbe45..e2674b2c4 100644 --- a/packages/bus/tests/model-keys.test.mjs +++ b/packages/bus/tests/model-keys.test.mjs @@ -11,7 +11,7 @@ import { tmpdir } from 'node:os'; import { createHash, randomBytes } from 'node:crypto'; import { Credentials } from '../src/credentials.mjs'; import { ManagementClient } from '../../model-proxy/src/client.mjs'; -import { ModelKeys, REVOKE_TIMEOUT_MS, REVOKE_POLL_MS, ISSUE_PROBE_TIMEOUT_MS } from '../src/model-keys.mjs'; +import { ModelKeys, REVOKE_TIMEOUT_MS, REVOKE_POLL_MS, ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS } from '../src/model-keys.mjs'; import { startBroker } from '../src/runtime.mjs'; import { busOrganization } from '../src/organization-context.mjs'; import { Registry } from '../../seat/src/session.mjs'; @@ -42,6 +42,7 @@ async function fixture(t, { delayed = 0 } = {}) { const key = req.headers.authorization?.slice(7); const accepted = state.keys.includes(key) || (state.removed.has(key) && performance.now() - state.removed.get(key) < state.delay); state.onProbe?.(state.status ?? (accepted ? 200 : 401)); + if (state.stallProbe?.()) return; res.writeHead(state.status ?? (accepted ? 200 : 401)); res.end('{}'); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); @@ -134,7 +135,16 @@ test('proxy-key-refused: a key that keeps answering 401 refuses once the short p const took = performance.now() - started; assert.ok(f.state.probes > 1, 'a 401 is probed again'); assert.ok(took >= ISSUE_PROBE_TIMEOUT_MS && took < ISSUE_PROBE_TIMEOUT_MS + 2000, `bounded: ${took}`); }); -// CLIProxyAPI 8.0.79 answers 401 for a just-issued key for a few milliseconds after the PUT. +test('proxy-down: a re-probe that stalls ends near the probe deadline, not the client default', async t => { + const f = await fixture(t); await f.manager.reconcile(); await f.manager.ensure('demo/coder'); f.state.status = 401; + f.state.stallProbe = () => f.state.probes >= 2; + const started = performance.now(); + await assert.rejects(f.manager.preflight('demo/coder'), failCode('proxy-down')); + const took = performance.now() - started; + assert.equal(f.state.probes, 2); assert.ok(took < ISSUE_PROBE_TIMEOUT_MS + ISSUE_REPROBE_MIN_MS + 1000, `bounded: ${took}`); +}); +// Every v8 CLIProxyAPI applies the PUT by an async reload; on 8.0.79 a just-issued key answers 401 +// past the confirming GET (#1873). test('first launch passes when the proxy applies the new key a moment after the PUT answers', async t => { const f = await fixture(t); await f.manager.reconcile(); f.state.status = 401; f.state.onProbe = () => { if (f.state.probes === 3) f.state.status = null; }; diff --git a/packages/model-proxy/README.md b/packages/model-proxy/README.md index c89fa73d7..b712113c7 100644 --- a/packages/model-proxy/README.md +++ b/packages/model-proxy/README.md @@ -58,7 +58,12 @@ the upgrade stops; Sage takes Q46's fallback to Jason through Mos. A running instance takes a new PIN in three steps, in this order. First run the new code: update the checkout or pinned copy the bus, Console and proxy -unit start from. Then run `mosaic proxy fetch` (or Fetch on the Console's +unit start from. The proxy unit runs `run.mjs` and reads `PIN` from the tree +it was installed from (`systemctl --user cat ` shows it). If the new +code is in that same tree, a restart picks it up. If it is in another tree, +the unit keeps the old PIN until it is installed again from the new tree: +stop it, remove its unit file, then `mosaic proxy unit install` from the new +tree, because install never replaces a unit. Then run `mosaic proxy fetch` (or Fetch on the Console's Model proxy settings), which installs the new release beside the old one under `/opt/cliproxyapi//`. Then restart the proxy with `mosaic proxy unit restart`. Before the fetch, the proxy fails to start with @@ -67,7 +72,8 @@ Console as well so their status and Fetch use the new PIN. `proxy.json` keeps the path from the PIN it was created under. That path stays valid, and the proxy always runs the hash-verified install for the current PIN. Signed-in accounts in `/proxy/auth/` carry over. From 8.0.55 the proxy also -keeps a `kv.db` file in that directory. The old release directory can be +keeps a `kv.db` file in that directory, and 8.0.79 adds a `.codex-refresh/` +directory there once a Codex account renews. The old release directory can be removed once the new one runs. The native `/proxy` page uses human-only bus verbs; only the bus holds the diff --git a/packages/model-proxy/tests/empty-provider.test.mjs b/packages/model-proxy/tests/empty-provider.test.mjs index ef5a3412d..866f63b90 100644 --- a/packages/model-proxy/tests/empty-provider.test.mjs +++ b/packages/model-proxy/tests/empty-provider.test.mjs @@ -1,4 +1,4 @@ -// P4 pinned8.0.42 regression: absent Claude provider config is HTTP404. +// P4 regression since PIN 8.0.42 (still so on 8.0.79): absent Claude provider config is HTTP 404. // Normalize only that fixed GET; all other methods/endpoints/statuses refuse. import test from 'node:test'; import assert from 'node:assert/strict'; diff --git a/packages/model-proxy/tests/page-client.test.mjs b/packages/model-proxy/tests/page-client.test.mjs index cc6331987..a630be9e2 100644 --- a/packages/model-proxy/tests/page-client.test.mjs +++ b/packages/model-proxy/tests/page-client.test.mjs @@ -1,4 +1,4 @@ -// P4 client contract; endpoint/payload cases are probe-pending on PIN8.0.42. +// P4 client contract; endpoint/payload cases were probe-pending on PIN 8.0.42. // Catches management key/body/query values leaking into receipts, redirects/retries. import test from 'node:test'; import assert from 'node:assert/strict';