diff --git a/packages/bus/src/model-keys.mjs b/packages/bus/src/model-keys.mjs index 159ada321..e25428bb2 100644 --- a/packages/bus/src/model-keys.mjs +++ b/packages/bus/src/model-keys.mjs @@ -6,6 +6,8 @@ import { isDeepStrictEqual } from 'node:util'; import { BusError } from './broker.mjs'; export const REVOKE_TIMEOUT_MS = 5000; export const REVOKE_POLL_MS = 25; +export const ISSUE_PROBE_TIMEOUT_MS = 2000; +export const ISSUE_REPROBE_MIN_MS = 250; export const MODEL_STOP_TIMEOUT_MS = 5000; const keyId = key => createHash('sha256').update(key).digest('hex').slice(0, 12); const fail = code => { throw new BusError(code); }; @@ -223,9 +225,21 @@ export class ModelKeys { this.#sentinel(keys); return this.#issue(instance, false, keys); }); - let status; - try { status = await this.#client.probeKey(proxy.key); } - catch { fail('proxy-down'); } + // Every v8 CLIProxyAPI applies a key-list PUT by an async reload after it answers, so a + // just-issued key can answer 401 for a moment. On 8.0.79 that moment outlasts the confirming + // GET (measured, #1873). Only 401 is probed again, until a short deadline; a re-probe waits + // about the time left, at least ISSUE_REPROBE_MIN_MS. Any other non-2xx refuses on the first + // answer, as before. + const deadline = performance.now() + ISSUE_PROBE_TIMEOUT_MS; + let status, first = true; + for (;;) { + const options = first ? undefined : { timeout: Math.max(ISSUE_REPROBE_MIN_MS, Math.ceil(deadline - performance.now())) }; + first = false; + try { status = await this.#client.probeKey(proxy.key, options); } + catch { fail('proxy-down'); } + if (status !== 401 || performance.now() >= deadline) break; + await new Promise(resolve => setTimeout(resolve, REVOKE_POLL_MS)); + } if (status < 200 || status >= 300) fail('proxy-key-refused'); return proxy; } diff --git a/packages/bus/tests/model-keys.test.mjs b/packages/bus/tests/model-keys.test.mjs index c765833cf..e2674b2c4 100644 --- a/packages/bus/tests/model-keys.test.mjs +++ b/packages/bus/tests/model-keys.test.mjs @@ -11,7 +11,7 @@ import { tmpdir } from 'node:os'; import { createHash, randomBytes } from 'node:crypto'; import { Credentials } from '../src/credentials.mjs'; import { ManagementClient } from '../../model-proxy/src/client.mjs'; -import { ModelKeys, REVOKE_TIMEOUT_MS, REVOKE_POLL_MS } from '../src/model-keys.mjs'; +import { ModelKeys, REVOKE_TIMEOUT_MS, REVOKE_POLL_MS, ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS } from '../src/model-keys.mjs'; import { startBroker } from '../src/runtime.mjs'; import { busOrganization } from '../src/organization-context.mjs'; import { Registry } from '../../seat/src/session.mjs'; @@ -42,6 +42,7 @@ async function fixture(t, { delayed = 0 } = {}) { const key = req.headers.authorization?.slice(7); const accepted = state.keys.includes(key) || (state.removed.has(key) && performance.now() - state.removed.get(key) < state.delay); state.onProbe?.(state.status ?? (accepted ? 200 : 401)); + if (state.stallProbe?.()) return; res.writeHead(state.status ?? (accepted ? 200 : 401)); res.end('{}'); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); @@ -123,10 +124,33 @@ test('proxy-revoke-revision-conflict keeps local revoking record and ids-only re assert.equal(JSON.stringify(f.receipts).includes(key), false); await f.create().reconcile(); assert.equal(f.credentials.modelRecord('demo/coder') === null, true); }); -for (const status of [401, 503, 302]) test(`proxy-key-refused: preflight rejects HTTP ${status} with exactly one probe`, async t => { +for (const status of [503, 302]) test(`proxy-key-refused: preflight rejects HTTP ${status} with exactly one probe`, async t => { const f = await fixture(t); await f.manager.reconcile(); await f.manager.ensure('demo/coder'); f.state.status = status; await assert.rejects(f.manager.preflight('demo/coder'), failCode('proxy-key-refused')); assert.equal(f.state.probes, 1); }); +test('proxy-key-refused: a key that keeps answering 401 refuses once the short probe deadline passes', async t => { + const f = await fixture(t); await f.manager.reconcile(); await f.manager.ensure('demo/coder'); f.state.status = 401; + const started = performance.now(); + await assert.rejects(f.manager.preflight('demo/coder'), failCode('proxy-key-refused')); + const took = performance.now() - started; + assert.ok(f.state.probes > 1, 'a 401 is probed again'); assert.ok(took >= ISSUE_PROBE_TIMEOUT_MS && took < ISSUE_PROBE_TIMEOUT_MS + 2000, `bounded: ${took}`); +}); +test('proxy-down: a re-probe that stalls ends near the probe deadline, not the client default', async t => { + const f = await fixture(t); await f.manager.reconcile(); await f.manager.ensure('demo/coder'); f.state.status = 401; + f.state.stallProbe = () => f.state.probes >= 2; + const started = performance.now(); + await assert.rejects(f.manager.preflight('demo/coder'), failCode('proxy-down')); + const took = performance.now() - started; + assert.equal(f.state.probes, 2); assert.ok(took < ISSUE_PROBE_TIMEOUT_MS + ISSUE_REPROBE_MIN_MS + 1000, `bounded: ${took}`); +}); +// Every v8 CLIProxyAPI applies the PUT by an async reload; on 8.0.79 a just-issued key answers 401 +// past the confirming GET (#1873). +test('first launch passes when the proxy applies the new key a moment after the PUT answers', async t => { + const f = await fixture(t); await f.manager.reconcile(); f.state.status = 401; + f.state.onProbe = () => { if (f.state.probes === 3) f.state.status = null; }; + const answer = await f.manager.preflight('demo/coder'); + assert.equal(f.state.probes, 3); assert.equal(f.state.keys.includes(answer.key), true); +}); test('rotate keeps old key while its run lives, new launches use new, exit removes old', async t => { const f = await fixture(t); await f.manager.reconcile(); const first = await f.manager.ensure('demo/coder'); f.runs.push({ instance: 'demo/coder', run: 'old', keyId: hash(first.key) }); diff --git a/packages/model-proxy/PIN b/packages/model-proxy/PIN index 93d86b0d1..7ad9facab 100644 --- a/packages/model-proxy/PIN +++ b/packages/model-proxy/PIN @@ -1,12 +1,12 @@ { - "version": "8.0.42", + "version": "8.0.79", "archive": { - "asset": "CLIProxyAPI_8.0.42_linux_amd64.tar.gz", - "url": "https://github.com/router-for-me/CLIProxyAPI/releases/download/v8.0.42/CLIProxyAPI_8.0.42_linux_amd64.tar.gz", - "sha256": "1e10566fe0042ed5a9b80306cf296bf86a83de590cd967a8b177988b896c8d31" + "asset": "CLIProxyAPI_8.0.79_linux_amd64.tar.gz", + "url": "https://github.com/router-for-me/CLIProxyAPI/releases/download/v8.0.79/CLIProxyAPI_8.0.79_linux_amd64.tar.gz", + "sha256": "7cbb9709ed939ff3684e10592a45cc538ee82d5da759daa7e18c7e1dc3b32acd" }, "binary": { - "sha256": "74661f8cf949977870231359e8bf4344e07946766a673c08fa25a3e5f72fc95e" + "sha256": "4968bd27b04f813849add020b95a26dc7abf3e30810b97d291c0d88f495fe601" }, "panel": { "version": "1.26.7", diff --git a/packages/model-proxy/README.md b/packages/model-proxy/README.md index 23b295df4..b712113c7 100644 --- a/packages/model-proxy/README.md +++ b/packages/model-proxy/README.md @@ -56,8 +56,28 @@ uses a new temp root and private network namespace, with made-up credentials, and calls the page's bus verbs. If the page cannot pass against the new PIN, the upgrade stops; Sage takes Q46's fallback to Jason through Mos. +A running instance takes a new PIN in three steps, in this order. First run +the new code: update the checkout or pinned copy the bus, Console and proxy +unit start from. The proxy unit runs `run.mjs` and reads `PIN` from the tree +it was installed from (`systemctl --user cat ` shows it). If the new +code is in that same tree, a restart picks it up. If it is in another tree, +the unit keeps the old PIN until it is installed again from the new tree: +stop it, remove its unit file, then `mosaic proxy unit install` from the new +tree, because install never replaces a unit. Then run `mosaic proxy fetch` (or Fetch on the Console's +Model proxy settings), which installs the new release beside the old one +under `/opt/cliproxyapi//`. Then restart the proxy with +`mosaic proxy unit restart`. Before the fetch, the proxy fails to start with +`proxy-install-missing`, and keeps failing until the fetch is done. Restart the bus and +Console as well so their status and Fetch use the new PIN. `proxy.json` keeps +the path from the PIN it was created under. That path stays valid, and the +proxy always runs the hash-verified install for the current PIN. Signed-in +accounts in `/proxy/auth/` carry over. From 8.0.55 the proxy also +keeps a `kv.db` file in that directory, and 8.0.79 adds a `.codex-refresh/` +directory there once a Codex account renews. The old release directory can be +removed once the new one runs. + The native `/proxy` page uses human-only bus verbs; only the bus holds the -management client. On PIN 8.0.42, OAuth credentials are listed under +management client. On PIN 8.0.79, OAuth credentials are listed under `/v8/management/credentials` and priority is patched as `{name, priority}`. API-key groups are absent from that list: z.ai keys use the Claude-compatible provider configuration at `/v8/management/config/api-keys/claude`, whose keys diff --git a/packages/model-proxy/src/settings.mjs b/packages/model-proxy/src/settings.mjs index 8dccb1271..3c497a5bc 100644 --- a/packages/model-proxy/src/settings.mjs +++ b/packages/model-proxy/src/settings.mjs @@ -7,12 +7,23 @@ export function installPaths(dataRoot, pin = PIN) { const dir = path.join(dataRoot, 'opt', 'cliproxyapi', pin.version); return { dir, binary: path.join(dir, 'cli-proxy-api'), panel: path.join(dir, 'static', 'management.html') }; } +// proxy.json records the install path from the PIN it was created under, and nothing +// rewrites it. After a PIN bump an earlier release's path under this data root stays valid +// and reads as the current pinned path. The runner executes only the hash-verified pinned +// install, never the recorded path, so an old value selects nothing. +const releasePath = (dataRoot, binary) => { + const base = path.join(dataRoot, 'opt', 'cliproxyapi') + path.sep; + if (typeof binary !== 'string' || !binary.startsWith(base)) return false; + const [version, file, ...rest] = binary.slice(base.length).split(path.sep); + return rest.length === 0 && file === 'cli-proxy-api' && /^\d{1,4}\.\d{1,4}\.\d{1,6}$/.test(version); +}; export function validateProxy(value, dataRoot, pin = PIN) { + const pinned = installPaths(dataRoot, pin).binary; if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(k => !['binary', 'port', 'rotateGraceMinutes', 'firstOutputSeconds'].includes(k)) || !Number.isInteger(value.port) || value.port < 1024 || value.port > 65535 || value.port === 8317 - || value.binary !== installPaths(dataRoot, pin).binary) refuse('proxy-config-invalid'); - return { port: value.port, binary: value.binary, ...proxyTiming(value) }; + || (value.binary !== pinned && !releasePath(dataRoot, value.binary))) refuse('proxy-config-invalid'); + return { port: value.port, binary: pinned, ...proxyTiming(value) }; } export function loadProxy(configDir, dataRoot) { const file = path.join(configDir, 'proxy.json'); diff --git a/packages/model-proxy/tests/empty-provider.test.mjs b/packages/model-proxy/tests/empty-provider.test.mjs index ef5a3412d..866f63b90 100644 --- a/packages/model-proxy/tests/empty-provider.test.mjs +++ b/packages/model-proxy/tests/empty-provider.test.mjs @@ -1,4 +1,4 @@ -// P4 pinned8.0.42 regression: absent Claude provider config is HTTP404. +// P4 regression since PIN 8.0.42 (still so on 8.0.79): absent Claude provider config is HTTP 404. // Normalize only that fixed GET; all other methods/endpoints/statuses refuse. import test from 'node:test'; import assert from 'node:assert/strict'; diff --git a/packages/model-proxy/tests/p2-settings.test.mjs b/packages/model-proxy/tests/p2-settings.test.mjs index 28fd3a8dd..f94e9d2d4 100644 --- a/packages/model-proxy/tests/p2-settings.test.mjs +++ b/packages/model-proxy/tests/p2-settings.test.mjs @@ -13,3 +13,13 @@ for (const [field, maximum, code] of [['rotateGraceMinutes', 1440, 'proxy-rotate assert.throws(() => validateProxy({ ...base, [field]: value }, dataRoot), e => e.code === code); }); } +// #1873 PIN bump: a sidecar written under an earlier PIN keeps the instance config valid. +test('a proxy.json written under an earlier PIN reads as the current pinned path', () => { + const old = { ...base, binary: '/fixture-data/opt/cliproxyapi/8.0.42/cli-proxy-api' }; + assert.deepEqual(validateProxy(old, dataRoot), base); + assert.deepEqual(validateProxy({ ...old, rotateGraceMinutes: 60 }, dataRoot), { ...base, rotateGraceMinutes: 60 }); +}); +for (const binary of ['/bad', '/fixture-data/opt/cliproxyapi/8.0.42/other', '/fixture-data/opt/cliproxyapi/latest/cli-proxy-api', + '/fixture-data/opt/cliproxyapi/8.0.42/x/cli-proxy-api', '/fixture-data/opt/cliproxyapi/../cliproxyapi/8.0.42/cli-proxy-api', + '/other-data/opt/cliproxyapi/8.0.42/cli-proxy-api', '/fixture-data/opt/cliproxyapi/8.0/cli-proxy-api', 7]) + test(`proxy-config-invalid: binary ${JSON.stringify(binary)}`, () => assert.throws(() => validateProxy({ ...base, binary }, dataRoot), e => e.code === 'proxy-config-invalid')); diff --git a/packages/model-proxy/tests/page-client.test.mjs b/packages/model-proxy/tests/page-client.test.mjs index cc6331987..a630be9e2 100644 --- a/packages/model-proxy/tests/page-client.test.mjs +++ b/packages/model-proxy/tests/page-client.test.mjs @@ -1,4 +1,4 @@ -// P4 client contract; endpoint/payload cases are probe-pending on PIN8.0.42. +// P4 client contract; endpoint/payload cases were probe-pending on PIN 8.0.42. // Catches management key/body/query values leaking into receipts, redirects/retries. import test from 'node:test'; import assert from 'node:assert/strict';