From 869d259036f3b298693f8d0098451b76bb88378c Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 10:46:13 -0600 Subject: [PATCH] fix(model-proxy): review round: an organization the proxy once put cliproxy on and that lacks it now is skipped as removed, read from the proxy's own receipts; an organization riding now is already riding whatever its history; a repeated catalogue key keeps the usable row; the page paints a Ride all answer while a load waits, names org-forbidden in plain words and gives a neutral fallback (#1873, lane ride-default) --- packages/model-proxy/src/admin-control.mjs | 25 +++++++--- packages/model-proxy/src/ride.mjs | 36 ++++++++++--- .../model-proxy/tests/ride-fence.test.mjs | 2 +- packages/model-proxy/tests/ride.test.mjs | 50 ++++++++++++++++++- packages/webui/src/public/display.js | 10 ++-- .../webui/tests/proxy-setup-browser.test.mjs | 43 ++++++++++++++++ 6 files changed, 145 insertions(+), 21 deletions(-) diff --git a/packages/model-proxy/src/admin-control.mjs b/packages/model-proxy/src/admin-control.mjs index d1086ce8e..6c5bb2d59 100644 --- a/packages/model-proxy/src/admin-control.mjs +++ b/packages/model-proxy/src/admin-control.mjs @@ -2,11 +2,12 @@ // `mosaic proxy status|unit|init-lock|fence clear --restart`. Host actions run here; init and // the fence clear go to the bus, which decides them. Results and receipts carry labels, key IDs // and process ids only: never a key, token, absolute path or upstream text. +import fs from 'node:fs'; import { homedir } from 'node:os'; import { fetchPinned } from './fetch.mjs'; import { PIN } from './common.mjs'; -import { localRide } from './ride.mjs'; -import { releaseState, initState, fenceState, initLockStatus, clearInitLock, appendReceipt, recentReceipts, createUnitControl, FILE_LABELS } from './admin.mjs'; +import { localRide, riddenOrgs } from './ride.mjs'; +import { releaseState, initState, fenceState, initLockStatus, clearInitLock, appendReceipt, recentReceipts, receiptFile, createUnitControl, FILE_LABELS } from './admin.mjs'; export const proxyCode = (error, fallback) => (/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : fallback); const failure = code => Object.assign(new Error(code), { code }); @@ -36,11 +37,19 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir async function unitStatus() { try { return await control.status(); } catch (error) { return { unit: control.unit, installed: false, active: 'unknown', healthy: false, error: proxyCode(error, 'proxy-unit-failed') }; } } + // The organizations this proxy's own receipts show it put cliproxy on (riddenOrgs). No receipt + // file yet is none; a file that cannot be read is null, and localRide skips as unreadable. + function ridden() { + let text; try { text = fs.readFileSync(receiptFile(dataRoot), 'utf8'); } catch (error) { text = error?.code === 'ENOENT' ? '' : null; } + const ids = riddenOrgs(text); + return id => (ids === null ? null : ids.has(id)); + } + const decide = (rider, wasRidden) => localRide({ ...rider, ridden: wasRidden(rider.id) }); // Ride all, one organization: the local rules first, then the organization's bus, which must // answer and must hold the proxy client. The bus decides the write (plugins.set) as it would // for the owner on the Settings page. The answer names a reason, never bus or upstream text. - async function rideOne(rider) { - const id = rider.id, local = localRide(rider); + async function rideOne(rider, wasRidden) { + const id = rider.id, local = decide(rider, wasRidden); if (local === 'fixture') { let running = true; try { await rider.call('plugins'); } catch { running = false; } return { id, outcome: 'fixture', running }; @@ -56,7 +65,7 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir return { id, outcome: 'added' }; } async function rideSummary() { - try { return (await riders()).map(r => ({ id: r.id, state: localRide(r) })); } catch { return null; } + try { const wasRidden = ridden(); return (await riders()).map(r => ({ id: r.id, state: decide(r, wasRidden) })); } catch { return null; } } function nextSteps({ release, init, unitState, fence, lock }) { if (!release.fetched || release.problem) return ['fetch']; @@ -171,9 +180,9 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir // separate confirmed action (rideMove). async rideAll() { return exclusive('ride-all', async () => { - const list = await riders(); + const list = await riders(), wasRidden = ridden(); const results = []; - for (const rider of list) results.push(await rideOne(rider)); + for (const rider of list) results.push(await rideOne(rider, wasRidden)); const added = results.filter(r => r.outcome === 'added').map(r => r.id); const skipped = results.filter(r => r.outcome === 'skipped' || r.outcome === 'failed').map(r => ({ id: r.id, reason: r.reason })); const fixtures = results.filter(r => r.outcome === 'fixture').map(r => ({ id: r.id, running: r.running })); @@ -188,7 +197,7 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir return exclusive('ride-move', async () => { const rider = (await riders()).find(r => r.id === org); if (!rider) return { org, moved: false, reason: 'unknown' }; - const local = localRide(rider); + const local = decide(rider, ridden()); if (local !== 'fixture') return { org, moved: false, reason: local }; let running = true; try { await rider.call('plugins'); } catch { running = false; } if (running) return { org, moved: false, reason: 'bus-running' }; diff --git a/packages/model-proxy/src/ride.mjs b/packages/model-proxy/src/ride.mjs index ab70e3e88..9381c0b1d 100644 --- a/packages/model-proxy/src/ride.mjs +++ b/packages/model-proxy/src/ride.mjs @@ -26,15 +26,35 @@ export function cliproxyRemoved(text) { } catch { return null; } } +// The organizations the proxy itself once put cliproxy on, from the text of the instance's +// proxy-admin.jsonl: each ride-all receipt's added list and each ride-move receipt's org. The +// organization admin page saves a whole document and leaves no change-log entry, so an +// organization named here that lacks cliproxy now had it taken off by its owner. A torn line is +// skipped; text that is not a string answers null and the caller treats every organization as +// unreadable. +export function riddenOrgs(text) { + if (typeof text !== 'string') return null; + const ids = new Set(); + for (const line of text.split('\n')) { + if (!line.trim()) continue; + let r; try { r = JSON.parse(line); } catch { continue; } + if (r?.op === 'ride-all' && Array.isArray(r.added)) for (const id of r.added) if (typeof id === 'string') ids.add(id); + if (r?.op === 'ride-move' && typeof r.org === 'string') ids.add(r.org); + } + return ids; +} + // The local part of the decision, with no bus call: what Ride all does with one organization -// before it asks the organization's bus anything. -export function localRide({ fixture, cliproxy, changes }) { +// before it asks the organization's bus anything. An organization that uses cliproxy now (and is +// not a fixture organization) is already riding, whatever its history. `ridden` is true when the +// proxy's receipts show cliproxy was put on it, and null when those receipts cannot be read. +export function localRide({ fixture, cliproxy, changes, ridden = false }) { if (typeof fixture !== 'boolean' || typeof cliproxy !== 'boolean') return 'unreadable'; + if (cliproxy && !fixture) return 'already'; const removed = cliproxyRemoved(changes); - if (removed === null) return 'unreadable'; - if (removed) return 'removed'; + if (removed === null || (ridden === null && !cliproxy)) return 'unreadable'; + if (removed || (ridden === true && !cliproxy)) return 'removed'; if (fixture) return 'fixture'; - if (cliproxy) return 'already'; return 'ready'; } @@ -63,7 +83,11 @@ export async function ownerModelsFrom({ proxy, riders }) { for (const catalogue of catalogues) { if (catalogue?.status !== 'ready' || !Array.isArray(catalogue.models)) continue; riding = true; - for (const m of catalogue.models) if (m?.module === 'cliproxy' && typeof m.key === 'string' && !models.has(m.key)) models.set(m.key, m); + // A key two buses list keeps the first usable row: one bus's stale row does not hide the model. + for (const m of catalogue.models) { + if (m?.module !== 'cliproxy' || typeof m.key !== 'string') continue; + if (!models.has(m.key) || (!usableModel(models.get(m.key)) && usableModel(m))) models.set(m.key, m); + } } return { proxy: true, riding, models: [...models.values()] }; } diff --git a/packages/model-proxy/tests/ride-fence.test.mjs b/packages/model-proxy/tests/ride-fence.test.mjs index 8e8967233..47a1bf982 100644 --- a/packages/model-proxy/tests/ride-fence.test.mjs +++ b/packages/model-proxy/tests/ride-fence.test.mjs @@ -67,7 +67,7 @@ test('a fenced proxy refuses each riding organization\'s launches with proxy-unc const b = await ridingBus(t); const dir = fs.mkdtempSync(join(tmpdir(), 'ride-fence-admin-')); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); const admin = createProxyAdmin({ dataRoot: dir, configDir: dir, unit: { status: async () => ({}) }, buses: async () => [], fetch: async () => ({}), - riders: async () => [{ id: 'demo', fixture: false, cliproxy: false, changes: '', call: b.call }] }); + riders: async () => [{ id: 'demo', fixture: false, cliproxy: (b.doc().plugins['model-provider'] ?? []).includes('cliproxy'), changes: '', call: b.call }] }); const ride = await admin.rideAll(); assert.deepEqual(ride.added, ['demo'], JSON.stringify(ride)); assert.deepEqual(b.doc().plugins['model-provider'], ['cliproxy']); diff --git a/packages/model-proxy/tests/ride.test.mjs b/packages/model-proxy/tests/ride.test.mjs index bc5c86cda..4057932a4 100644 --- a/packages/model-proxy/tests/ride.test.mjs +++ b/packages/model-proxy/tests/ride.test.mjs @@ -8,9 +8,9 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { createHash } from 'node:crypto'; import { createProxyAdmin } from '../src/admin-control.mjs'; -import { initFiles, receiptFile } from '../src/admin.mjs'; +import { initFiles, receiptFile, appendReceipt } from '../src/admin.mjs'; import { PIN } from '../src/common.mjs'; -import { cliproxyRemoved, localRide, pickModel, fixtureMove, moveFixtureOrg, readRider, ownerModelsFrom } from '../src/ride.mjs'; +import { cliproxyRemoved, localRide, riddenOrgs, pickModel, fixtureMove, moveFixtureOrg, readRider, ownerModelsFrom } from '../src/ride.mjs'; import * as orgService from '../../organization/src/org-service.mjs'; import { orgUsesFixture } from '../../cli/src/dev-fixture.mjs'; @@ -69,6 +69,48 @@ test('the local rule names unreadable, removed, fixture, already and ready in th assert.equal(localRide({ fixture: true, cliproxy: false, changes: '' }), 'fixture'); assert.equal(localRide({ fixture: false, cliproxy: true, changes: '' }), 'already'); assert.equal(localRide({ fixture: false, cliproxy: false, changes: '' }), 'ready'); + // Review round: an organization that uses cliproxy now is already riding, whatever its history. + assert.equal(localRide({ fixture: false, cliproxy: true, changes: removed }), 'already', 'removed once and added back'); + assert.equal(localRide({ fixture: false, cliproxy: true, changes: null }), 'already'); + assert.equal(localRide({ fixture: true, cliproxy: true, changes: '' }), 'fixture'); + // The proxy's receipts show it put cliproxy on the organization, and it lacks cliproxy now. + assert.equal(localRide({ fixture: false, cliproxy: false, changes: '', ridden: true }), 'removed'); + assert.equal(localRide({ fixture: true, cliproxy: false, changes: '', ridden: true }), 'removed'); + assert.equal(localRide({ fixture: false, cliproxy: true, changes: '', ridden: true }), 'already'); + assert.equal(localRide({ fixture: false, cliproxy: false, changes: '', ridden: null }), 'unreadable'); + assert.equal(localRide({ fixture: false, cliproxy: true, changes: '', ridden: null }), 'already'); +}); + +test('the proxy receipts name each organization ride all added or ride move moved', () => { + const text = [{ op: 'ride-all', added: ['a', 'b'], skipped: [{ id: 'c', reason: 'bus-down' }], fixtures: ['d'] }, { op: 'ride-move', org: 'e', model: 'm' }, + { op: 'fetch', added: ['f'] }, { op: 'ride-all', added: [7] }].map(r => JSON.stringify(r)).join('\n') + '\n{torn'; + assert.deepEqual([...riddenOrgs(text)].sort(), ['a', 'b', 'e']); + assert.deepEqual([...riddenOrgs('')], []); + assert.equal(riddenOrgs(null), null); +}); + +test('an organization ride all once added and its owner then saved without cliproxy is skipped as removed', async t => { + // The organization's file as the Console reads it. The organization admin page saves a whole + // document and writes no change-log entry, so the log stays empty after the owner's removal. + let cliproxy = false; + const b = bus({ providers: [] }); + const { admin: a, dataRoot } = admin(t, { riders: async () => [{ id: 'a', fixture: false, cliproxy, changes: '', call: b.call }] }); + assert.deepEqual((await a.rideAll()).added, ['a']); + cliproxy = true; + assert.deepEqual((await a.rideAll()).already, ['a']); + cliproxy = false; // the owner saved the organization without cliproxy on its admin page + const second = await a.rideAll(); + assert.deepEqual(second.skipped, [{ id: 'a', reason: 'removed' }], JSON.stringify(second)); + assert.equal(b.calls.filter(([verb]) => verb === 'plugins.set').length, 1, 'no second write'); + assert.deepEqual((await a.status()).ride, [{ id: 'a', state: 'removed' }]); + // A fixture organization the proxy once moved, and its owner then put back on stand-in models, is not moved again. + const moved = { id: 'f', fixture: true, cliproxy: false, changes: '', call: bus({ down: true }).call }; + const m = admin(t, { riders: async () => [moved], ownerModels: async () => ({ proxy: true, riding: true, models: [row()] }), move: async () => ({ moved: true, model: row().key }) }); + appendReceipt(m.dataRoot, { op: 'ride-move', org: 'f', model: row().key }); + assert.deepEqual(await m.admin.rideMove({ org: 'f' }), { org: 'f', moved: false, reason: 'removed' }); + // Receipts that cannot be read name every organization without cliproxy unreadable. + fs.chmodSync(receiptFile(dataRoot), 0o200); + assert.deepEqual((await a.rideAll()).skipped, [{ id: 'a', reason: 'unreadable' }]); }); test('the owner model is a usable catalogue row that serves every harness, claude first', () => { @@ -266,4 +308,8 @@ test('the owner models come only from riding buses that answer a ready catalogue await new Promise(r => setImmediate(r)); assert.equal(asked.filter(([id]) => id === 'a').length, 2, 'the second bus was asked while the first hung'); release(null); assert.deepEqual((await pending).models, [row()]); + // Review round: a key one bus lists as unavailable and another as usable keeps the usable row. + const stale = row({ available: false, reason: 'proxy-down', label: 'stale' }); + assert.deepEqual((await ownerModelsFrom({ proxy: true, riders: [rider('s', true, ready([stale])), rider('a', true, ready([row()]))] })).models, [row()]); + assert.deepEqual((await ownerModelsFrom({ proxy: true, riders: [rider('a', true, ready([row()])), rider('s', true, ready([stale]))] })).models, [row()]); }); diff --git a/packages/webui/src/public/display.js b/packages/webui/src/public/display.js index 473856fd6..cbf4b0dab 100644 --- a/packages/webui/src/public/display.js +++ b/packages/webui/src/public/display.js @@ -768,7 +768,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo 'bus-down': 'Its bus is not running. Start it, then ride again.', 'no-proxy-client': 'Its bus started before the proxy was set up. Restart its bus, then ride again.', 'proxy-check-failed': 'Its bus could not check the proxy. Check the proxy, then ride again.', - unreadable: 'Its organization file or change log could not be read.', + unreadable: 'Its organization file, its change log or the proxy\'s receipts could not be read.', failed: 'Its bus refused the change.', 'bus-running': 'Its bus is running. Stop its bus first, then move it.', 'no-proxy': 'Set up the model proxy first.', @@ -776,12 +776,12 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo 'no-model': 'The proxy lists no model for the harness its agents use.', invalid: 'The moved organization file did not pass its check. Nothing was written.', 'revision-conflict': 'The organization changed while it was being moved. Try again.', - 'org-forbidden': 'Only the instance owner can move it.', + 'org-forbidden': 'Moving organizations is not available on this Console yet. It works on a local development Console.', unknown: 'It is not an active organization.', 'not-available': 'Moving is not available on this Console.', fixture: 'It uses stand-in models.', }; - const rideReason = code => RIDE_REASONS[code] ?? `Its bus refused the change (${code}).`; + const rideReason = code => RIDE_REASONS[code] ?? `The change was refused (${code}).`; let rideResult = null; const li = text => { const el = document.createElement('li'); el.textContent = text; return el; }; function paintRide() { @@ -845,7 +845,9 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo async function act(label, action) { if (!sendOk || state?.busy) return; state = { ...state, busy: label }; paintNext(); paintControls(); say('Working…'); - try { await action(); } catch (error) { say(error.message, 'error'); } finally { if (state) state = { ...state, busy: null }; await load(); } + // A load still waiting on accounts makes the load below return at once, so the answer is + // painted here first from the state already read. + try { await action(); } catch (error) { say(error.message, 'error'); } finally { if (state) { state = { ...state, busy: null }; paint(); } await load(); } } const fetchRelease = () => act('fetch', async () => { await admin('fetch', {}); seenFetch = null; say('Fetch started.'); }); const clearLock = () => act('init-lock-clear', async () => { const result = await admin('init-lock/clear', {}); addReceipt(result.receipt); say('Stale init lock cleared.', 'success'); }); diff --git a/packages/webui/tests/proxy-setup-browser.test.mjs b/packages/webui/tests/proxy-setup-browser.test.mjs index 272153c00..281c34bcb 100644 --- a/packages/webui/tests/proxy-setup-browser.test.mjs +++ b/packages/webui/tests/proxy-setup-browser.test.mjs @@ -144,3 +144,46 @@ test('after a reload the receipts come from status, and an initialized proxy fol await wait('window.__writes.some(w=>w.path==="/api/proxy/admin/unit/install")'); await wait('document.querySelector("#proxy-setup-init-box").open===false'); }); + +// #1873 ride-default: the Ride all box. The confirm names the stricter admission, the answer's +// skips and stand-in organizations are painted even while an earlier load still waits on the +// accounts read, and the move confirm says the organization spends real tokens after it. +test('Ride all confirms the stricter admission, paints skips and stand-in organizations while a load waits, and the move warns of real tokens', { timeout: 90000 }, async t => { + const running = { ...fenced, fence: { set: false }, restart: null, next: ['ride-all'], + ride: [{ id: 'north', state: 'ready' }, { id: 'south', state: 'ready' }, { id: 'west', state: 'fixture' }] }; + const { f, b, wait } = await w15Fixture(t, { scenario: { ...scenario(running), + 'POST /api/proxy/admin/ride-all': { body: { added: ['south'], already: [], skipped: [], fixtures: [{ id: 'west' }], receipt: { op: 'ride.all', time: T } } }, + 'POST /api/proxy/admin/ride-move': { body: { moved: false, reason: 'bus-running' } } } }); + // The first accounts and keys reads answer not-configured; every later pair never answers, so a load stays in flight. + await b.call('Page.addScriptToEvaluateOnNewDocument', { source: `(() => { const inner = window.fetch; let n = 0; window.__confirms = []; + window.confirm = text => { window.__confirms.push(text); return true; }; + window.fetch = (input, init) => { const url = String(typeof input === 'string' ? input : input.url); + if (url.includes('/api/proxy/accounts') && n++ > 0) return new Promise(() => {}); + if (url.includes('/api/proxy/keys') && n > 1) return new Promise(() => {}); + return inner(input, init); }; })();` }); + await b.navigate(f.base + '/#/settings/model-proxy'); + await wait('document.querySelector("#proxy-ride")?.hidden===false && document.querySelector("#proxy-setup-go")?.dataset.next==="ride-all"'); + assert.match(await b.evaluate('document.querySelector("#proxy-ride-text").textContent'), /every launch route is checked, direct Anthropic routes too.*2 organizations can ride now/); + assert.match(await b.evaluate('document.querySelector("#proxy-ride-fixtures li").textContent'), /^west uses stand-in models, can be moved to the owner's models\./); + // The first Ride all's own reload paints, then waits on the accounts read for good. + await b.evaluate('document.querySelector("#proxy-ride-go").click()'); + await wait('window.__writes.some(w=>w.path==="/api/proxy/admin/ride-all")'); + assert.match(await b.evaluate('window.__confirms[0]'), /Admission gets stricter there: every launch route is checked, direct Anthropic routes too, and while the proxy is down or fenced every launch in that organization is refused/); + await wait('document.querySelector("#proxy-ride-message").textContent.includes("Added to 1")'); + await wait('document.querySelector("#proxy-ride-go").disabled===false'); + // The second answer lands while that load is still in flight. + await b.evaluate(`window.__scenario['POST /api/proxy/admin/ride-all'] = { body: ${JSON.stringify({ added: [], already: ['south'], skipped: [{ id: 'north', reason: 'no-proxy-client' }], fixtures: [{ id: 'west', running: true }], receipt: { op: 'ride.all', time: T } })} }`); + await b.evaluate('document.querySelector("#proxy-ride-go").click()'); + await wait('window.__writes.filter(w=>w.path==="/api/proxy/admin/ride-all").length===2'); + await wait('document.querySelector("#proxy-ride-message").textContent.includes("skipped 1")'); + await wait('document.querySelectorAll("#proxy-ride-skips li").length===1'); + assert.equal(await b.evaluate('document.querySelector("#proxy-ride-skips li").textContent'), 'Skipped north: Its bus started before the proxy was set up. Restart its bus, then ride again.'); + assert.match(await b.evaluate('document.querySelector("#proxy-ride-fixtures li").textContent'), /Its bus is running: stop it first\./); + assert.equal(await b.evaluate('document.querySelector("#proxy-ride-go").disabled'), false, 'the box is not left busy'); + await b.evaluate('document.querySelector("[data-ride-move=west]").click()'); + await wait('window.__writes.some(w=>w.path==="/api/proxy/admin/ride-move")'); + assert.match(await b.evaluate('window.__confirms[2]'), /^Move west to your models\?.*its agents spend real tokens on your account/); + assert.deepEqual(JSON.parse(await b.evaluate('window.__writes.find(w=>w.path==="/api/proxy/admin/ride-move").body')), { org: 'west' }); + await wait('document.querySelector("#proxy-ride-message").textContent.startsWith("west was not moved.")'); + assert.deepEqual(await b.evaluate('window.__unexpected'), []); +});