jason.woltje
d070ed6d4a
A stale-record init now renames the record aside, checks it is still the one it observed, and puts it back otherwise. Two inits racing over the same stale record can no longer delete the record of a live endpoint the other one just started. Fixes the P2 from the test-health review.
204 lines
14 KiB
JavaScript
204 lines
14 KiB
JavaScript
// `mosaic dev fixture`: refusals, the state file's checks, one real
|
|
// init/status/stop of the loopback endpoint in a scratch HOME, and the
|
|
// replacement of a record whose endpoint has ended.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { devCommand, fixtureFile, fixtureLaunchEnv, fixtureLauncherOptions, isEndpoint, orgUsesFixture, readFixture, takeStale } from '../src/dev-fixture.mjs';
|
|
import { FIXTURE_ENTRY, FIXTURE_KEY } from '../../plugins/model-provider/fixture/index.mjs';
|
|
import { TTL_DEFAULT_S, TTL_MAX_S } from '../../plugins/model-provider/fixture/endpoint.mjs';
|
|
import { io } from './helpers.mjs';
|
|
|
|
// devOwnerInstance reads the instance from MOSAIC_CONFIG under $HOME/.config.
|
|
const HOME = mkdtempSync(join(tmpdir(), 'devfx'));
|
|
process.env.HOME = HOME;
|
|
test.after(() => rmSync(HOME, { recursive: true, force: true }));
|
|
const config = name => join(HOME, '.config', `mosaic-${name}`, 'config.json');
|
|
const envFor = (name = 'fxtest', extra = {}) => ({ PATH: process.env.PATH, HOME, MOSAIC_CONFIG: config(name), ...extra });
|
|
let n = 0;
|
|
const dataRoot = () => { const d = join(HOME, `data${++n}`); mkdirSync(d, { mode: 0o700 }); return d; };
|
|
|
|
async function refused(args, env, system, code, exitCode = 3) {
|
|
const x = io(env);
|
|
await assert.rejects(devCommand(args, x, { system }), e => { assert.equal(e.exitCode, exitCode, e.message); if (code) assert.equal(e.code, code, e.message); return true; });
|
|
return x;
|
|
}
|
|
const INIT = ['fixture', 'init', '--model', 'deterministic'];
|
|
|
|
test('init refuses before it starts anything', async () => {
|
|
const system = { dataRoot: dataRoot() };
|
|
await refused(['fixture', 'init'], envFor(), system, undefined, 4);
|
|
await refused(['fixture', 'init', '--model', 'claude'], envFor(), system, undefined, 4);
|
|
await refused([...INIT, '--bind', '0.0.0.0'], envFor(), system, 'fixture-endpoint-not-loopback');
|
|
await refused(INIT, { PATH: process.env.PATH, HOME }, system, 'fixture-instance-refused');
|
|
await refused(INIT, envFor('dev'), system, 'fixture-instance-refused');
|
|
await refused([...INIT, '--instance', 'other'], envFor(), system, 'fixture-instance-mismatch');
|
|
await refused(INIT, envFor('fxtest', { ANTHROPIC_API_KEY: 'sk-ant-value-never-printed' }), system, 'fixture-credential-present');
|
|
await assert.rejects(devCommand(INIT, io(envFor('fxtest', { ZAI_API_KEY: 'zai-value-never-printed' })), { system }), e => /ZAI_API_KEY/.test(e.message) && !/never-printed/.test(e.message));
|
|
await refused(INIT, envFor(), { ...system, proxy: { url: 'http://127.0.0.1:1' } }, 'fixture-proxy-configured');
|
|
for (const ttl of ['-1', '1.5', '59', String(TTL_MAX_S + 1), '060', '1e3', 'soon', ''])
|
|
await assert.rejects(devCommand([...INIT, '--ttl', ttl], io(envFor()), { system }), e => e.exitCode === 4 && /--ttl takes 0 \(no expiry\) or whole seconds/.test(e.message), `--ttl ${ttl}`);
|
|
assert.equal(existsSync(join(system.dataRoot, 'dev-fixture')), false);
|
|
});
|
|
|
|
function writeState(root, record, { dirMode = 0o700, fileMode = 0o600 } = {}) {
|
|
const dir = join(root, 'dev-fixture');
|
|
mkdirSync(dir, { recursive: true, mode: 0o700 }); chmodSync(dir, dirMode);
|
|
writeFileSync(fixtureFile(root), JSON.stringify(record), { mode: fileMode }); chmodSync(fixtureFile(root), fileMode);
|
|
}
|
|
const RECORD = { fixtureVersion: 1, model: 'deterministic', endpoint: 'http://127.0.0.1:45678', port: 45678, pid: 999999, instance: 'fxtest', entry: FIXTURE_ENTRY };
|
|
|
|
test('the state file must be private, a fixture record and loopback', () => {
|
|
const unsafe = e => e.code === 'fixture-state-unsafe';
|
|
assert.equal(readFixture(dataRoot()), null);
|
|
let root = dataRoot(); writeState(root, RECORD);
|
|
assert.equal(readFixture(root).port, 45678);
|
|
root = dataRoot(); writeState(root, RECORD, { dirMode: 0o755 }); assert.throws(() => readFixture(root), unsafe);
|
|
root = dataRoot(); writeState(root, RECORD, { fileMode: 0o644 }); assert.throws(() => readFixture(root), unsafe);
|
|
root = dataRoot(); writeState(root, { ...RECORD, endpoint: 'http://0.0.0.0:45678' }); assert.throws(() => readFixture(root), e => e.code === 'fixture-endpoint-not-loopback');
|
|
root = dataRoot(); writeState(root, { ...RECORD, endpoint: 'http://127.0.0.1:1' }); assert.throws(() => readFixture(root), e => e.code === 'fixture-state-invalid');
|
|
root = dataRoot(); writeState(root, { ...RECORD, entry: { ...FIXTURE_ENTRY, upstream: 'anthropic' } }); assert.throws(() => readFixture(root), e => e.code === 'fixture-state-invalid');
|
|
root = dataRoot(); mkdirSync(join(root, 'dev-fixture'), { mode: 0o700 }); writeFileSync(join(root, 'elsewhere.json'), JSON.stringify(RECORD), { mode: 0o600 });
|
|
symlinkSync(join(root, 'elsewhere.json'), fixtureFile(root)); assert.throws(() => readFixture(root), unsafe);
|
|
root = dataRoot(); writeState(root, RECORD); writeFileSync(fixtureFile(root), 'not json'); assert.throws(() => readFixture(root), e => e.code === 'fixture-state-invalid');
|
|
});
|
|
|
|
test('launcher options: none without a fixture, the endpoint with one, refusals around it', () => {
|
|
assert.deepEqual(fixtureLauncherOptions({ system: { dataRoot: dataRoot() }, env: {}, instance: 'fxtest' }), {});
|
|
const root = dataRoot(); writeState(root, RECORD);
|
|
assert.deepEqual(fixtureLauncherOptions({ system: { dataRoot: root }, env: { PATH: '/usr/bin' }, instance: 'fxtest' }), { testProxy: { baseUrl: RECORD.endpoint, key: FIXTURE_KEY } });
|
|
assert.throws(() => fixtureLauncherOptions({ system: { dataRoot: root, proxy: {} }, env: {}, instance: 'fxtest' }), e => e.code === 'fixture-proxy-configured');
|
|
assert.throws(() => fixtureLauncherOptions({ system: { dataRoot: root }, env: { ANTHROPIC_AUTH_TOKEN: 'x' }, instance: 'fxtest' }), e => e.code === 'fixture-credential-present');
|
|
assert.throws(() => fixtureLauncherOptions({ system: { dataRoot: root }, env: {}, instance: 'other' }), e => e.code === 'fixture-instance-mismatch');
|
|
});
|
|
|
|
test('a fixture serves only organizations that use it; their launches lose provider credentials', () => {
|
|
const fixtureOrg = { plugins: { harness: ['pi'], 'model-provider': ['fixture'] }, launch: { models: { 'fixture-deterministic': FIXTURE_ENTRY } } };
|
|
const realOrg = { plugins: { harness: ['claude-code'] }, launch: { models: { opus: { provider: 'anthropic', model: 'claude-opus-5-5' } } } };
|
|
assert.equal(orgUsesFixture(fixtureOrg), true);
|
|
assert.equal(orgUsesFixture({ launch: { models: { k: { module: 'fixture' } } } }), true);
|
|
assert.equal(orgUsesFixture(realOrg), false);
|
|
const env = { PATH: '/usr/bin', CLAUDE_CODE_OAUTH_TOKEN: 'x', ANTHROPIC_API_KEY: 'y' };
|
|
assert.deepEqual(fixtureLaunchEnv(fixtureOrg, env), { PATH: '/usr/bin' });
|
|
assert.equal(fixtureLaunchEnv(realOrg, env), env);
|
|
const root = dataRoot(); writeState(root, RECORD);
|
|
// A real-model organization on a fixture instance launches without the fixture, credential kept.
|
|
assert.deepEqual(fixtureLauncherOptions({ system: { dataRoot: root }, env, instance: 'fxtest', doc: realOrg }), {});
|
|
// A fixture organization gets the endpoint once its environment is stripped.
|
|
assert.deepEqual(fixtureLauncherOptions({ system: { dataRoot: root }, env: fixtureLaunchEnv(fixtureOrg, env), instance: 'fxtest', doc: fixtureOrg }), { testProxy: { baseUrl: RECORD.endpoint, key: FIXTURE_KEY } });
|
|
// Handed the raw environment it still refuses.
|
|
assert.throws(() => fixtureLauncherOptions({ system: { dataRoot: root }, env, instance: 'fxtest', doc: fixtureOrg }), e => e.code === 'fixture-credential-present');
|
|
});
|
|
|
|
test('a real init, status and stop on loopback', async t => {
|
|
const system = { dataRoot: dataRoot() };
|
|
const env = envFor();
|
|
const x = io(env);
|
|
await devCommand([...INIT, '--instance', 'fxtest', '--json'], x, { system });
|
|
const receipt = JSON.parse(x.out.text);
|
|
t.after(() => { if (isEndpoint(receipt.pid)) process.kill(receipt.pid, 'SIGKILL'); });
|
|
assert.equal(receipt.receipt, 'dev-fixture-init');
|
|
assert.match(receipt.endpoint, /^http:\/\/127\.0\.0\.1:[0-9]+$/);
|
|
assert.deepEqual([receipt.loopback, receipt.external, receipt.upstream, receipt.instance, receipt.principal, receipt.modelKey], [true, false, 'none', 'fxtest', 'local-dev:local-dev-owner', 'fixture-deterministic']);
|
|
assert.deepEqual([receipt.ttlSeconds, receipt.replaced], [TTL_DEFAULT_S, null]);
|
|
assert.equal(isEndpoint(receipt.pid), true);
|
|
assert.equal(JSON.parse(readFileSync(fixtureFile(system.dataRoot), 'utf8')).pid, receipt.pid);
|
|
assert.doesNotMatch(readFileSync(fixtureFile(system.dataRoot), 'utf8'), new RegExp(FIXTURE_KEY));
|
|
|
|
const s = io(env);
|
|
await devCommand(['fixture', 'status', '--json'], s, { system });
|
|
const status = JSON.parse(s.out.text);
|
|
assert.deepEqual([status.ready, status.alive, status.external, status.upstream, status.proxyConfigured, status.health.upstream], [true, true, false, 'none', false, 'none']);
|
|
// The default lifetime reaches the endpoint process.
|
|
assert.deepEqual([status.ttlSeconds, status.health.ttlSeconds], [TTL_DEFAULT_S, TTL_DEFAULT_S]);
|
|
const p = io(env);
|
|
await devCommand(['fixture', 'status', '--json'], p, { system: { ...system, proxy: {} } });
|
|
assert.equal(JSON.parse(p.out.text).ready, false);
|
|
|
|
await refused(INIT, env, system, 'fixture-exists');
|
|
|
|
const st = io(env);
|
|
await devCommand(['fixture', 'stop', '--json'], st, { system });
|
|
assert.deepEqual(JSON.parse(st.out.text).signaled, true);
|
|
assert.equal(isEndpoint(receipt.pid), false);
|
|
assert.equal(readFixture(system.dataRoot), null);
|
|
await refused(['fixture', 'status'], env, system, undefined, 2);
|
|
});
|
|
|
|
// P19 #882. The endpoint's lifetime ends through the same stop as SIGTERM, which leaves the
|
|
// record behind; init used to refuse fixture-exists on it until `dev fixture stop`. The
|
|
// lifetime's timer is tested in packages/plugins/tests/model-provider-fixture.test.mjs.
|
|
test('init replaces the record of an endpoint that has ended, and --ttl reaches the endpoint', async t => {
|
|
const system = { dataRoot: dataRoot() };
|
|
const env = envFor();
|
|
const pids = [];
|
|
t.after(() => { for (const pid of pids) if (isEndpoint(pid)) process.kill(pid, 'SIGKILL'); });
|
|
const init = async (...extra) => {
|
|
const x = io(env); await devCommand([...INIT, '--json', ...extra], x, { system });
|
|
const receipt = JSON.parse(x.out.text); pids.push(receipt.pid); return receipt;
|
|
};
|
|
const status = async () => { const x = io(env); await devCommand(['fixture', 'status', '--json'], x, { system }); return JSON.parse(x.out.text); };
|
|
|
|
const first = await init('--ttl', '0');
|
|
assert.deepEqual([first.ttlSeconds, first.expiresAt, first.replaced], [0, null, null]);
|
|
let s = await status();
|
|
assert.deepEqual([s.ready, s.health.ttlSeconds, s.health.expiresAt], [true, 0, null]);
|
|
|
|
process.kill(first.pid, 'SIGTERM');
|
|
for (let i = 0; i < 50 && isEndpoint(first.pid); i++) await new Promise(r => setTimeout(r, 100));
|
|
assert.equal(isEndpoint(first.pid), false);
|
|
assert.equal(readFixture(system.dataRoot).pid, first.pid, 'an ended endpoint leaves its record');
|
|
s = await status();
|
|
assert.deepEqual([s.ready, s.alive], [false, false]);
|
|
|
|
const second = await init('--ttl', '120');
|
|
assert.notEqual(second.pid, first.pid);
|
|
assert.deepEqual(second.replaced, { pid: first.pid, startedAt: first.startedAt });
|
|
assert.equal(readFixture(system.dataRoot).pid, second.pid);
|
|
s = await status();
|
|
assert.deepEqual([s.ready, s.ttlSeconds, s.health.ttlSeconds], [true, 120, 120]);
|
|
assert.ok(Math.abs(Date.parse(s.health.expiresAt) - Date.parse(second.expiresAt)) < 10000, `${s.health.expiresAt} vs ${second.expiresAt}`);
|
|
assert.equal(Date.parse(second.expiresAt) - Date.parse(second.startedAt), 120000);
|
|
|
|
// A live endpoint's record still refuses.
|
|
await refused(INIT, env, system, 'fixture-exists');
|
|
const stopped = io(env);
|
|
await devCommand(['fixture', 'stop', '--json'], stopped, { system });
|
|
assert.equal(JSON.parse(stopped.out.text).signaled, true);
|
|
|
|
// A record whose pid now names another process (this test runner) is stale too.
|
|
const reused = dataRoot();
|
|
writeState(reused, { ...RECORD, pid: process.pid });
|
|
const x = io(env); await devCommand([...INIT, '--json'], x, { system: { dataRoot: reused } });
|
|
const third = JSON.parse(x.out.text); pids.push(third.pid);
|
|
assert.deepEqual(third.replaced, { pid: process.pid, startedAt: null });
|
|
await devCommand(['fixture', 'stop'], io(env), { system: { dataRoot: reused } });
|
|
assert.equal(isEndpoint(third.pid), false);
|
|
});
|
|
|
|
// Review P2 (org-test-health): two inits both observe the same stale record. The first replaces
|
|
// it with its live endpoint's record; the second, resuming, must not delete that new record.
|
|
test('a stale record is removed only if it is still the one observed; a newer record stays', () => {
|
|
const observed = { ...RECORD, startedAt: '2026-11-08T00:00:00.000Z' };
|
|
const root = dataRoot();
|
|
writeState(root, observed);
|
|
takeStale(root, observed, 'fxtest');
|
|
assert.equal(existsSync(fixtureFile(root)), false, 'the observed stale record goes');
|
|
assert.deepEqual(readdirSync(join(root, 'dev-fixture')), [], 'nothing is left aside');
|
|
|
|
const raced = dataRoot();
|
|
const newer = { ...RECORD, pid: 4242, startedAt: '2026-11-08T00:00:05.000Z' };
|
|
writeState(raced, newer);
|
|
const before = readFileSync(fixtureFile(raced), 'utf8');
|
|
assert.throws(() => takeStale(raced, observed, 'fxtest'), e => e.code === 'fixture-exists');
|
|
assert.equal(readFileSync(fixtureFile(raced), 'utf8'), before, 'the other init\'s record is back, unchanged');
|
|
assert.deepEqual(readdirSync(join(raced, 'dev-fixture')), ['fixture.json'], 'nothing is left aside');
|
|
|
|
const gone = dataRoot();
|
|
mkdirSync(join(gone, 'dev-fixture'), { mode: 0o700 });
|
|
takeStale(gone, observed, 'fxtest');
|
|
assert.equal(existsSync(fixtureFile(gone)), false, 'a record already taken by the other init is not an error');
|
|
});
|