fix(harness): mark first streamed model output (#797)

This commit is contained in:
jason.woltje committed 2026-11-06 02:02:43 -06:00
1 parent 5df42478b6
commit c7ddc7d31c
8 files changed
+229 -9

No files matched your search

@@ -0,0 +1,48 @@
// Real confined runner and adapter, real local model stream; only fixture children are stopped.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { spawnSync } from 'node:child_process';
import { join } from 'node:path';
import { sandboxProblem } from '../../seat/src/sandbox.mjs';
import { managedLaunchFixture, pause, waitFor } from './model-managed-launch-fixture.mjs';
const skip = sandboxProblem() ?? false;
const claude = spawnSync('sh', ['-c', 'command -v claude'], {encoding:'utf8'}).stdout.trim();
for (const harness of ['pi', 'claude-code']) {
for (const mode of ['stream', 'headers', 'keepalive']) {
test(`797 ${harness} ${mode}: real model stream controls the managed guard`, {skip: skip || (harness === 'claude-code' && !claude ? 'claude is not on PATH' : false), timeout:40000}, async t => {
const modelStream = { mode, seconds:6, duration:8500 };
const f = await managedLaunchFixture(t, { harness, real:[harness], modelStream,
hostEnv: harness === 'claude-code' ? {MOSAIC_CLAUDE_BIN:claude} : {} });
const launched = await f.launcher.launchPm();
const session = f.sessions().find(s => s.run === launched.run);
assert.ok(session?.live);
const sent = f.send('Respond to the test.');
try { await waitFor(() => f.state.modelRequests, 'model request reached the local server', 10000); }
catch (e) {
const file = join(session.runDir, 'turns', '0001.stderr');
let diagnostic = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : 'no adapter stderr';
for (const key of f.state.keys) diagnostic = diagnostic.replaceAll(key, '[fixture key]');
t.diagnostic(`adapter stderr: ${diagnostic}; launcher log: ${f.logs.join('; ')}`);
throw e;
}
if (mode === 'stream') {
await waitFor(() => fs.existsSync(join(session.runDir,'proxy-first-output')) || f.trace.stalls.length, 'model delta or stall observed', 10000);
assert.equal(f.trace.stalls.length, 0, 'streaming must not be stopped before its final output');
assert.equal(fs.existsSync(join(session.runDir,'proxy-first-output')), true, 'real model delta produces marker');
assert.equal(fs.readFileSync(join(session.runDir,'proxy-first-output'),'utf8'), 'output\n');
await pause(7000);
assert.equal(f.trace.stalls.length, 0, 'streaming past the window never requests a stop');
assert.equal(f.sessions().some(s => s.run === session.run && s.live), true);
const reply = await f.reply(sent);
assert.match(reply.body, /first/);
} else {
await waitFor(() => f.trace.stalls.length === 1, 'silent model triggers trusted stop', 12000);
assert.equal(fs.existsSync(join(session.runDir,'proxy-first-output')), false);
assert.equal(f.trace.stalls[0].stopped, true);
}
f.valueFree();
});
}
}
@@ -21,12 +21,12 @@ const id = value => createHash('sha256').update(value).digest('hex').slice(0,12)
// #774 R2: extra names more harnesses to switch on and load (a fallback route's); real names the
// harnesses loaded as the real module, so their own preflight runs; hostEnv adds launcher env
// such as MOSAIC_CLAUDE_BIN.
export async function managedLaunchFixture(t,{proxy=true,launches=true,refuseProbe=false,harness='pi',bwrap=null,admit=null,extra=[],real=[],hostEnv={}}={}){
export async function managedLaunchFixture(t,{proxy=true,launches=true,refuseProbe=false,harness='pi',bwrap=null,admit=null,extra=[],real=[],hostEnv={},modelStream=null}={}){
const root=fs.mkdtempSync(join(tmpdir(),'ml653-')),closers=[],history=new Set(['fx653-manage']),proxyReceipts=[],logs=[];
let server,bus,launcher,stderr;const trace={preflightAttempts:0,successfulPreflights:0,enableBeforeSpawn:[],bindings:[],stalls:[],ops:[]};
const checkHistory=entries=>{const text=JSON.stringify(entries);for(const value of history)if(text.includes(value))throw Error('fixture credential leaked');};
t.after(async()=>{const failures=[];for(const close of [...closers].reverse()){try{await close();}catch{failures.push('owned-close');}}try{server?.closeAllConnections();if(server?.listening)await new Promise((resolve,reject)=>server.close(e=>e?reject(e):resolve()));}catch{failures.push('owned-server-close');}try{checkHistory({logs,proxyReceipts,trace});}catch{failures.push('value-free-receipts');}if(stderr)process.stderr.write=stderr;try{fs.rmSync(root,{recursive:true,force:true});}catch{failures.push('owned-root-close');}if(failures.length)throw new AggregateError(failures.map(code=>Error(code)),'owned managed fixture cleanup failed');});
const f=fixture(root,'acme',doc=>{doc.plugins={...doc.plugins,harness:[harness,...extra]};for(const role of Object.values(doc.roles))role.vars={...role.vars,harness};doc.roles.chief.vars.model='claude-opus-5-5';return doc;});
const f=fixture(root,'acme',doc=>{doc.plugins={...doc.plugins,harness:[harness,...extra]};for(const role of Object.values(doc.roles))role.vars={...role.vars,harness};doc.roles.chief.vars.model='claude-opus-5-5';if(modelStream){doc.launch.models['claude-opus-5-5']={provider:'anthropic',model:'claude-opus-5-5'};}return doc;});
const env={PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR,MOSAIC_CONFIG:f.config,MOSAIC_ROLES_DIR:f.roles,...hostEnv};
// The nonproxy positive fixture supplies its own fake pi login. Managed
// proxy fixtures deliberately hold no upstream credential at all.
@@ -34,7 +34,25 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro
const configDir=join(root,'config'),prefix=`msk-${id(configDir)}-`,sentinel=prefix+'sentinel-'+randomBytes(32).toString('base64url');history.add(sentinel);
const state={keys:[sentinel],calls:[],refuseProbe};
server=http.createServer(async(req,res)=>{try{
const supplied=req.headers.authorization?.slice(7);if(supplied)history.add(supplied);for(const key of state.keys)history.add(key);
const supplied=req.headers.authorization?.slice(7) ?? req.headers['x-api-key'];if(supplied)history.add(supplied);for(const key of state.keys)history.add(key);
if(modelStream && req.method === 'POST' && /\/messages(?:\?|$)/.test(req.url)) {
if (!state.keys.includes(supplied)) { res.writeHead(401);res.end();return; }
req.resume(); req.on('end', () => {
state.modelRequests = (state.modelRequests ?? 0) + 1;
res.writeHead(200, {'content-type':'text/event-stream','cache-control':'no-cache'});res.flushHeaders();
const send=(type,data)=>res.write(`event: ${type}\ndata: ${JSON.stringify({type,...data})}\n\n`);
let ticker;
if(modelStream.mode === 'stream') {
send('message_start',{message:{id:'msg_test',type:'message',role:'assistant',model:'claude-opus-5-5',content:[],stop_reason:null,stop_sequence:null,usage:{input_tokens:5,output_tokens:1,cache_creation_input_tokens:0,cache_read_input_tokens:0}}});
send('content_block_start',{index:0,content_block:{type:'text',text:''}});
send('content_block_delta',{index:0,delta:{type:'text_delta',text:'first'}});
ticker=setInterval(()=>send('content_block_delta',{index:0,delta:{type:'text_delta',text:' more'}}),400);
const end=setTimeout(()=>{clearInterval(ticker);send('content_block_stop',{index:0});send('message_delta',{delta:{stop_reason:'end_turn',stop_sequence:null},usage:{output_tokens:5}});send('message_stop',{});res.end();},modelStream.duration);
res.on('close',()=>clearTimeout(end));
} else if(modelStream.mode === 'keepalive') ticker=setInterval(()=>{res.write(': keepalive\n\n');send('ping',{});},300);
res.on('close',()=>clearInterval(ticker));
});return;
}
if(req.url==='/v8/management/config/access/api-keys'){
state.calls.push(req.method);if(supplied!=='fx653-manage'){res.writeHead(401);res.end('{}');return;}
if(req.method==='GET'){res.end(JSON.stringify(state.keys));return;}
@@ -48,7 +66,7 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro
const git=(...args)=>{const result=spawnSync('git',['-c','user.name=test','-c','[email protected]','-C',f.doc.projects.stack.root,...args],{env:gitEnv(env),encoding:'utf8'});assert.equal(result.status,0,'owned local git fixture succeeds');};
git('init','-q','-b','main');fs.writeFileSync(join(f.doc.projects.stack.root,'file.txt'),'one\n');git('add','file.txt');git('commit','-q','-m','one');
const adapter=join(root,'adapter.sh');fs.writeFileSync(adapter,'case "$MOSAIC_REQUEST" in *SILENT*) exec /usr/bin/sleep 30;; *) printf %s fx653-private-output; printf "done\\n" > "$MOSAIC_TURN_MARKER";; esac\n',{mode:0o644});
const system={...loadSystem({env}),...(proxy?{proxy:{port,binary:'/fixture-unused',firstOutputSeconds:1}}:{})};
const system={...loadSystem({env}),...(proxy?{proxy:{port,binary:'/fixture-unused',firstOutputSeconds:modelStream?.seconds ?? 1}}:{})};
const {trackers,...boot}=bootConfig({system,businessId:'acme',env});assert.equal(trackers,undefined);
stderr=process.stderr.write;process.stderr.write=function(chunk,...args){if(String(chunk).startsWith('model-proxy: ')){proxyReceipts.push(String(chunk));const callback=args.find(a=>typeof a==='function');callback?.();return true;}return stderr.call(this,chunk,...args);};
bus=await startBroker(boot);closers.push(()=>bus.close());