fix(harness): mark first streamed model output (#797)
This commit is contained in:
1 parent
5df42478b6
commit
c7ddc7d31c
8 files changed
+229
-9
No files matched your search
@@ -0,0 +1,48 @@
|
||||
// Real confined runner and adapter, real local model stream; only fixture children are stopped.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
import { sandboxProblem } from '../../seat/src/sandbox.mjs';
|
||||
import { managedLaunchFixture, pause, waitFor } from './model-managed-launch-fixture.mjs';
|
||||
|
||||
const skip = sandboxProblem() ?? false;
|
||||
const claude = spawnSync('sh', ['-c', 'command -v claude'], {encoding:'utf8'}).stdout.trim();
|
||||
for (const harness of ['pi', 'claude-code']) {
|
||||
for (const mode of ['stream', 'headers', 'keepalive']) {
|
||||
test(`797 ${harness} ${mode}: real model stream controls the managed guard`, {skip: skip || (harness === 'claude-code' && !claude ? 'claude is not on PATH' : false), timeout:40000}, async t => {
|
||||
const modelStream = { mode, seconds:6, duration:8500 };
|
||||
const f = await managedLaunchFixture(t, { harness, real:[harness], modelStream,
|
||||
hostEnv: harness === 'claude-code' ? {MOSAIC_CLAUDE_BIN:claude} : {} });
|
||||
const launched = await f.launcher.launchPm();
|
||||
const session = f.sessions().find(s => s.run === launched.run);
|
||||
assert.ok(session?.live);
|
||||
const sent = f.send('Respond to the test.');
|
||||
try { await waitFor(() => f.state.modelRequests, 'model request reached the local server', 10000); }
|
||||
catch (e) {
|
||||
const file = join(session.runDir, 'turns', '0001.stderr');
|
||||
let diagnostic = fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : 'no adapter stderr';
|
||||
for (const key of f.state.keys) diagnostic = diagnostic.replaceAll(key, '[fixture key]');
|
||||
t.diagnostic(`adapter stderr: ${diagnostic}; launcher log: ${f.logs.join('; ')}`);
|
||||
throw e;
|
||||
}
|
||||
if (mode === 'stream') {
|
||||
await waitFor(() => fs.existsSync(join(session.runDir,'proxy-first-output')) || f.trace.stalls.length, 'model delta or stall observed', 10000);
|
||||
assert.equal(f.trace.stalls.length, 0, 'streaming must not be stopped before its final output');
|
||||
assert.equal(fs.existsSync(join(session.runDir,'proxy-first-output')), true, 'real model delta produces marker');
|
||||
assert.equal(fs.readFileSync(join(session.runDir,'proxy-first-output'),'utf8'), 'output\n');
|
||||
await pause(7000);
|
||||
assert.equal(f.trace.stalls.length, 0, 'streaming past the window never requests a stop');
|
||||
assert.equal(f.sessions().some(s => s.run === session.run && s.live), true);
|
||||
const reply = await f.reply(sent);
|
||||
assert.match(reply.body, /first/);
|
||||
} else {
|
||||
await waitFor(() => f.trace.stalls.length === 1, 'silent model triggers trusted stop', 12000);
|
||||
assert.equal(fs.existsSync(join(session.runDir,'proxy-first-output')), false);
|
||||
assert.equal(f.trace.stalls[0].stopped, true);
|
||||
}
|
||||
f.valueFree();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -21,12 +21,12 @@ const id = value => createHash('sha256').update(value).digest('hex').slice(0,12)
|
||||
// #774 R2: extra names more harnesses to switch on and load (a fallback route's); real names the
|
||||
// harnesses loaded as the real module, so their own preflight runs; hostEnv adds launcher env
|
||||
// such as MOSAIC_CLAUDE_BIN.
|
||||
export async function managedLaunchFixture(t,{proxy=true,launches=true,refuseProbe=false,harness='pi',bwrap=null,admit=null,extra=[],real=[],hostEnv={}}={}){
|
||||
export async function managedLaunchFixture(t,{proxy=true,launches=true,refuseProbe=false,harness='pi',bwrap=null,admit=null,extra=[],real=[],hostEnv={},modelStream=null}={}){
|
||||
const root=fs.mkdtempSync(join(tmpdir(),'ml653-')),closers=[],history=new Set(['fx653-manage']),proxyReceipts=[],logs=[];
|
||||
let server,bus,launcher,stderr;const trace={preflightAttempts:0,successfulPreflights:0,enableBeforeSpawn:[],bindings:[],stalls:[],ops:[]};
|
||||
const checkHistory=entries=>{const text=JSON.stringify(entries);for(const value of history)if(text.includes(value))throw Error('fixture credential leaked');};
|
||||
t.after(async()=>{const failures=[];for(const close of [...closers].reverse()){try{await close();}catch{failures.push('owned-close');}}try{server?.closeAllConnections();if(server?.listening)await new Promise((resolve,reject)=>server.close(e=>e?reject(e):resolve()));}catch{failures.push('owned-server-close');}try{checkHistory({logs,proxyReceipts,trace});}catch{failures.push('value-free-receipts');}if(stderr)process.stderr.write=stderr;try{fs.rmSync(root,{recursive:true,force:true});}catch{failures.push('owned-root-close');}if(failures.length)throw new AggregateError(failures.map(code=>Error(code)),'owned managed fixture cleanup failed');});
|
||||
const f=fixture(root,'acme',doc=>{doc.plugins={...doc.plugins,harness:[harness,...extra]};for(const role of Object.values(doc.roles))role.vars={...role.vars,harness};doc.roles.chief.vars.model='claude-opus-5-5';return doc;});
|
||||
const f=fixture(root,'acme',doc=>{doc.plugins={...doc.plugins,harness:[harness,...extra]};for(const role of Object.values(doc.roles))role.vars={...role.vars,harness};doc.roles.chief.vars.model='claude-opus-5-5';if(modelStream){doc.launch.models['claude-opus-5-5']={provider:'anthropic',model:'claude-opus-5-5'};}return doc;});
|
||||
const env={PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR,MOSAIC_CONFIG:f.config,MOSAIC_ROLES_DIR:f.roles,...hostEnv};
|
||||
// The nonproxy positive fixture supplies its own fake pi login. Managed
|
||||
// proxy fixtures deliberately hold no upstream credential at all.
|
||||
@@ -34,7 +34,25 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro
|
||||
const configDir=join(root,'config'),prefix=`msk-${id(configDir)}-`,sentinel=prefix+'sentinel-'+randomBytes(32).toString('base64url');history.add(sentinel);
|
||||
const state={keys:[sentinel],calls:[],refuseProbe};
|
||||
server=http.createServer(async(req,res)=>{try{
|
||||
const supplied=req.headers.authorization?.slice(7);if(supplied)history.add(supplied);for(const key of state.keys)history.add(key);
|
||||
const supplied=req.headers.authorization?.slice(7) ?? req.headers['x-api-key'];if(supplied)history.add(supplied);for(const key of state.keys)history.add(key);
|
||||
if(modelStream && req.method === 'POST' && /\/messages(?:\?|$)/.test(req.url)) {
|
||||
if (!state.keys.includes(supplied)) { res.writeHead(401);res.end();return; }
|
||||
req.resume(); req.on('end', () => {
|
||||
state.modelRequests = (state.modelRequests ?? 0) + 1;
|
||||
res.writeHead(200, {'content-type':'text/event-stream','cache-control':'no-cache'});res.flushHeaders();
|
||||
const send=(type,data)=>res.write(`event: ${type}\ndata: ${JSON.stringify({type,...data})}\n\n`);
|
||||
let ticker;
|
||||
if(modelStream.mode === 'stream') {
|
||||
send('message_start',{message:{id:'msg_test',type:'message',role:'assistant',model:'claude-opus-5-5',content:[],stop_reason:null,stop_sequence:null,usage:{input_tokens:5,output_tokens:1,cache_creation_input_tokens:0,cache_read_input_tokens:0}}});
|
||||
send('content_block_start',{index:0,content_block:{type:'text',text:''}});
|
||||
send('content_block_delta',{index:0,delta:{type:'text_delta',text:'first'}});
|
||||
ticker=setInterval(()=>send('content_block_delta',{index:0,delta:{type:'text_delta',text:' more'}}),400);
|
||||
const end=setTimeout(()=>{clearInterval(ticker);send('content_block_stop',{index:0});send('message_delta',{delta:{stop_reason:'end_turn',stop_sequence:null},usage:{output_tokens:5}});send('message_stop',{});res.end();},modelStream.duration);
|
||||
res.on('close',()=>clearTimeout(end));
|
||||
} else if(modelStream.mode === 'keepalive') ticker=setInterval(()=>{res.write(': keepalive\n\n');send('ping',{});},300);
|
||||
res.on('close',()=>clearInterval(ticker));
|
||||
});return;
|
||||
}
|
||||
if(req.url==='/v8/management/config/access/api-keys'){
|
||||
state.calls.push(req.method);if(supplied!=='fx653-manage'){res.writeHead(401);res.end('{}');return;}
|
||||
if(req.method==='GET'){res.end(JSON.stringify(state.keys));return;}
|
||||
@@ -48,7 +66,7 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro
|
||||
const git=(...args)=>{const result=spawnSync('git',['-c','user.name=test','-c','[email protected]','-C',f.doc.projects.stack.root,...args],{env:gitEnv(env),encoding:'utf8'});assert.equal(result.status,0,'owned local git fixture succeeds');};
|
||||
git('init','-q','-b','main');fs.writeFileSync(join(f.doc.projects.stack.root,'file.txt'),'one\n');git('add','file.txt');git('commit','-q','-m','one');
|
||||
const adapter=join(root,'adapter.sh');fs.writeFileSync(adapter,'case "$MOSAIC_REQUEST" in *SILENT*) exec /usr/bin/sleep 30;; *) printf %s fx653-private-output; printf "done\\n" > "$MOSAIC_TURN_MARKER";; esac\n',{mode:0o644});
|
||||
const system={...loadSystem({env}),...(proxy?{proxy:{port,binary:'/fixture-unused',firstOutputSeconds:1}}:{})};
|
||||
const system={...loadSystem({env}),...(proxy?{proxy:{port,binary:'/fixture-unused',firstOutputSeconds:modelStream?.seconds ?? 1}}:{})};
|
||||
const {trackers,...boot}=bootConfig({system,businessId:'acme',env});assert.equal(trackers,undefined);
|
||||
stderr=process.stderr.write;process.stderr.write=function(chunk,...args){if(String(chunk).startsWith('model-proxy: ')){proxyReceipts.push(String(chunk));const callback=args.find(a=>typeof a==='function');callback?.();return true;}return stderr.call(this,chunk,...args);};
|
||||
bus=await startBroker(boot);closers.push(()=>bus.close());
|
||||
|
||||
Reference in new issue
Block a user