From 8ae9d8acd9afe6b7776964e427e3e1c8c2050da1 Mon Sep 17 00:00:00 2001 From: Herb Date: Wed, 21 Oct 2026 03:11:34 -0500 Subject: [PATCH] test: add row 197 proof evidence driver for review (#1695) --- .../work/queue-197/proof/config.example.json | 10 + .../herb/work/queue-197/proof/run-proof.mjs | 176 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 agents/herb/work/queue-197/proof/config.example.json create mode 100644 agents/herb/work/queue-197/proof/run-proof.mjs diff --git a/agents/herb/work/queue-197/proof/config.example.json b/agents/herb/work/queue-197/proof/config.example.json new file mode 100644 index 000000000..7520fa508 --- /dev/null +++ b/agents/herb/work/queue-197/proof/config.example.json @@ -0,0 +1,10 @@ +{ + "repo": "/mnt/storage/src/mosaic-stack", + "launcherRoot": "SAGE_OWNED_CLEAN_LAUNCHER_CHECKOUT", + "launcherCandidate": "9aa6d52d9431efd215aa31d748d7e273eb321474", + "candidateA": "9aa6d52d9431efd215aa31d748d7e273eb321474", + "candidateB": "SAGE_APPROVED_212_PLUS_197_COMMIT", + "candidateDifference": "A is approved 197 r5; B must be the approved combined 212+197 integration commit", + "imageReceipt": "SAGE_PINNED_IMAGE_PROVENANCE_RECEIPT", + "runRoot": "/mnt/storage/scratch/mg197-proof-REPLACE_WITH_UNIQUE_NONCE" +} diff --git a/agents/herb/work/queue-197/proof/run-proof.mjs b/agents/herb/work/queue-197/proof/run-proof.mjs new file mode 100644 index 000000000..d668f9ce1 --- /dev/null +++ b/agents/herb/work/queue-197/proof/run-proof.mjs @@ -0,0 +1,176 @@ +// Evidence driver only. Runs no test without Sage's whole-slot grant. +import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, realpathSync, openSync, closeSync, symlinkSync, readdirSync, statSync, readSync } from 'node:fs'; +import { resolve, join, dirname, relative, isAbsolute } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { execFileSync, spawn } from 'node:child_process'; +const proofStart=performance.now(); +const fail=m=>{throw Error(m);}; +const within=(root,path)=>{const p=relative(root,path);return p===''||!p.startsWith('..')&&!isAbsolute(p);}; +const grant=process.env.MOSAIC_PROOF_GRANT; +if(!/^sage-[0-9]+$/.test(grant??''))fail('explicit whole-slot MOSAIC_PROOF_GRANT required'); +const config=JSON.parse(readFileSync(process.argv[2],'utf8')); +for(const key of ['candidateA','candidateB'])if(!/^[a-f0-9]{40}$/.test(config[key]??''))fail('full approved candidate SHA required: '+key); +if(config.candidateA===config.candidateB)fail('two distinct approved candidates required'); +if(!config.candidateDifference?.trim())fail('record why the approved candidates differ'); +const launcher=realpathSync(config.launcherRoot),repo=realpathSync(config.repo),receipt=realpathSync(config.imageReceipt); +const git=(root,...args)=>execFileSync('git',['-C',root,...args],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim(); +if(git(launcher,'rev-parse','HEAD')!==config.launcherCandidate)fail('launcher pin differs'); +if(git(launcher,'status','--porcelain').split('\n').some(l=>l&&l!=='?? node_modules'))fail('launcher checkout dirty'); +const root=resolve(config.runRoot); +if(existsSync(root)||!within('/mnt/storage/scratch',realpathSync(dirname(root))))fail('fresh scratch runRoot required'); +const {validateTooling}=await import(pathToFileURL(join(launcher,'scripts/testing/gate-lane.mjs'))); +const {verifyImage}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-image.mjs'))); +const {runRecords,status}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-state.mjs'))); +const {summarizePairs}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-evidence.mjs'))); +const {cleanupProject}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-docker.mjs'))); +const docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim(); +const image=JSON.parse(readFileSync(receipt,'utf8')); +mkdirSync(root,{mode:0o700});const state=join(root,'state');mkdirSync(state,{mode:0o700}); +const put=(name,value)=>writeFileSync(join(root,name),JSON.stringify(value,null,2)+'\n',{mode:0o600,flag:'wx'}); +const waves=[ + [['solo-a','a',config.candidateA]], + [['solo-b','b',config.candidateB]], + [['pair1-a','a',config.candidateA],['pair1-b','b',config.candidateB]], + [['pair2-b','b',config.candidateA],['pair2-a','a',config.candidateB]], + [['pair3-a','a',config.candidateA],['pair3-b','b',config.candidateB]], +]; +const plan=waves.map(w=>w.map(([label,lane,candidate])=>({label,lane,candidate,dir:join(root,label),checkout:join(root,label,'wt'),out:join(root,label,'out')}))); +put('plan.json',{grant,...config,plan,image,started:new Date().toISOString()}); +// Prepare every clean candidate before the first test. Each invocation has its own clone. +for(const run of plan.flat()){ + mkdirSync(run.dir,{mode:0o700}); + execFileSync('git',['clone','--quiet','--no-hardlinks','--local',repo,run.checkout],{stdio:'pipe'}); + execFileSync('git',['-C',run.checkout,'checkout','--quiet','--detach',run.candidate],{stdio:'pipe'}); + symlinkSync('/mnt/storage/src/mosaic-stack/node_modules',join(run.checkout,'node_modules')); + validateTooling(run.checkout);verifyImage(image,run.checkout); +} +const active=new Set(),watchers=new Map(),objects=new Map();let interrupted=false,firstFailure=null; +const eventError=[]; +const knownRuns=new Map();let abortTimer; +const ownedRuns=()=>runRecords(state).flatMap(r=>r.records.filter(e=>e.kind==='start'&&plan.flat().some(p=>p.checkout===e.checkout))); +function logObject(project,kind,id,name,action){ + if(!/^mg197-[ab]-[a-f0-9]{12}$/.test(project)||!['container','network'].includes(kind)||!/^[a-f0-9]{12,64}$/.test(id))fail('invalid Docker ownership metadata'); + if(name&&!String(name).replace(/^\//,'').startsWith(project))fail('Docker name outside its recorded project'); + const key=kind+':'+id,old=objects.get(key);const value={project,kind,id,name:name??old?.name??null,created:old?.created||action==='create',destroyed:old?.destroyed||action==='destroy'}; + objects.set(key,value);appendFileSync(join(root,'docker-objects.jsonl'),JSON.stringify({...value,action,at:new Date().toISOString()})+'\n',{mode:0o600}); +} +function ensureWatchers(final=false){ + if(final||[...active].some(j=>!knownRuns.has(j.run.checkout)))for(const run of ownedRuns())knownRuns.set(run.checkout,run); + for(const run of knownRuns.values())if(!watchers.has(run.project)){ + // Replay the few seconds before the immutable start was noticed; the random + // exact project label excludes all foreign resources. No raw events are saved. + const child=spawn(docker,['events','--since',String(Math.floor(Date.now()/1000)-5),'--filter','label=com.docker.compose.project='+run.project,'--filter','type=container','--filter','type=network','--filter','event=create','--filter','event=destroy','--format','{{json .}}'],{stdio:['ignore','pipe','pipe']}); + const watcher={child,stopping:false,buffer:''};watchers.set(run.project,watcher); + child.stdout.on('data',bytes=>{watcher.buffer+=bytes;let end;while((end=watcher.buffer.indexOf('\n'))>=0){const line=watcher.buffer.slice(0,end);watcher.buffer=watcher.buffer.slice(end+1);if(!line)continue;try{const event=JSON.parse(line),kind=event.Type,action=event.Action;const attrs=event.Actor?.Attributes??{};if(attrs['com.docker.compose.project']&&attrs['com.docker.compose.project']!==run.project)throw Error('foreign project event');if(['container','network'].includes(kind)&&['create','destroy'].includes(action))logObject(run.project,kind,event.Actor.ID,attrs.name,action);}catch(error){eventError.push(error.message);}}}); + child.stderr.resume();watcher.done=new Promise(resolve=>{child.once('error',error=>{eventError.push(error.message);resolve();});child.once('close',code=>{if(!watcher.stopping)eventError.push('Docker event stream ended early: '+code);resolve();});}); + } +} +function inventory(project){ + const found=[]; + for(const kind of ['container','network']){ + const query=kind==='container'?['ps','-aq']:['network','ls','-q']; + const ids=execFileSync(docker,[...query,'--filter','label=com.docker.compose.project='+project],{encoding:'utf8'}).trim().split(/\s+/).filter(Boolean); + for(const id of ids){ + if(!/^[a-f0-9]{12,64}$/.test(id))fail('invalid inventory ID'); + // Read only name and ownership label, never Config.Env or credentials. + const labels=kind==='container'?'Config.Labels':'Labels'; + const format='{"name":{{json .Name}},"project":{{json (index .'+labels+' "com.docker.compose.project")}}}'; + const item=JSON.parse(execFileSync(docker,[kind,'inspect',id,'--format',format],{encoding:'utf8'})); + if(item.project!==project)fail('foreign Docker object'); + logObject(project,kind,id,item.name,'inventory');found.push({kind,id,name:item.name}); + } + } + return found; +} +function failureDetails(job){ + const events=runRecords(state).find(r=>r.records.some(e=>e.kind==='start'&&e.checkout===job.run.checkout))?.records??[]; + const steps=events.filter(e=>e.kind==='step'); + const paths=firstFailure?.gate===job.run.label&&firstFailure.log?[firstFailure.log]:existsSync(job.run.out)?readdirSync(job.run.out).filter(n=>n.endsWith('.log')).map(n=>join(job.run.out,n)):[]; + for(const log of paths){ + const text=readFileSync(log,'utf8');const bad=text.match(/^(?:\s*✖ (?!failing tests:)|not ok \d+ - |FAIL\s+)(.+)$/m);if(!bad)continue; + const detail=text.match(/✖ failing tests:[\s\S]*$/)?.[0]??text; + const file=detail.match(/^test at (.+?\.test\.(?:mjs|js|cjs))(?::\d+)+/m)?.[1]??detail.match(/file:\/\/([^\s)]+\.test\.(?:mjs|js|cjs))/)?.[1]; + return {lane:job.run.lane,gate:job.run.label,step:log.split('/').at(-1).slice(0,-4),file:file??'unresolved; inspect preserved log',test:bad[1],log}; + } + const failed=steps.find(e=>e.rc||e.requiredSkips?.length); + return {lane:job.run.lane,gate:job.run.label,step:failed?.name??'preflight or cleanup',file:'see gate.log',test:failed?.requiredSkips?.join('; ')??'wrapper failed',log:join(job.run.dir,'gate.log')}; +} +function abortActive(){ + for(const job of active)job.child.kill('SIGTERM'); + if(!abortTimer){abortTimer=setTimeout(()=>{for(const job of active)job.child.kill('SIGKILL');},30000);abortTimer.unref();} +} +function newFailure(job){ + if(!existsSync(job.run.out))return null;job.logs??=new Map(); + for(const name of readdirSync(job.run.out).filter(n=>n.endsWith('.log'))){ + const log=join(job.run.out,name),st=statSync(log),cursor=job.logs.get(log)??{size:0,tail:''};if(st.size===cursor.size)continue; + const fd=openSync(log,'r'),bytes=Buffer.alloc(st.size-cursor.size);try{readSync(fd,bytes,0,bytes.length,cursor.size);}finally{closeSync(fd);} + const text=cursor.tail+bytes.toString('utf8'),match=text.match(/^(?:\s*✖ (?!failing tests:)|not ok \d+ - |FAIL\s+)(.+)$/m); + job.logs.set(log,{size:st.size,tail:text.slice(-8192)}); + if(match)return {lane:job.run.lane,gate:job.run.label,step:name.slice(0,-4),file:'awaiting runner failure details',test:match[1],log}; + } + return null; +} +function monitor(){ + try{ + ensureWatchers(); + for(const job of active){const detail=newFailure(job);if(detail&&!firstFailure){firstFailure=detail;process.stderr.write('FIRST_FAILURE '+JSON.stringify(detail)+'\n');abortActive();}} + if(eventError.length&&!firstFailure){firstFailure={kind:'Docker evidence',errors:[...eventError]};abortActive();} + }catch(error){eventError.push(error.message);abortActive();} +} +function launch(run){ + const fd=openSync(join(run.dir,'gate.log'),'wx',0o600); + const child=spawn('bash',[join(launcher,'scripts/testing/gate-lane.sh'),'run','--lane',run.lane,'--candidate',run.candidate,'--checkout',run.checkout,'--out',run.out,'--state-root',state,'--image-receipt',receipt,'--grant',grant],{stdio:['ignore',fd,fd]});closeSync(fd); + const job={child,run};active.add(job); + job.done=new Promise(resolve=>{child.once('error',error=>{active.delete(job);resolve({rc:127,error:error.message});});child.once('exit',(code,signal)=>{active.delete(job);resolve({rc:code??128,signal});});}); + return job; +} +function gateReceipt(run){ + const item=runRecords(state).find(r=>r.records.some(e=>e.kind==='start'&&e.checkout===run.checkout)); + if(!item)fail('missing start receipt: '+run.label); + const terminal=item.records.filter(e=>e.kind==='terminal').at(-1); + if(!terminal||terminal.rc!==0||!terminal.clean||terminal.imageBuilds!==0)fail('gate failed or unclean: '+run.label); + if(item.records.some(e=>e.requiredSkips?.length))fail('required prerequisite skipped: '+run.label); + return {...terminal,label:run.label,lane:run.lane,candidate:run.candidate,receiptDir:item.dir}; +} +async function stop(){ + const jobs=[...active];for(const j of jobs)j.child.kill('SIGTERM'); + const timer=setTimeout(()=>{for(const j of jobs)j.child.kill('SIGKILL');},30000);timer.unref(); + await Promise.all(jobs.map(j=>j.done));clearTimeout(timer); +} +const signal=()=>{interrupted=true;abortActive();}; +for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.on(name,signal); +const measured=[],start=proofStart; +const monitorTimer=setInterval(monitor,100); +let summary; +try{ + for(const wave of plan){ + if(interrupted)fail('proof interrupted'); + const observed=status(state); + if(observed.legacy.length||observed.a.state!=='CLEAR'||observed.b.state!=='CLEAR')fail('proof requires both lanes clear and no legacy gate'); + const jobs=wave.map(launch); + for(const j of jobs)j.done.then(result=>{if(result.rc){firstFailure??=failureDetails(j);abortActive();}}).catch(error=>{eventError.push(error.message);abortActive();}); + const results=await Promise.all(jobs.map(j=>j.done)); + if(firstFailure?.gate){const failedJob=jobs.find(j=>j.run.label===firstFailure.gate);if(failedJob)firstFailure=failureDetails(failedJob);} + if(interrupted||firstFailure||eventError.length||results.some(r=>r.rc))fail('failed wave: '+wave.map(r=>r.label).join(',')+' '+JSON.stringify(firstFailure)); + measured.push(wave.map(gateReceipt)); + const end=status(state); + if(end.legacy.length||end.a.state!=='CLEAR'||end.b.state!=='CLEAR')fail('unclean wave status'); + } + const solos=measured.slice(0,2).flat(),pairs=measured.slice(2);summary={grant,solos,pairs,summary:summarizePairs(solos,pairs),wallMs:performance.now()-start,finished:new Date().toISOString()}; +}catch(error){ + await stop();put('failed.json',{grant,message:error.message,firstFailure,measured,at:new Date().toISOString()});process.stderr.write(error.message+'\n');process.exitCode=1; +}finally{ + clearInterval(monitorTimer);clearTimeout(abortTimer);await stop();const cleanupErrors=[]; + try{ensureWatchers(true);}catch(error){cleanupErrors.push({message:error.message});} + for(const run of knownRuns.values())try{inventory(run.project);cleanupProject({project:run.project},docker);if(inventory(run.project).length)fail('Docker resources remain');}catch(error){cleanupErrors.push({project:run.project,message:error.message});} + for(const watcher of watchers.values()){watcher.stopping=true;watcher.child.kill('SIGTERM');} + await Promise.all([...watchers.values()].map(w=>w.done)); + let finalStatus;try{finalStatus=status(state);if(finalStatus.legacy.length||finalStatus.a.state!=='CLEAR'||finalStatus.b.state!=='CLEAR')cleanupErrors.push({message:'lane status not clear',status:finalStatus});}catch(error){cleanupErrors.push({message:error.message});} + put('docker-summary.json',{objects:[...objects.values()],eventError,cleanupErrors,finalStatus}); + if(cleanupErrors.length||eventError.length||[...objects.values()].some(o=>!o.name))process.exitCode=1; + if(summary&&!process.exitCode){ + for(const run of knownRuns.values())for(const kind of ['container','network'])if(![...objects.values()].some(o=>o.project===run.project&&o.kind===kind&&o.created)){process.exitCode=1;process.stderr.write('missing Docker creation evidence: '+run.project+' '+kind+'\n');} + if(!process.exitCode){put('summary.json',summary);process.stdout.write('PROOF_PASSED '+join(root,'summary.json')+'\n');} + } + for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.off(name,signal); +}