test: add row 197 proof evidence driver for review (#1695)

agents/herb/work/queue-197/proof/config.example.json

@@ -0,0 +1,10 @@
+{
+  "repo": "/mnt/storage/src/mosaic-stack",
+  "launcherRoot": "SAGE_OWNED_CLEAN_LAUNCHER_CHECKOUT",
+  "launcherCandidate": "9aa6d52d9431efd215aa31d748d7e273eb321474",
+  "candidateA": "9aa6d52d9431efd215aa31d748d7e273eb321474",
+  "candidateB": "SAGE_APPROVED_212_PLUS_197_COMMIT",
+  "candidateDifference": "A is approved 197 r5; B must be the approved combined 212+197 integration commit",
+  "imageReceipt": "SAGE_PINNED_IMAGE_PROVENANCE_RECEIPT",
+  "runRoot": "/mnt/storage/scratch/mg197-proof-REPLACE_WITH_UNIQUE_NONCE"
+}

agents/herb/work/queue-197/proof/run-proof.mjs

@@ -0,0 +1,176 @@
+// Evidence driver only. Runs no test without Sage's whole-slot grant.
+import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, realpathSync, openSync, closeSync, symlinkSync, readdirSync, statSync, readSync } from 'node:fs';
+import { resolve, join, dirname, relative, isAbsolute } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { execFileSync, spawn } from 'node:child_process';
+const proofStart=performance.now();
+const fail=m=>{throw Error(m);};
+const within=(root,path)=>{const p=relative(root,path);return p===''||!p.startsWith('..')&&!isAbsolute(p);};
+const grant=process.env.MOSAIC_PROOF_GRANT;
+if(!/^sage-[0-9]+$/.test(grant??''))fail('explicit whole-slot MOSAIC_PROOF_GRANT required');
+const config=JSON.parse(readFileSync(process.argv[2],'utf8'));
+for(const key of ['candidateA','candidateB'])if(!/^[a-f0-9]{40}$/.test(config[key]??''))fail('full approved candidate SHA required: '+key);
+if(config.candidateA===config.candidateB)fail('two distinct approved candidates required');
+if(!config.candidateDifference?.trim())fail('record why the approved candidates differ');
+const launcher=realpathSync(config.launcherRoot),repo=realpathSync(config.repo),receipt=realpathSync(config.imageReceipt);
+const git=(root,...args)=>execFileSync('git',['-C',root,...args],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim();
+if(git(launcher,'rev-parse','HEAD')!==config.launcherCandidate)fail('launcher pin differs');
+if(git(launcher,'status','--porcelain').split('\n').some(l=>l&&l!=='?? node_modules'))fail('launcher checkout dirty');
+const root=resolve(config.runRoot);
+if(existsSync(root)||!within('/mnt/storage/scratch',realpathSync(dirname(root))))fail('fresh scratch runRoot required');
+const {validateTooling}=await import(pathToFileURL(join(launcher,'scripts/testing/gate-lane.mjs')));
+const {verifyImage}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-image.mjs')));
+const {runRecords,status}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-state.mjs')));
+const {summarizePairs}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-evidence.mjs')));
+const {cleanupProject}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-docker.mjs')));
+const docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim();
+const image=JSON.parse(readFileSync(receipt,'utf8'));
+mkdirSync(root,{mode:0o700});const state=join(root,'state');mkdirSync(state,{mode:0o700});
+const put=(name,value)=>writeFileSync(join(root,name),JSON.stringify(value,null,2)+'\n',{mode:0o600,flag:'wx'});
+const waves=[
+ [['solo-a','a',config.candidateA]],
+ [['solo-b','b',config.candidateB]],
+ [['pair1-a','a',config.candidateA],['pair1-b','b',config.candidateB]],
+ [['pair2-b','b',config.candidateA],['pair2-a','a',config.candidateB]],
+ [['pair3-a','a',config.candidateA],['pair3-b','b',config.candidateB]],
+];
+const plan=waves.map(w=>w.map(([label,lane,candidate])=>({label,lane,candidate,dir:join(root,label),checkout:join(root,label,'wt'),out:join(root,label,'out')})));
+put('plan.json',{grant,...config,plan,image,started:new Date().toISOString()});
+// Prepare every clean candidate before the first test. Each invocation has its own clone.
+for(const run of plan.flat()){
+ mkdirSync(run.dir,{mode:0o700});
+ execFileSync('git',['clone','--quiet','--no-hardlinks','--local',repo,run.checkout],{stdio:'pipe'});
+ execFileSync('git',['-C',run.checkout,'checkout','--quiet','--detach',run.candidate],{stdio:'pipe'});
+ symlinkSync('/mnt/storage/src/mosaic-stack/node_modules',join(run.checkout,'node_modules'));
+ validateTooling(run.checkout);verifyImage(image,run.checkout);
+}
+const active=new Set(),watchers=new Map(),objects=new Map();let interrupted=false,firstFailure=null;
+const eventError=[];
+const knownRuns=new Map();let abortTimer;
+const ownedRuns=()=>runRecords(state).flatMap(r=>r.records.filter(e=>e.kind==='start'&&plan.flat().some(p=>p.checkout===e.checkout)));
+function logObject(project,kind,id,name,action){
+ if(!/^mg197-[ab]-[a-f0-9]{12}$/.test(project)||!['container','network'].includes(kind)||!/^[a-f0-9]{12,64}$/.test(id))fail('invalid Docker ownership metadata');
+ if(name&&!String(name).replace(/^\//,'').startsWith(project))fail('Docker name outside its recorded project');
+ const key=kind+':'+id,old=objects.get(key);const value={project,kind,id,name:name??old?.name??null,created:old?.created||action==='create',destroyed:old?.destroyed||action==='destroy'};
+ objects.set(key,value);appendFileSync(join(root,'docker-objects.jsonl'),JSON.stringify({...value,action,at:new Date().toISOString()})+'\n',{mode:0o600});
+}
+function ensureWatchers(final=false){
+ if(final||[...active].some(j=>!knownRuns.has(j.run.checkout)))for(const run of ownedRuns())knownRuns.set(run.checkout,run);
+ for(const run of knownRuns.values())if(!watchers.has(run.project)){
+  // Replay the few seconds before the immutable start was noticed; the random
+  // exact project label excludes all foreign resources. No raw events are saved.
+  const child=spawn(docker,['events','--since',String(Math.floor(Date.now()/1000)-5),'--filter','label=com.docker.compose.project='+run.project,'--filter','type=container','--filter','type=network','--filter','event=create','--filter','event=destroy','--format','{{json .}}'],{stdio:['ignore','pipe','pipe']});
+  const watcher={child,stopping:false,buffer:''};watchers.set(run.project,watcher);
+  child.stdout.on('data',bytes=>{watcher.buffer+=bytes;let end;while((end=watcher.buffer.indexOf('\n'))>=0){const line=watcher.buffer.slice(0,end);watcher.buffer=watcher.buffer.slice(end+1);if(!line)continue;try{const event=JSON.parse(line),kind=event.Type,action=event.Action;const attrs=event.Actor?.Attributes??{};if(attrs['com.docker.compose.project']&&attrs['com.docker.compose.project']!==run.project)throw Error('foreign project event');if(['container','network'].includes(kind)&&['create','destroy'].includes(action))logObject(run.project,kind,event.Actor.ID,attrs.name,action);}catch(error){eventError.push(error.message);}}});
+  child.stderr.resume();watcher.done=new Promise(resolve=>{child.once('error',error=>{eventError.push(error.message);resolve();});child.once('close',code=>{if(!watcher.stopping)eventError.push('Docker event stream ended early: '+code);resolve();});});
+ }
+}
+function inventory(project){
+ const found=[];
+ for(const kind of ['container','network']){
+  const query=kind==='container'?['ps','-aq']:['network','ls','-q'];
+  const ids=execFileSync(docker,[...query,'--filter','label=com.docker.compose.project='+project],{encoding:'utf8'}).trim().split(/\s+/).filter(Boolean);
+  for(const id of ids){
+   if(!/^[a-f0-9]{12,64}$/.test(id))fail('invalid inventory ID');
+   // Read only name and ownership label, never Config.Env or credentials.
+   const labels=kind==='container'?'Config.Labels':'Labels';
+   const format='{"name":{{json .Name}},"project":{{json (index .'+labels+' "com.docker.compose.project")}}}';
+   const item=JSON.parse(execFileSync(docker,[kind,'inspect',id,'--format',format],{encoding:'utf8'}));
+   if(item.project!==project)fail('foreign Docker object');
+   logObject(project,kind,id,item.name,'inventory');found.push({kind,id,name:item.name});
+  }
+ }
+ return found;
+}
+function failureDetails(job){
+ const events=runRecords(state).find(r=>r.records.some(e=>e.kind==='start'&&e.checkout===job.run.checkout))?.records??[];
+ const steps=events.filter(e=>e.kind==='step');
+ const paths=firstFailure?.gate===job.run.label&&firstFailure.log?[firstFailure.log]:existsSync(job.run.out)?readdirSync(job.run.out).filter(n=>n.endsWith('.log')).map(n=>join(job.run.out,n)):[];
+ for(const log of paths){
+  const text=readFileSync(log,'utf8');const bad=text.match(/^(?:\s*✖ (?!failing tests:)|not ok \d+ - |FAIL\s+)(.+)$/m);if(!bad)continue;
+  const detail=text.match(/✖ failing tests:[\s\S]*$/)?.[0]??text;
+  const file=detail.match(/^test at (.+?\.test\.(?:mjs|js|cjs))(?::\d+)+/m)?.[1]??detail.match(/file:\/\/([^\s)]+\.test\.(?:mjs|js|cjs))/)?.[1];
+  return {lane:job.run.lane,gate:job.run.label,step:log.split('/').at(-1).slice(0,-4),file:file??'unresolved; inspect preserved log',test:bad[1],log};
+ }
+ const failed=steps.find(e=>e.rc||e.requiredSkips?.length);
+ return {lane:job.run.lane,gate:job.run.label,step:failed?.name??'preflight or cleanup',file:'see gate.log',test:failed?.requiredSkips?.join('; ')??'wrapper failed',log:join(job.run.dir,'gate.log')};
+}
+function abortActive(){
+ for(const job of active)job.child.kill('SIGTERM');
+ if(!abortTimer){abortTimer=setTimeout(()=>{for(const job of active)job.child.kill('SIGKILL');},30000);abortTimer.unref();}
+}
+function newFailure(job){
+ if(!existsSync(job.run.out))return null;job.logs??=new Map();
+ for(const name of readdirSync(job.run.out).filter(n=>n.endsWith('.log'))){
+  const log=join(job.run.out,name),st=statSync(log),cursor=job.logs.get(log)??{size:0,tail:''};if(st.size===cursor.size)continue;
+  const fd=openSync(log,'r'),bytes=Buffer.alloc(st.size-cursor.size);try{readSync(fd,bytes,0,bytes.length,cursor.size);}finally{closeSync(fd);}
+  const text=cursor.tail+bytes.toString('utf8'),match=text.match(/^(?:\s*✖ (?!failing tests:)|not ok \d+ - |FAIL\s+)(.+)$/m);
+  job.logs.set(log,{size:st.size,tail:text.slice(-8192)});
+  if(match)return {lane:job.run.lane,gate:job.run.label,step:name.slice(0,-4),file:'awaiting runner failure details',test:match[1],log};
+ }
+ return null;
+}
+function monitor(){
+ try{
+  ensureWatchers();
+  for(const job of active){const detail=newFailure(job);if(detail&&!firstFailure){firstFailure=detail;process.stderr.write('FIRST_FAILURE '+JSON.stringify(detail)+'\n');abortActive();}}
+  if(eventError.length&&!firstFailure){firstFailure={kind:'Docker evidence',errors:[...eventError]};abortActive();}
+ }catch(error){eventError.push(error.message);abortActive();}
+}
+function launch(run){
+ const fd=openSync(join(run.dir,'gate.log'),'wx',0o600);
+ const child=spawn('bash',[join(launcher,'scripts/testing/gate-lane.sh'),'run','--lane',run.lane,'--candidate',run.candidate,'--checkout',run.checkout,'--out',run.out,'--state-root',state,'--image-receipt',receipt,'--grant',grant],{stdio:['ignore',fd,fd]});closeSync(fd);
+ const job={child,run};active.add(job);
+ job.done=new Promise(resolve=>{child.once('error',error=>{active.delete(job);resolve({rc:127,error:error.message});});child.once('exit',(code,signal)=>{active.delete(job);resolve({rc:code??128,signal});});});
+ return job;
+}
+function gateReceipt(run){
+ const item=runRecords(state).find(r=>r.records.some(e=>e.kind==='start'&&e.checkout===run.checkout));
+ if(!item)fail('missing start receipt: '+run.label);
+ const terminal=item.records.filter(e=>e.kind==='terminal').at(-1);
+ if(!terminal||terminal.rc!==0||!terminal.clean||terminal.imageBuilds!==0)fail('gate failed or unclean: '+run.label);
+ if(item.records.some(e=>e.requiredSkips?.length))fail('required prerequisite skipped: '+run.label);
+ return {...terminal,label:run.label,lane:run.lane,candidate:run.candidate,receiptDir:item.dir};
+}
+async function stop(){
+ const jobs=[...active];for(const j of jobs)j.child.kill('SIGTERM');
+ const timer=setTimeout(()=>{for(const j of jobs)j.child.kill('SIGKILL');},30000);timer.unref();
+ await Promise.all(jobs.map(j=>j.done));clearTimeout(timer);
+}
+const signal=()=>{interrupted=true;abortActive();};
+for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.on(name,signal);
+const measured=[],start=proofStart;
+const monitorTimer=setInterval(monitor,100);
+let summary;
+try{
+ for(const wave of plan){
+  if(interrupted)fail('proof interrupted');
+  const observed=status(state);
+  if(observed.legacy.length||observed.a.state!=='CLEAR'||observed.b.state!=='CLEAR')fail('proof requires both lanes clear and no legacy gate');
+  const jobs=wave.map(launch);
+  for(const j of jobs)j.done.then(result=>{if(result.rc){firstFailure??=failureDetails(j);abortActive();}}).catch(error=>{eventError.push(error.message);abortActive();});
+  const results=await Promise.all(jobs.map(j=>j.done));
+  if(firstFailure?.gate){const failedJob=jobs.find(j=>j.run.label===firstFailure.gate);if(failedJob)firstFailure=failureDetails(failedJob);}
+  if(interrupted||firstFailure||eventError.length||results.some(r=>r.rc))fail('failed wave: '+wave.map(r=>r.label).join(',')+' '+JSON.stringify(firstFailure));
+  measured.push(wave.map(gateReceipt));
+  const end=status(state);
+  if(end.legacy.length||end.a.state!=='CLEAR'||end.b.state!=='CLEAR')fail('unclean wave status');
+ }
+ const solos=measured.slice(0,2).flat(),pairs=measured.slice(2);summary={grant,solos,pairs,summary:summarizePairs(solos,pairs),wallMs:performance.now()-start,finished:new Date().toISOString()};
+}catch(error){
+ await stop();put('failed.json',{grant,message:error.message,firstFailure,measured,at:new Date().toISOString()});process.stderr.write(error.message+'\n');process.exitCode=1;
+}finally{
+ clearInterval(monitorTimer);clearTimeout(abortTimer);await stop();const cleanupErrors=[];
+ try{ensureWatchers(true);}catch(error){cleanupErrors.push({message:error.message});}
+ for(const run of knownRuns.values())try{inventory(run.project);cleanupProject({project:run.project},docker);if(inventory(run.project).length)fail('Docker resources remain');}catch(error){cleanupErrors.push({project:run.project,message:error.message});}
+ for(const watcher of watchers.values()){watcher.stopping=true;watcher.child.kill('SIGTERM');}
+ await Promise.all([...watchers.values()].map(w=>w.done));
+ let finalStatus;try{finalStatus=status(state);if(finalStatus.legacy.length||finalStatus.a.state!=='CLEAR'||finalStatus.b.state!=='CLEAR')cleanupErrors.push({message:'lane status not clear',status:finalStatus});}catch(error){cleanupErrors.push({message:error.message});}
+ put('docker-summary.json',{objects:[...objects.values()],eventError,cleanupErrors,finalStatus});
+ if(cleanupErrors.length||eventError.length||[...objects.values()].some(o=>!o.name))process.exitCode=1;
+ if(summary&&!process.exitCode){
+  for(const run of knownRuns.values())for(const kind of ['container','network'])if(![...objects.values()].some(o=>o.project===run.project&&o.kind===kind&&o.created)){process.exitCode=1;process.stderr.write('missing Docker creation evidence: '+run.project+' '+kind+'\n');}
+  if(!process.exitCode){put('summary.json',summary);process.stdout.write('PROOF_PASSED '+join(root,'summary.json')+'\n');}
+ }
+ for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.off(name,signal);
+}