stack/scripts/testing/gate-lane.mjs

import { createHash } from 'node:crypto';
import { realpathSync, existsSync, statSync, readdirSync } from 'node:fs';
import { join, resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { shortRoot, checkHost, hostFacts, refuse, within, SCRATCH, identity } from './lane-state.mjs';
const BUILDER=resolve(dirname(fileURLToPath(import.meta.url)),'../..');
const git=(root,...args)=>execFileSync('git',['-C',root,...args],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim();
export function validateRun(o){
  if(!['a','b'].includes(o.lane))refuse('invalid lane');
  if(typeof o.grant!=='string'||!o.grant.trim())refuse('missing grant reference');
  for(const key of ['checkout','out','stateRoot'])if(typeof o[key]!=='string'||!o[key])refuse(`missing ${key}`);
  const tempRoot=shortRoot(o.tempRoot);
  const checkout=realpathSync(o.checkout);
  if(checkout===BUILDER||checkout==='/mnt/storage/src/mosaic-stack'||!within(SCRATCH,checkout))refuse('canonical or builder checkout');
  if(!/^[a-f0-9]{40}$/.test(o.candidate)||git(checkout,'rev-parse','HEAD')!==o.candidate)refuse('wrong candidate SHA');
  const changed=git(checkout,'status','--porcelain','--untracked-files=normal').split('\n').filter(x=>x&&x!=='?? node_modules');
  if(changed.length)refuse('dirty checkout');
  if(!existsSync(join(checkout,'node_modules'))||!statSync(join(checkout,'node_modules')).isDirectory())refuse('missing node_modules');
  const out=resolve(o.out);if(existsSync(out))refuse('output already exists');
  if(!within(dirname(checkout),out)||!within(SCRATCH,realpathSync(dirname(out))))refuse('output escapes run scratch');
  const stateRoot=resolve(o.stateRoot);if(!within(SCRATCH,stateRoot))refuse('state outside scratch');
  checkHost(o.lane,o.hostFacts??hostFacts());
  return {...o,tempRoot,checkout,out,stateRoot};
}
export function validateTooling(checkout){
  for(const name of readdirSync(join(BUILDER,'scripts/testing')).filter(n=>/\.(mjs|sh)$/.test(n))){
    const candidate=join(checkout,'scripts/testing',name),launcher=join(BUILDER,'scripts/testing',name);
    if(!existsSync(candidate)||!readFileSync(candidate).equals(readFileSync(launcher)))refuse('candidate tooling differs from launcher');
  }
}
export function discoverSteps(checkout,concurrency=2){
  if(concurrency!==2)refuse('test concurrency must be 2');
  const steps=[];
  for(const p of readdirSync(join(checkout,'packages')).sort())if(p!=='queue'&&existsSync(join(checkout,'packages',p,'tests'))&&statSync(join(checkout,'packages',p,'tests')).isDirectory())steps.push({name:`pkg-${p}`,cwd:join(checkout,'packages',p),argv:['node','--test','--test-concurrency=2',...readdirSync(join(checkout,'packages',p,'tests')).filter(n=>n.endsWith('.test.mjs')).sort().map(n=>`tests/${n}`)]});
  for(const f of readdirSync(join(checkout,'scripts')).sort())if(/^test-.+\.sh$/.test(f)&&statSync(join(checkout,'scripts',f)).isFile())steps.push({name:`script-${f.slice(5,-3)}`,cwd:checkout,argv:['bash',`scripts/${f}`]});
  return steps;
}

export async function executeSteps(steps,{out,env,record,onChild=()=>{},signal}){
  const {openSync,closeSync}=await import('node:fs');const {spawn}=await import('node:child_process');
  for(const step of steps){
    if(signal?.aborted)return {rc:130};
    const begin=performance.now(),startMs=Date.now();const fd=openSync(join(out,`${step.name}.log`),'wx',0o600);
    // Keep the supervisor private while matching the legacy test environment.
    const child=spawn('/bin/bash',['-c','umask 022; exec "$@"','gate-step',...step.argv],{cwd:step.cwd,env,detached:true,stdio:['ignore',fd,fd]});closeSync(fd);
    onChild(child);record({kind:'child',group:child.pid,owner:identity(child.pid)});
    let escalation;
    const abort=()=>{try{process.kill(-child.pid,'SIGTERM');}catch{};escalation=setTimeout(()=>{try{process.kill(-child.pid,'SIGKILL');}catch{}},3000);escalation.unref();};
    signal?.addEventListener('abort',abort,{once:true});
    const rc=await new Promise(resolve=>{child.once('error',()=>resolve(127));child.once('close',(code,sig)=>resolve(signal?.aborted?130:(code??(sig?128:1))));});
    clearTimeout(escalation);signal?.removeEventListener('abort',abort);onChild(null);
    const bytes=readFileSync(join(out,`${step.name}.log`)),text=bytes.toString('utf8');
    const counts={};for(const name of ['pass','fail','skipped']){const m=text.match(new RegExp('^(?:ℹ |# )'+name+' (\\d+)','m'));if(m)counts[name]=Number(m[1]);}
    if(counts.pass===undefined){const summaries=[...text.matchAll(/(\d+) passed, (\d+) failed/g)];const last=summaries.at(-1);if(last){counts.pass=Number(last[1]);counts.fail=Number(last[2]);}}
    const requiredSkips=text.split('\n').filter(line=>/^skip\b/i.test(line)&&/(docker|daemon|sandbox|systemd|credential)/i.test(line));
    record({kind:'step',name:step.name,startMs,endMs:Date.now(),wallMs:performance.now()-begin,rc,counts,requiredSkips,logSha256:createHash('sha256').update(bytes).digest('hex')});
    if(rc!==0||requiredSkips.length)return {rc:rc||2};
  }
  return {rc:0};
}

export async function executePinnedSteps(steps,options,{receipt,checkout,inspect,afterStep=()=>{}}){
  for(const step of steps){
    verifyImage(receipt,checkout,inspect);
    const result=await executeSteps([step],options);afterStep(step);
    if(result.rc){verifyImage(receipt,checkout,inspect);return {...result,imageBuilds:0};}
  }
  verifyImage(receipt,checkout,inspect);return {rc:0,imageBuilds:0};
}

import { mkdirSync, lstatSync, fstatSync, mkdtempSync, writeFileSync, readFileSync, openSync, readlinkSync, rmdirSync, chmodSync } from 'node:fs';
import { pathToFileURL } from 'node:url';
import { spawnSync } from 'node:child_process';
import { status, appendRecord, runRecords, processInventory, sameProcess, nonce } from './lane-state.mjs';
import { verifyImage, imageFingerprint, inspectImage, prepareImage } from './lane-image.mjs';
import { cleanupProject } from './lane-docker.mjs';
import { homeMetadata, attributeEntries, socketInventory } from './lane-evidence.mjs';
function options(argv){
  const result={verb:'run',stateRoot:'/mnt/storage/scratch/mosaic-gate-lanes',lane:'a',exclusive:false};
  const names={'--state-root':'stateRoot','--lane':'lane','--candidate':'candidate','--checkout':'checkout','--out':'out','--temp-root':'tempRoot','--grant':'grant','--image-receipt':'imageReceipt','--run-id':'runId'};
  const args=[...argv];if(args[0]&&!args[0].startsWith('--'))result.verb=args.shift();
  if(!['run','status','recover','prepare-image','lock-dir'].includes(result.verb))refuse('unknown verb');
  while(args.length){const key=args.shift();if(key==='--exclusive'){result.exclusive=true;continue;}if(!names[key]||!args.length)refuse('invalid or missing option');result[names[key]]=args.shift();}
  return result;
}
function stateDirectory(path,create=false){
  const root=resolve(path);if(!within(SCRATCH,root)||root===SCRATCH)refuse('state must be private scratch');
  if(create){const parent=realpathSync(dirname(root));if(!within(SCRATCH,parent))refuse('state parent escapes scratch');mkdirSync(root,{recursive:true,mode:0o700});}
  if(existsSync(root)){const st=lstatSync(root);if(st.isSymbolicLink()||!st.isDirectory()||st.uid!==process.getuid()||(st.mode&0o077))refuse('state is not a private owned directory');
    for(const n of ['a.lock','b.lock','image.lock'])if(existsSync(join(root,n))&&!lstatSync(join(root,n)).isFile())refuse('lock is not an ordinary file');
  }
  return root;
}
export function verifyLocks(o){
  for(const [fd,name] of o.exclusive?[[3,'a.lock'],[4,'b.lock'],[5,'image.lock']]:[[3,`${o.lane}.lock`],[5,'image.lock']]){
    let opened,path;try{opened=fstatSync(fd);path=lstatSync(join(o.stateRoot,name));}catch{refuse('lock descriptors missing');}
    if(!opened.isFile()||!path.isFile()||opened.ino!==path.ino||opened.dev!==path.dev||path.uid!==process.getuid())refuse('lock file replaced');
    // A separate open must fail while our shared/exclusive lock is held.
    const r=spawnSync('flock',['-n','-x',join(o.stateRoot,name),'true']);if(r.status!==1)refuse('lock descriptor is not locked');
  }
}
export function activeConflicts(o,observations={}){
  const current=status(o.stateRoot,{...observations,ignoreLocks:o.exclusive?['a','b','image']:[o.lane,'image']});
  if(current.legacy.length)refuse('LEGACY BUSY: unmanaged gate, test or Docker process');
  for(const lane of o.exclusive?['a','b']:[o.lane])if(current[lane].state!=='CLEAR')refuse(`${lane} ${current[lane].state}`);
  for(const r of runRecords(o.stateRoot)){const start=r.records.find(e=>e.kind==='start');if(!r.records.some(e=>['terminal','recovered'].includes(e.kind)&&e.clean)&&start?.checkout===o.checkout)refuse('checkout already owned');}
}
export function cleanEnv(o,run,docker){
  const kept=['PATH','HOME','USER','LOGNAME','SHELL','LANG','LC_ALL','LC_CTYPE','TZ','TERM','XDG_RUNTIME_DIR','DBUS_SESSION_BUS_ADDRESS','DOCKER_HOST','CHROMIUM'];
  const env=Object.fromEntries(kept.filter(k=>typeof process.env[k]==='string').map(k=>[k,process.env[k]]));
  Object.assign(env,{TMPDIR:o.tempRoot,TMP:o.tempRoot,TEMP:o.tempRoot,MOSAIC_GATE_LANE:o.lane,MOSAIC_GATE_STATE:o.stateRoot,MOSAIC_TEST_CONCURRENCY:'2',COMPOSE_PROJECT_NAME:run.project,JITI_FS_CACHE:join(o.tempRoot,'jiti'),XDG_CACHE_HOME:join(o.tempRoot,'cache'),MOSAIC_CONFIG:join(o.tempRoot,'missing-config.json'),MOSAIC_GATE_DOCKER_RECORD:join(o.out,'docker.json'),
    // A step reaching Compose mounts this fixture, never the host's pi auth file (row 244).
    PI_AUTH_FILE:join(o.tempRoot,'pi-auth.json')});
  env.PATH=`${join(o.out,'bin')}:${env.PATH??'/usr/bin:/bin'}`;
  return env;
}
function fixtureProcesses(run){
  const all=processInventory(),ancestors=new Set([process.pid]);let parent=process.pid;
  while(parent){const p=all.find(p=>p.pid===parent);if(!p||ancestors.has(p.ppid))break;ancestors.add(p.ppid);parent=p.ppid;}
  return all.filter(p=>{
    if(ancestors.has(p.pid)||p.state==='Z')return false;
    let cwd;try{cwd=readlinkSync(`/proc/${p.pid}/cwd`);}catch{}
    return p.argv.some(a=>typeof a==='string'&&(a===run.tempRoot||a.startsWith(run.tempRoot+'/')))||cwd&&(within(run.tempRoot,cwd)||run.checkout&&within(run.checkout,cwd));
  });
}
async function cleanProcesses(run,record){
  const before=fixtureProcesses(run);
  if(run.stateRoot&&run.id){
    const procs=processInventory(),known=new Set(before.map(p=>p.pid));
    const {records}=await import('./lane-state.mjs');
    for(const child of records(join(run.stateRoot,'runs',run.id)).filter(e=>e.kind==='child')){
      const members=procs.filter(p=>p.group===child.group&&p.state!=='Z');
      if(members.length&&sameProcess(child.owner,procs.find(p=>p.pid===child.owner?.pid))){
        try{process.kill(-child.group,'SIGKILL');}catch(e){if(e.code!=='ESRCH')throw e;}
      }else if(members.some(p=>!known.has(p.pid)))refuse('unproven orphan process-group ownership');
    }
  }
  record({kind:'leftover-processes',objects:before.map(({pid,start,boot})=>({pid,start,boot}))});
  for(const p of before){if(!sameProcess(p))continue;let raw;try{raw=readFileSync(`/proc/${p.pid}/cgroup`,'utf8');}catch(e){if(e.code==='ENOENT')continue;throw e;}const scope=raw.trim().split('/').find(x=>/^mosaic-chat-[a-z0-9]+\.scope$/.test(x));
    if(scope){const shown=execFileSync('systemctl',['--user','show',scope,'-p','InvocationID','--value'],{encoding:'utf8'}).trim();if(!/^[a-f0-9]{32}$/.test(shown))refuse('scope identity unknown');record({kind:'scope-cleanup',unit:scope,invocation:shown});execFileSync('systemctl',['--user','kill','--signal=SIGKILL',scope],{stdio:'pipe'});}
    if(sameProcess(p))try{process.kill(p.pid,'SIGKILL');}catch(e){if(e.code!=='ESRCH')throw e;}
  }
  for(let n=0;n<30;n++){if(!fixtureProcesses(run).some(p=>p.state!=='Z'))return before.length===0;await new Promise(r=>setTimeout(r,100));}
  refuse('owned fixture process remains');
}
async function recoverRun(o){
  if(!o.grant)refuse('missing recovery grant');
  if(!/^[a-f0-9]{12}$/.test(o.runId??''))refuse('invalid run ID');
  const r=runRecords(o.stateRoot).find(r=>r.id===o.runId);if(!r)refuse('run not found');
  const start=r.records.find(x=>x.kind==='start');if(start?.mode==='prepare-image'&&status(o.stateRoot).legacy.length)refuse('image builder still active or unknown');if(!start||sameProcess(start.owner))refuse('run owner live or unknown');
  if(r.records.some(x=>['terminal','recovered'].includes(x.kind)&&x.clean))refuse('run already clean');
  shortRoot(start.tempRoot);const record=e=>appendRecord(r.dir,e);
  const procs=processInventory(),known=new Set(fixtureProcesses(start).map(p=>p.pid));
  for(const event of r.records.filter(e=>e.kind==='child')){
    const members=procs.filter(p=>p.group===event.group&&p.state!=='Z');
    if(members.length&&!sameProcess(event.owner,procs.find(p=>p.pid===event.owner?.pid))&&members.some(p=>!known.has(p.pid)))refuse('orphan process-group ownership unknown');
    if(members.length&&sameProcess(event.owner)){process.kill(-event.group,'SIGKILL');}
  }
  await cleanProcesses(start,record);cleanupProject(start,start.docker);
  appendRecord(r.dir,{kind:'recovered',clean:true,grant:o.grant,owner:identity()});return {recovered:r.id};
}
async function runGate(input){
  for(const key of ['checkout','out','candidate','grant','imageReceipt'])if(!input[key])refuse(`missing ${key}`);
  const automatic=!input.tempRoot,tempRoot=input.tempRoot??mkdtempSync(`/mnt/storage/scratch/g${input.lane}-`);
  let o,receipt,docker;
  try{
    o=validateRun({...input,tempRoot});
    validateTooling(o.checkout);
    activeConflicts(o);
    receipt=JSON.parse(readFileSync(o.imageReceipt,'utf8'));verifyImage(receipt,o.checkout);
    docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim();
  }catch(e){if(automatic)rmdirSync(tempRoot);throw e;}
  const id=nonce(),run={kind:'start',id,stateRoot:o.stateRoot,lane:o.lane,exclusive:o.exclusive,owner:identity(),grant:o.grant,candidate:o.candidate,checkout:o.checkout,tempRoot:o.tempRoot,out:o.out,project:`mg197-${o.lane}-${id}`,tag:receipt.tag,imageId:receipt.imageId,docker};
  const dir=join(o.stateRoot,'runs',id);appendRecord(dir,run);const record=e=>{appendRecord(dir,e);if(e.kind==='step')process.stdout.write(`${e.name} rc=${e.rc} wall_ms=${Math.round(e.wallMs)} pass=${e.counts.pass??'-'} fail=${e.counts.fail??'-'} skipped=${e.counts.skipped??'-'}\n`);};
  const begin=performance.now(),startMs=Date.now();let rc=1,clean=false,current=null;
  const aborter=new AbortController();const stop=()=>aborter.abort();for(const signal of ['SIGINT','SIGTERM','SIGHUP'])process.on(signal,stop);
  try{
    mkdirSync(o.out,{mode:0o700});mkdirSync(join(o.out,'bin'),{mode:0o700});
    writeFileSync(join(o.out,'docker.json'),JSON.stringify({...run,docker}),{mode:0o600,flag:'wx'});
    const guard=join(BUILDER,'scripts/testing/lane-docker.mjs');
    // Launch the guard directly so it validates every real Docker invocation.
    writeFileSync(join(o.out,'bin','docker'),`#!/bin/sh\nexec '${process.execPath.replaceAll("'","'\\''")}' '${guard.replaceAll("'","'\\''")}' "$@"\n`,{mode:0o700,flag:'wx'});
    const auth=join(o.tempRoot,'pi-auth.json');writeFileSync(auth,'{}',{mode:0o600,flag:'wx'});chmodSync(auth,0o600);
    const steps=discoverSteps(o.checkout);record({kind:'plan',steps});
    const before=homeMetadata(process.env.HOME);record({kind:'home-before',entries:before});
    const env=cleanEnv(o,run,docker);
    const result=await executePinnedSteps(steps,{out:o.out,env,record,onChild:c=>current=c,signal:aborter.signal},{receipt,checkout:o.checkout,afterStep:()=>record({kind:'sockets',entries:socketInventory(o.tempRoot)})});
    rc=result.rc;
    const after=homeMetadata(process.env.HOME);record({kind:'home-after',entries:after});
    const prior=new Map(before.map(e=>[e.path,JSON.stringify(e)]));const changed=after.filter(e=>prior.get(e.path)!==JSON.stringify(e));
    if(attributeEntries(changed,{...run,nonce:id,fixturePaths:[]}).owned.length)refuse('lane-attributed real HOME write');
    if(git(o.checkout,'diff','--name-only'))refuse('tracked checkout changed during gate');
    const noLeaks=await cleanProcesses(run,record);cleanupProject(run,docker);clean=true;if(!noLeaks&&rc===0)rc=1;
  }catch(e){record({kind:'failure',message:e.message});process.stderr.write(`${e.message}\n`);rc=rc||1;
    try{await cleanProcesses(run,record);cleanupProject(run,docker);clean=true;}catch(error){record({kind:'cleanup-failed',message:error.message});}
  }finally{
    if(current){try{process.kill(-current.pid,'SIGKILL');}catch{}}
    for(const signal of ['SIGINT','SIGTERM','SIGHUP'])process.off(signal,stop);
    record({kind:'terminal',clean,rc,startMs,endMs:Date.now(),wallMs:performance.now()-begin,imageBuilds:0});
  }
  if(rc===0&&clean)process.stdout.write('GATE_PASSED\n');return rc;
}
async function main(argv){
  const locked=argv[0]==='--locked';if(locked)argv=argv.slice(1);
  if(argv[0]==='lock-dir'){
    const o=options(argv.slice(1));if(!o.grant)refuse('missing grant reference');if(!['a','b'].includes(o.lane))refuse('invalid lane');process.stdout.write(stateDirectory(o.stateRoot,true)+'\n');return 0;
  }
  const o=options(argv);o.stateRoot=stateDirectory(o.stateRoot);
  if(o.verb==='status'){process.stdout.write(JSON.stringify(status(o.stateRoot))+'\n');return 0;}
  if(!locked)refuse('use gate-lane.sh for locked operations');
  if(['recover','prepare-image'].includes(o.verb))o.exclusive=true;
  verifyLocks(o);
  if(o.verb==='recover'){process.stdout.write(JSON.stringify(await recoverRun(o))+'\n');return 0;}
  if(o.verb==='prepare-image'){
    for(const key of ['checkout','out','candidate','grant'])if(!o[key])refuse(`missing ${key}`);
    const checked=validateRun({...o,tempRoot:o.tempRoot??mkdtempSync('/mnt/storage/scratch/ga-')});activeConflicts(checked);
    const id=nonce(),dir=join(checked.stateRoot,'runs',id),docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim();
    appendRecord(dir,{kind:'start',mode:'prepare-image',lane:checked.lane,exclusive:true,owner:identity(),grant:checked.grant,candidate:checked.candidate,checkout:checked.checkout,tempRoot:checked.tempRoot,out:checked.out,project:`mg197-${checked.lane}-${id}`,docker});
    mkdirSync(checked.out,{mode:0o700});
    try{
      const receipt=prepareImage(checked.checkout,checked);
      writeFileSync(join(checked.out,'image-receipt.json'),JSON.stringify(receipt,null,2)+'\n',{mode:0o600,flag:'wx'});
      appendRecord(dir,{kind:'terminal',clean:true,rc:0,imageId:receipt.imageId,builds:1,wallMs:receipt.buildWallMs});
      process.stdout.write(JSON.stringify({imageReceipt:join(checked.out,'image-receipt.json'),imageId:receipt.imageId,buildWallMs:receipt.buildWallMs})+'\n');return 0;
    }catch(e){appendRecord(dir,{kind:'terminal',clean:false,rc:1,message:e.message});throw e;}
  }
  return runGate(o);
}
if(process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href)main(process.argv.slice(2)).then(rc=>process.exitCode=rc).catch(e=>{process.stderr.write(`${e.message}\n`);process.exitCode=2;});