import { createHash } from 'node:crypto'; import { realpathSync, existsSync, statSync, readdirSync } from 'node:fs'; import { join, resolve, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { execFileSync } from 'node:child_process'; import { shortRoot, checkHost, hostFacts, refuse, within, SCRATCH, identity } from './lane-state.mjs'; const BUILDER=resolve(dirname(fileURLToPath(import.meta.url)),'../..'); const git=(root,...args)=>execFileSync('git',['-C',root,...args],{encoding:'utf8',stdio:['ignore','pipe','pipe']}).trim(); export function validateRun(o){ if(!['a','b'].includes(o.lane))refuse('invalid lane'); if(typeof o.grant!=='string'||!o.grant.trim())refuse('missing grant reference'); for(const key of ['checkout','out','stateRoot'])if(typeof o[key]!=='string'||!o[key])refuse(`missing ${key}`); const tempRoot=shortRoot(o.tempRoot); const checkout=realpathSync(o.checkout); if(checkout===BUILDER||checkout==='/mnt/storage/src/mosaic-stack'||!within(SCRATCH,checkout))refuse('canonical or builder checkout'); if(!/^[a-f0-9]{40}$/.test(o.candidate)||git(checkout,'rev-parse','HEAD')!==o.candidate)refuse('wrong candidate SHA'); const changed=git(checkout,'status','--porcelain','--untracked-files=normal').split('\n').filter(x=>x&&x!=='?? node_modules'); if(changed.length)refuse('dirty checkout'); if(!existsSync(join(checkout,'node_modules'))||!statSync(join(checkout,'node_modules')).isDirectory())refuse('missing node_modules'); const out=resolve(o.out);if(existsSync(out))refuse('output already exists'); if(!within(dirname(checkout),out)||!within(SCRATCH,realpathSync(dirname(out))))refuse('output escapes run scratch'); const stateRoot=resolve(o.stateRoot);if(!within(SCRATCH,stateRoot))refuse('state outside scratch'); checkHost(o.lane,o.hostFacts??hostFacts()); return {...o,tempRoot,checkout,out,stateRoot}; } export function validateTooling(checkout){ for(const name of readdirSync(join(BUILDER,'scripts/testing')).filter(n=>/\.(mjs|sh)$/.test(n))){ const candidate=join(checkout,'scripts/testing',name),launcher=join(BUILDER,'scripts/testing',name); if(!existsSync(candidate)||!readFileSync(candidate).equals(readFileSync(launcher)))refuse('candidate tooling differs from launcher'); } } export function discoverSteps(checkout,concurrency=2){ if(concurrency!==2)refuse('test concurrency must be 2'); const steps=[]; for(const p of readdirSync(join(checkout,'packages')).sort())if(p!=='queue'&&existsSync(join(checkout,'packages',p,'tests'))&&statSync(join(checkout,'packages',p,'tests')).isDirectory())steps.push({name:`pkg-${p}`,cwd:join(checkout,'packages',p),argv:['node','--test','--test-concurrency=2',...readdirSync(join(checkout,'packages',p,'tests')).filter(n=>n.endsWith('.test.mjs')).sort().map(n=>`tests/${n}`)]}); for(const f of readdirSync(join(checkout,'scripts')).sort())if(/^test-.+\.sh$/.test(f)&&statSync(join(checkout,'scripts',f)).isFile())steps.push({name:`script-${f.slice(5,-3)}`,cwd:checkout,argv:['bash',`scripts/${f}`]}); return steps; } export async function executeSteps(steps,{out,env,record,onChild=()=>{},signal}){ const {openSync,closeSync}=await import('node:fs');const {spawn}=await import('node:child_process'); for(const step of steps){ if(signal?.aborted)return {rc:130}; const begin=performance.now(),startMs=Date.now();const fd=openSync(join(out,`${step.name}.log`),'wx',0o600); // Keep the supervisor private while matching the legacy test environment. const child=spawn('/bin/bash',['-c','umask 022; exec "$@"','gate-step',...step.argv],{cwd:step.cwd,env,detached:true,stdio:['ignore',fd,fd]});closeSync(fd); onChild(child);record({kind:'child',group:child.pid,owner:identity(child.pid)}); let escalation; const abort=()=>{try{process.kill(-child.pid,'SIGTERM');}catch{};escalation=setTimeout(()=>{try{process.kill(-child.pid,'SIGKILL');}catch{}},3000);escalation.unref();}; signal?.addEventListener('abort',abort,{once:true}); const rc=await new Promise(resolve=>{child.once('error',()=>resolve(127));child.once('close',(code,sig)=>resolve(signal?.aborted?130:(code??(sig?128:1))));}); clearTimeout(escalation);signal?.removeEventListener('abort',abort);onChild(null); const bytes=readFileSync(join(out,`${step.name}.log`)),text=bytes.toString('utf8'); const counts={};for(const name of ['pass','fail','skipped']){const m=text.match(new RegExp('^(?:ℹ |# )'+name+' (\\d+)','m'));if(m)counts[name]=Number(m[1]);} if(counts.pass===undefined){const summaries=[...text.matchAll(/(\d+) passed, (\d+) failed/g)];const last=summaries.at(-1);if(last){counts.pass=Number(last[1]);counts.fail=Number(last[2]);}} const requiredSkips=text.split('\n').filter(line=>/^skip\b/i.test(line)&&/(docker|daemon|sandbox|systemd|credential)/i.test(line)); record({kind:'step',name:step.name,startMs,endMs:Date.now(),wallMs:performance.now()-begin,rc,counts,requiredSkips,logSha256:createHash('sha256').update(bytes).digest('hex')}); if(rc!==0||requiredSkips.length)return {rc:rc||2}; } return {rc:0}; } export async function executePinnedSteps(steps,options,{receipt,checkout,inspect,afterStep=()=>{}}){ for(const step of steps){ verifyImage(receipt,checkout,inspect); const result=await executeSteps([step],options);afterStep(step); if(result.rc){verifyImage(receipt,checkout,inspect);return {...result,imageBuilds:0};} } verifyImage(receipt,checkout,inspect);return {rc:0,imageBuilds:0}; } import { mkdirSync, lstatSync, fstatSync, mkdtempSync, writeFileSync, readFileSync, openSync, readlinkSync, rmdirSync, chmodSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; import { spawnSync } from 'node:child_process'; import { status, appendRecord, runRecords, processInventory, sameProcess, nonce } from './lane-state.mjs'; import { verifyImage, imageFingerprint, inspectImage, prepareImage } from './lane-image.mjs'; import { cleanupProject } from './lane-docker.mjs'; import { homeMetadata, attributeEntries, socketInventory } from './lane-evidence.mjs'; function options(argv){ const result={verb:'run',stateRoot:'/mnt/storage/scratch/mosaic-gate-lanes',lane:'a',exclusive:false}; const names={'--state-root':'stateRoot','--lane':'lane','--candidate':'candidate','--checkout':'checkout','--out':'out','--temp-root':'tempRoot','--grant':'grant','--image-receipt':'imageReceipt','--run-id':'runId'}; const args=[...argv];if(args[0]&&!args[0].startsWith('--'))result.verb=args.shift(); if(!['run','status','recover','prepare-image','lock-dir'].includes(result.verb))refuse('unknown verb'); while(args.length){const key=args.shift();if(key==='--exclusive'){result.exclusive=true;continue;}if(!names[key]||!args.length)refuse('invalid or missing option');result[names[key]]=args.shift();} return result; } function stateDirectory(path,create=false){ const root=resolve(path);if(!within(SCRATCH,root)||root===SCRATCH)refuse('state must be private scratch'); if(create){const parent=realpathSync(dirname(root));if(!within(SCRATCH,parent))refuse('state parent escapes scratch');mkdirSync(root,{recursive:true,mode:0o700});} if(existsSync(root)){const st=lstatSync(root);if(st.isSymbolicLink()||!st.isDirectory()||st.uid!==process.getuid()||(st.mode&0o077))refuse('state is not a private owned directory'); for(const n of ['a.lock','b.lock','image.lock'])if(existsSync(join(root,n))&&!lstatSync(join(root,n)).isFile())refuse('lock is not an ordinary file'); } return root; } export function verifyLocks(o){ for(const [fd,name] of o.exclusive?[[3,'a.lock'],[4,'b.lock'],[5,'image.lock']]:[[3,`${o.lane}.lock`],[5,'image.lock']]){ let opened,path;try{opened=fstatSync(fd);path=lstatSync(join(o.stateRoot,name));}catch{refuse('lock descriptors missing');} if(!opened.isFile()||!path.isFile()||opened.ino!==path.ino||opened.dev!==path.dev||path.uid!==process.getuid())refuse('lock file replaced'); // A separate open must fail while our shared/exclusive lock is held. const r=spawnSync('flock',['-n','-x',join(o.stateRoot,name),'true']);if(r.status!==1)refuse('lock descriptor is not locked'); } } export function activeConflicts(o,observations={}){ const current=status(o.stateRoot,{...observations,ignoreLocks:o.exclusive?['a','b','image']:[o.lane,'image']}); if(current.legacy.length)refuse('LEGACY BUSY: unmanaged gate, test or Docker process'); for(const lane of o.exclusive?['a','b']:[o.lane])if(current[lane].state!=='CLEAR')refuse(`${lane} ${current[lane].state}`); for(const r of runRecords(o.stateRoot)){const start=r.records.find(e=>e.kind==='start');if(!r.records.some(e=>['terminal','recovered'].includes(e.kind)&&e.clean)&&start?.checkout===o.checkout)refuse('checkout already owned');} } export function cleanEnv(o,run,docker){ const kept=['PATH','HOME','USER','LOGNAME','SHELL','LANG','LC_ALL','LC_CTYPE','TZ','TERM','XDG_RUNTIME_DIR','DBUS_SESSION_BUS_ADDRESS','DOCKER_HOST','CHROMIUM']; const env=Object.fromEntries(kept.filter(k=>typeof process.env[k]==='string').map(k=>[k,process.env[k]])); Object.assign(env,{TMPDIR:o.tempRoot,TMP:o.tempRoot,TEMP:o.tempRoot,MOSAIC_GATE_LANE:o.lane,MOSAIC_GATE_STATE:o.stateRoot,MOSAIC_TEST_CONCURRENCY:'2',COMPOSE_PROJECT_NAME:run.project,JITI_FS_CACHE:join(o.tempRoot,'jiti'),XDG_CACHE_HOME:join(o.tempRoot,'cache'),MOSAIC_CONFIG:join(o.tempRoot,'missing-config.json'),MOSAIC_GATE_DOCKER_RECORD:join(o.out,'docker.json'), // A step reaching Compose mounts this fixture, never the host's pi auth file (row 244). PI_AUTH_FILE:join(o.tempRoot,'pi-auth.json')}); env.PATH=`${join(o.out,'bin')}:${env.PATH??'/usr/bin:/bin'}`; return env; } function fixtureProcesses(run){ const all=processInventory(),ancestors=new Set([process.pid]);let parent=process.pid; while(parent){const p=all.find(p=>p.pid===parent);if(!p||ancestors.has(p.ppid))break;ancestors.add(p.ppid);parent=p.ppid;} return all.filter(p=>{ if(ancestors.has(p.pid)||p.state==='Z')return false; let cwd;try{cwd=readlinkSync(`/proc/${p.pid}/cwd`);}catch{} return p.argv.some(a=>typeof a==='string'&&(a===run.tempRoot||a.startsWith(run.tempRoot+'/')))||cwd&&(within(run.tempRoot,cwd)||run.checkout&&within(run.checkout,cwd)); }); } async function cleanProcesses(run,record){ const before=fixtureProcesses(run); if(run.stateRoot&&run.id){ const procs=processInventory(),known=new Set(before.map(p=>p.pid)); const {records}=await import('./lane-state.mjs'); for(const child of records(join(run.stateRoot,'runs',run.id)).filter(e=>e.kind==='child')){ const members=procs.filter(p=>p.group===child.group&&p.state!=='Z'); if(members.length&&sameProcess(child.owner,procs.find(p=>p.pid===child.owner?.pid))){ try{process.kill(-child.group,'SIGKILL');}catch(e){if(e.code!=='ESRCH')throw e;} }else if(members.some(p=>!known.has(p.pid)))refuse('unproven orphan process-group ownership'); } } record({kind:'leftover-processes',objects:before.map(({pid,start,boot})=>({pid,start,boot}))}); for(const p of before){if(!sameProcess(p))continue;let raw;try{raw=readFileSync(`/proc/${p.pid}/cgroup`,'utf8');}catch(e){if(e.code==='ENOENT')continue;throw e;}const scope=raw.trim().split('/').find(x=>/^mosaic-chat-[a-z0-9]+\.scope$/.test(x)); if(scope){const shown=execFileSync('systemctl',['--user','show',scope,'-p','InvocationID','--value'],{encoding:'utf8'}).trim();if(!/^[a-f0-9]{32}$/.test(shown))refuse('scope identity unknown');record({kind:'scope-cleanup',unit:scope,invocation:shown});execFileSync('systemctl',['--user','kill','--signal=SIGKILL',scope],{stdio:'pipe'});} if(sameProcess(p))try{process.kill(p.pid,'SIGKILL');}catch(e){if(e.code!=='ESRCH')throw e;} } for(let n=0;n<30;n++){if(!fixtureProcesses(run).some(p=>p.state!=='Z'))return before.length===0;await new Promise(r=>setTimeout(r,100));} refuse('owned fixture process remains'); } async function recoverRun(o){ if(!o.grant)refuse('missing recovery grant'); if(!/^[a-f0-9]{12}$/.test(o.runId??''))refuse('invalid run ID'); const r=runRecords(o.stateRoot).find(r=>r.id===o.runId);if(!r)refuse('run not found'); const start=r.records.find(x=>x.kind==='start');if(start?.mode==='prepare-image'&&status(o.stateRoot).legacy.length)refuse('image builder still active or unknown');if(!start||sameProcess(start.owner))refuse('run owner live or unknown'); if(r.records.some(x=>['terminal','recovered'].includes(x.kind)&&x.clean))refuse('run already clean'); shortRoot(start.tempRoot);const record=e=>appendRecord(r.dir,e); const procs=processInventory(),known=new Set(fixtureProcesses(start).map(p=>p.pid)); for(const event of r.records.filter(e=>e.kind==='child')){ const members=procs.filter(p=>p.group===event.group&&p.state!=='Z'); if(members.length&&!sameProcess(event.owner,procs.find(p=>p.pid===event.owner?.pid))&&members.some(p=>!known.has(p.pid)))refuse('orphan process-group ownership unknown'); if(members.length&&sameProcess(event.owner)){process.kill(-event.group,'SIGKILL');} } await cleanProcesses(start,record);cleanupProject(start,start.docker); appendRecord(r.dir,{kind:'recovered',clean:true,grant:o.grant,owner:identity()});return {recovered:r.id}; } async function runGate(input){ for(const key of ['checkout','out','candidate','grant','imageReceipt'])if(!input[key])refuse(`missing ${key}`); const automatic=!input.tempRoot,tempRoot=input.tempRoot??mkdtempSync(`/mnt/storage/scratch/g${input.lane}-`); let o,receipt,docker; try{ o=validateRun({...input,tempRoot}); validateTooling(o.checkout); activeConflicts(o); receipt=JSON.parse(readFileSync(o.imageReceipt,'utf8'));verifyImage(receipt,o.checkout); docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim(); }catch(e){if(automatic)rmdirSync(tempRoot);throw e;} const id=nonce(),run={kind:'start',id,stateRoot:o.stateRoot,lane:o.lane,exclusive:o.exclusive,owner:identity(),grant:o.grant,candidate:o.candidate,checkout:o.checkout,tempRoot:o.tempRoot,out:o.out,project:`mg197-${o.lane}-${id}`,tag:receipt.tag,imageId:receipt.imageId,docker}; const dir=join(o.stateRoot,'runs',id);appendRecord(dir,run);const record=e=>{appendRecord(dir,e);if(e.kind==='step')process.stdout.write(`${e.name} rc=${e.rc} wall_ms=${Math.round(e.wallMs)} pass=${e.counts.pass??'-'} fail=${e.counts.fail??'-'} skipped=${e.counts.skipped??'-'}\n`);}; const begin=performance.now(),startMs=Date.now();let rc=1,clean=false,current=null; const aborter=new AbortController();const stop=()=>aborter.abort();for(const signal of ['SIGINT','SIGTERM','SIGHUP'])process.on(signal,stop); try{ mkdirSync(o.out,{mode:0o700});mkdirSync(join(o.out,'bin'),{mode:0o700}); writeFileSync(join(o.out,'docker.json'),JSON.stringify({...run,docker}),{mode:0o600,flag:'wx'}); const guard=join(BUILDER,'scripts/testing/lane-docker.mjs'); // Launch the guard directly so it validates every real Docker invocation. writeFileSync(join(o.out,'bin','docker'),`#!/bin/sh\nexec '${process.execPath.replaceAll("'","'\\''")}' '${guard.replaceAll("'","'\\''")}' "$@"\n`,{mode:0o700,flag:'wx'}); const auth=join(o.tempRoot,'pi-auth.json');writeFileSync(auth,'{}',{mode:0o600,flag:'wx'});chmodSync(auth,0o600); const steps=discoverSteps(o.checkout);record({kind:'plan',steps}); const before=homeMetadata(process.env.HOME);record({kind:'home-before',entries:before}); const env=cleanEnv(o,run,docker); const result=await executePinnedSteps(steps,{out:o.out,env,record,onChild:c=>current=c,signal:aborter.signal},{receipt,checkout:o.checkout,afterStep:()=>record({kind:'sockets',entries:socketInventory(o.tempRoot)})}); rc=result.rc; const after=homeMetadata(process.env.HOME);record({kind:'home-after',entries:after}); const prior=new Map(before.map(e=>[e.path,JSON.stringify(e)]));const changed=after.filter(e=>prior.get(e.path)!==JSON.stringify(e)); if(attributeEntries(changed,{...run,nonce:id,fixturePaths:[]}).owned.length)refuse('lane-attributed real HOME write'); if(git(o.checkout,'diff','--name-only'))refuse('tracked checkout changed during gate'); const noLeaks=await cleanProcesses(run,record);cleanupProject(run,docker);clean=true;if(!noLeaks&&rc===0)rc=1; }catch(e){record({kind:'failure',message:e.message});process.stderr.write(`${e.message}\n`);rc=rc||1; try{await cleanProcesses(run,record);cleanupProject(run,docker);clean=true;}catch(error){record({kind:'cleanup-failed',message:error.message});} }finally{ if(current){try{process.kill(-current.pid,'SIGKILL');}catch{}} for(const signal of ['SIGINT','SIGTERM','SIGHUP'])process.off(signal,stop); record({kind:'terminal',clean,rc,startMs,endMs:Date.now(),wallMs:performance.now()-begin,imageBuilds:0}); } if(rc===0&&clean)process.stdout.write('GATE_PASSED\n');return rc; } async function main(argv){ const locked=argv[0]==='--locked';if(locked)argv=argv.slice(1); if(argv[0]==='lock-dir'){ const o=options(argv.slice(1));if(!o.grant)refuse('missing grant reference');if(!['a','b'].includes(o.lane))refuse('invalid lane');process.stdout.write(stateDirectory(o.stateRoot,true)+'\n');return 0; } const o=options(argv);o.stateRoot=stateDirectory(o.stateRoot); if(o.verb==='status'){process.stdout.write(JSON.stringify(status(o.stateRoot))+'\n');return 0;} if(!locked)refuse('use gate-lane.sh for locked operations'); if(['recover','prepare-image'].includes(o.verb))o.exclusive=true; verifyLocks(o); if(o.verb==='recover'){process.stdout.write(JSON.stringify(await recoverRun(o))+'\n');return 0;} if(o.verb==='prepare-image'){ for(const key of ['checkout','out','candidate','grant'])if(!o[key])refuse(`missing ${key}`); const checked=validateRun({...o,tempRoot:o.tempRoot??mkdtempSync('/mnt/storage/scratch/ga-')});activeConflicts(checked); const id=nonce(),dir=join(checked.stateRoot,'runs',id),docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim(); appendRecord(dir,{kind:'start',mode:'prepare-image',lane:checked.lane,exclusive:true,owner:identity(),grant:checked.grant,candidate:checked.candidate,checkout:checked.checkout,tempRoot:checked.tempRoot,out:checked.out,project:`mg197-${checked.lane}-${id}`,docker}); mkdirSync(checked.out,{mode:0o700}); try{ const receipt=prepareImage(checked.checkout,checked); writeFileSync(join(checked.out,'image-receipt.json'),JSON.stringify(receipt,null,2)+'\n',{mode:0o600,flag:'wx'}); appendRecord(dir,{kind:'terminal',clean:true,rc:0,imageId:receipt.imageId,builds:1,wallMs:receipt.buildWallMs}); process.stdout.write(JSON.stringify({imageReceipt:join(checked.out,'image-receipt.json'),imageId:receipt.imageId,buildWallMs:receipt.buildWallMs})+'\n');return 0; }catch(e){appendRecord(dir,{kind:'terminal',clean:false,rc:1,message:e.message});throw e;} } return runGate(o); } if(process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href)main(process.argv.slice(2)).then(rc=>process.exitCode=rc).catch(e=>{process.stderr.write(`${e.message}\n`);process.exitCode=2;});