From b8461266cd97e5231fb1aa816bc0f9af48138a70 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Wed, 21 Oct 2026 03:29:20 -0500 Subject: [PATCH] test: clean proof temp roots and catch preparation signals (#1695) --- agents/herb/work/queue-197/proof/BUILD.md | 126 ++++++++++++++++++ .../herb/work/queue-197/proof/run-proof.mjs | 53 ++++++-- 2 files changed, 165 insertions(+), 14 deletions(-) create mode 100644 agents/herb/work/queue-197/proof/BUILD.md diff --git a/agents/herb/work/queue-197/proof/BUILD.md b/agents/herb/work/queue-197/proof/BUILD.md new file mode 100644 index 000000000..5ecdc1e54 --- /dev/null +++ b/agents/herb/work/queue-197/proof/BUILD.md @@ -0,0 +1,126 @@ +# Two-lane proof command and schedule + +This evidence stays outside r5's 32 manifest paths. No proof or image build has +run. The driver and example config are committed on herb/row197-proof, rooted +at 9aa6d52d9. Sage runs the committed copy from their own checkout. + +Driver: `agents/herb/work/queue-197/proof/run-proof.mjs`. +The commit message to Sage carries its exact SHA-256. Check it with +`git show :agents/herb/work/queue-197/proof/run-proof.mjs | sha256sum`. + +After Sage fills config.json from config.example.json and grants the whole block: + +```sh +MOSAIC_PROOF_GRANT=ACTUAL_SAGE_GRANT_ID node /SAGE_PROOF_CHECKOUT/agents/herb/work/queue-197/proof/run-proof.mjs /SAGE_OWNED_CONFIG/config.json +``` + +The example config pins the launcher and candidate A to 197 r5: +9aa6d52d9431efd215aa31d748d7e273eb321474. The launcher is a separate clean +checkout owned by Sage, so later row 215 work cannot change it. Fill candidateB with +Sage's approved combined 212+197 commit, imageReceipt with the granted pinned +image-provenance receipt, and runRoot with a fresh private scratch path. Both +candidates must carry identical lane tooling and image inputs and use the same +pinned image ID. The driver refuses missing parameters, equal candidate SHAs, +dirty launcher, changed tooling, mismatched image inputs or image ID. + +D5 order is exactly two solo gates, then three pairs: + +| Wave | Candidate A | Candidate B | +| --- | --- | --- | +| 1 | solo, lane a | idle | +| 2 | idle | solo, lane b | +| 3 | paired, lane a | paired, lane b | +| 4 | paired, lane b | paired, lane a | +| 5 | paired, lane a | paired, lane b | + +Every invocation gets its own clean clone, output, short temp root and random +Compose project. The wrapper discovers all package and shell steps, with two +Node test-file workers per lane. The summary requires three overlaps, the lane +swap, successful clean receipts and zero warm image builds. Raw logs and per-step +receipts remain available. No failed gate or series is retried automatically. + +A live monitor stops both gates on the first failure line, records lane, step, +source file when the runner emits it, test name and preserved log, then enriches +that report after shutdown. If the runner gives no source path, the report says +unresolved and points to the log rather than guessing. It stops earlier than the +two-failure limit. TERM/INT/HUP stop owned supervisors; cleanup waits for them. + +Container and network names come from Compose's random recorded project. The +project and service configuration have no fixed global names. Exact-label Docker +event streams record create/destroy IDs and names, including --rm containers. +An exit trap inventories names/labels, removes only exact-project objects with +the approved cleanup helper, verifies emptiness and stops its own event streams. +The driver's Docker inspection never reads Config.Env or credential contents, and never builds, pulls, +retags or prunes. Missing resource-name evidence, an ended event stream, unknown +ownership, leftover objects or quarantined lanes fail the proof. A SIGKILL cannot +run an exit trap; durable wrapper receipts then require granted recovery. + +Run this as its own whole-slot phase. A still-running legacy landing.mjs gate or +gate.sh is LEGACY BUSY, including a parent process. The driver does not suppress +that refusal. The ordinary landing gate must exit before this proof phase starts. + +Planning runtime from sage-1837 is 12-15 minutes per wave, so reserve 60-75 minutes +for all five waves, plus any separately granted image preparation. This is an +estimate, not a measurement. Measured proof runtime is unavailable until this +first run. Historical row 189 full gate was 659 seconds; the r4 focused 19-file +run was 761.33 seconds and is not a full solo baseline. The resulting summary +records actual per-gate times, pair makespans, slowdowns and total wall time. + +No image provenance receipt or combined integration SHA is available to Herb yet. +The driver remains unexecuted. Sage will read the exact hashed driver and request +Webby's static review before the landing/proof grant. + +Static node --check passed after sage-1839 single cleared. This is syntax-only; +the driver has not run a proof or Docker command. Its runtime behavior awaits +Sage and Webby review and the separately granted proof block. + +## Driver review fixes, sage-1857 + +Webby's first review was CHANGES, B1, at bba870ee9. The revised driver takes +tempRoot only from immutable start records for the plan's checkouts. After +every owned gate exits, Docker cleanup and the final status/process checks +run. The driver requires the exact short ga-/gb- scratch spelling, a matching +lane, an owned real directory and no live owner, child group or process with +the run paths in its argv before removal. It also requires a clean terminal +receipt. A successful gate's root is removed recursively. A failed gate's +root stays for diagnosis; an unverified cleanup also stays and fails the +proof. docker-summary.json records removed, kept and error entries. Any +cleanup error makes the driver fail. No unrelated scratch directory is read. + +Signal handlers are installed before configuration and preparation. Clone +preparation is inside the try/finally, and yields between synchronous clones +so a pending signal stops it before another invocation. A preparation signal +writes failure evidence and reaches the cleanup path. Partial clones remain +under runRoot as permitted by Sage; they are never confused with lane roots. +SIGKILL cannot be trapped and still needs granted recovery. + +At 2026-10-21T08:26:41Z, Herb removed four earlier roots recorded in their +own packet: ga-VPXhG7, gb-XeTAUS, ga-35Zo9Y and ga-R5IOT3. Each was an owned +real directory with no process argv/cwd reference before removal. Names +ga-AMtfI3 and ga-Vd9Ef2 were visible in the scratch directory listing but had +no ownership evidence in Herb's packet; neither was opened or changed. + +## N4 read-only trace, pre-existing and out of scope + +Sage ruled in sage-1859 that N4 does not block 197. Legacy suites already +reach Compose's read-only host auth mount, and the lane adds no exposure. +Sage owns a DEFERRED entry for fixture PI_AUTH_FILE coverage. No fix is made +in 197, and this procedure does not claim that the full gate is secret-free. + +At 9aa6d52d9: + +- gate-lane.mjs:109-111 keeps HOME real but neither keeps nor sets + PI_AUTH_FILE. compose.yaml:43 therefore selects its host auth-file default. +- script-task is discovered by gate-lane.mjs:36. test-task.sh has no + PI_AUTH_FILE assignment. Calls at :149-150 (mock), :444, :460, :480, :483 + and :520 (live) reach run-task. mosaic-task.mjs:321 copies the environment + and :395-400 invokes compose run mosaic-agent without changing that path. +- script-release calls release activate at test-release.sh:69,85,93. + release.sh:41-54 health_check calls run-task with no PI_AUTH_FILE override, + then reaches the same Compose call. These tests use scratch MOSAIC_CONFIG; + that does not override the separate auth-file bind mount. +- test-auth.sh:48-92 sets explicit fixtures, but calls auth.sh instead of + Compose. Package runs/task-cli.test.mjs exercises list/show, not run. + +This was a static source trace only. No auth file was opened, no credential +value was read, and no Docker command, image build or proof was executed. diff --git a/agents/herb/work/queue-197/proof/run-proof.mjs b/agents/herb/work/queue-197/proof/run-proof.mjs index d668f9ce1..9aa5191b9 100644 --- a/agents/herb/work/queue-197/proof/run-proof.mjs +++ b/agents/herb/work/queue-197/proof/run-proof.mjs @@ -1,9 +1,12 @@ // Evidence driver only. Runs no test without Sage's whole-slot grant. -import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, realpathSync, openSync, closeSync, symlinkSync, readdirSync, statSync, readSync } from 'node:fs'; +import { readFileSync, writeFileSync, appendFileSync, mkdirSync, existsSync, realpathSync, openSync, closeSync, symlinkSync, readdirSync, statSync, lstatSync, readSync, rmSync } from 'node:fs'; import { resolve, join, dirname, relative, isAbsolute } from 'node:path'; import { pathToFileURL } from 'node:url'; import { execFileSync, spawn } from 'node:child_process'; const proofStart=performance.now(); +let interrupted=false,abortOnSignal=()=>{}; +const signal=()=>{interrupted=true;abortOnSignal();}; +for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.on(name,signal); const fail=m=>{throw Error(m);}; const within=(root,path)=>{const p=relative(root,path);return p===''||!p.startsWith('..')&&!isAbsolute(p);}; const grant=process.env.MOSAIC_PROOF_GRANT; @@ -20,7 +23,7 @@ const root=resolve(config.runRoot); if(existsSync(root)||!within('/mnt/storage/scratch',realpathSync(dirname(root))))fail('fresh scratch runRoot required'); const {validateTooling}=await import(pathToFileURL(join(launcher,'scripts/testing/gate-lane.mjs'))); const {verifyImage}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-image.mjs'))); -const {runRecords,status}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-state.mjs'))); +const {runRecords,status,processInventory,sameProcess}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-state.mjs'))); const {summarizePairs}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-evidence.mjs'))); const {cleanupProject}=await import(pathToFileURL(join(launcher,'scripts/testing/lane-docker.mjs'))); const docker=execFileSync('which',['docker'],{encoding:'utf8'}).trim(); @@ -36,15 +39,7 @@ const waves=[ ]; const plan=waves.map(w=>w.map(([label,lane,candidate])=>({label,lane,candidate,dir:join(root,label),checkout:join(root,label,'wt'),out:join(root,label,'out')}))); put('plan.json',{grant,...config,plan,image,started:new Date().toISOString()}); -// Prepare every clean candidate before the first test. Each invocation has its own clone. -for(const run of plan.flat()){ - mkdirSync(run.dir,{mode:0o700}); - execFileSync('git',['clone','--quiet','--no-hardlinks','--local',repo,run.checkout],{stdio:'pipe'}); - execFileSync('git',['-C',run.checkout,'checkout','--quiet','--detach',run.candidate],{stdio:'pipe'}); - symlinkSync('/mnt/storage/src/mosaic-stack/node_modules',join(run.checkout,'node_modules')); - validateTooling(run.checkout);verifyImage(image,run.checkout); -} -const active=new Set(),watchers=new Map(),objects=new Map();let interrupted=false,firstFailure=null; +const active=new Set(),watchers=new Map(),objects=new Map();let firstFailure=null; const eventError=[]; const knownRuns=new Map();let abortTimer; const ownedRuns=()=>runRecords(state).flatMap(r=>r.records.filter(e=>e.kind==='start'&&plan.flat().some(p=>p.checkout===e.checkout))); @@ -99,6 +94,7 @@ function abortActive(){ for(const job of active)job.child.kill('SIGTERM'); if(!abortTimer){abortTimer=setTimeout(()=>{for(const job of active)job.child.kill('SIGKILL');},30000);abortTimer.unref();} } +abortOnSignal=abortActive; function newFailure(job){ if(!existsSync(job.run.out))return null;job.logs??=new Map(); for(const name of readdirSync(job.run.out).filter(n=>n.endsWith('.log'))){ @@ -137,12 +133,21 @@ async function stop(){ const timer=setTimeout(()=>{for(const j of jobs)j.child.kill('SIGKILL');},30000);timer.unref(); await Promise.all(jobs.map(j=>j.done));clearTimeout(timer); } -const signal=()=>{interrupted=true;abortActive();}; -for(const name of ['SIGINT','SIGTERM','SIGHUP'])process.on(name,signal); const measured=[],start=proofStart; const monitorTimer=setInterval(monitor,100); let summary; try{ + // Preparation is inside the cleanup path. Yield between synchronous clones + // so a pending signal stops preparation before another invocation is made. + for(const run of plan.flat()){ + await new Promise(resolve=>setImmediate(resolve)); + if(interrupted)fail('proof interrupted during preparation'); + mkdirSync(run.dir,{mode:0o700}); + execFileSync('git',['clone','--quiet','--no-hardlinks','--local',repo,run.checkout],{stdio:'pipe'}); + execFileSync('git',['-C',run.checkout,'checkout','--quiet','--detach',run.candidate],{stdio:'pipe'}); + symlinkSync('/mnt/storage/src/mosaic-stack/node_modules',join(run.checkout,'node_modules')); + validateTooling(run.checkout);verifyImage(image,run.checkout); + } for(const wave of plan){ if(interrupted)fail('proof interrupted'); const observed=status(state); @@ -166,7 +171,27 @@ try{ for(const watcher of watchers.values()){watcher.stopping=true;watcher.child.kill('SIGTERM');} await Promise.all([...watchers.values()].map(w=>w.done)); let finalStatus;try{finalStatus=status(state);if(finalStatus.legacy.length||finalStatus.a.state!=='CLEAR'||finalStatus.b.state!=='CLEAR')cleanupErrors.push({message:'lane status not clear',status:finalStatus});}catch(error){cleanupErrors.push({message:error.message});} - put('docker-summary.json',{objects:[...objects.values()],eventError,cleanupErrors,finalStatus}); + // Start receipts establish which short roots belong to this proof. Never + // discover roots by scanning scratch or follow a substituted root symlink. + const tempRoots=[];let procs; + try{procs=processInventory();}catch(error){cleanupErrors.push({message:'process inventory failed: '+error.message});} + for(const run of knownRuns.values()){ + const record={gate:plan.flat().find(p=>p.checkout===run.checkout)?.label,lane:run.lane,tempRoot:run.tempRoot}; + try{ + if(!/^\/mnt\/storage\/scratch\/g[ab]-[A-Za-z0-9]{6}$/.test(run.tempRoot??'')||!run.tempRoot.startsWith('/mnt/storage/scratch/g'+run.lane+'-'))fail('invalid recorded temp root'); + const st=lstatSync(run.tempRoot); + if(!st.isDirectory()||st.isSymbolicLink()||st.uid!==process.getuid()||realpathSync(run.tempRoot)!==run.tempRoot)fail('temp root is not an owned real directory'); + const receipt=runRecords(state).find(r=>r.records.some(e=>e.kind==='start'&&e.checkout===run.checkout)); + const groups=new Set(receipt?.records.filter(e=>e.kind==='child').map(e=>e.group)); + if(!procs||sameProcess(run.owner)||procs.some(p=>p.state!=='Z'&&p.pid!==process.pid&&(groups.has(p.group)||p.argv.some(a=>a.includes(run.tempRoot)||a.includes(run.checkout)))))fail('run process still present or inventory unavailable; temp root retained'); + const terminal=receipt?.records.filter(e=>e.kind==='terminal').at(-1); + if(!terminal?.clean){record.action='kept';record.reason='run cleanup unverified';cleanupErrors.push({...record});tempRoots.push(record);continue;} + if(terminal.rc!==0||firstFailure?.gate===record.gate){record.action='kept';record.reason='failed gate diagnosis';} + else{rmSync(run.tempRoot,{recursive:true});record.action='removed';} + }catch(error){record.action='error';record.message=error.message;cleanupErrors.push({...record});} + tempRoots.push(record); + } + put('docker-summary.json',{objects:[...objects.values()],eventError,cleanupErrors,finalStatus,tempRoots}); if(cleanupErrors.length||eventError.length||[...objects.values()].some(o=>!o.name))process.exitCode=1; if(summary&&!process.exitCode){ for(const run of knownRuns.values())for(const kind of ['container','network'])if(![...objects.values()].some(o=>o.project===run.project&&o.kind===kind&&o.created)){process.exitCode=1;process.stderr.write('missing Docker creation evidence: '+run.project+' '+kind+'\n');}