// scripts/stack-release.sh (D1, #1659). Design, the T table and the mutants: // agents/darkwing/work/queue-162/BUILD.md. Every run uses a temp data root, temp units and stand-in // systemctl, journalctl and systemd-analyze on PATH; nothing here reads ~/.mosaic-dev or a live unit. import test, {after} from 'node:test'; import assert from 'node:assert/strict'; import {execFileSync,spawn,spawnSync} from 'node:child_process'; import {mkdtempSync,mkdirSync,writeFileSync,readFileSync,appendFileSync,cpSync,rmSync,chmodSync,existsSync,statSync,lstatSync,symlinkSync,linkSync,readdirSync,realpathSync} from 'node:fs'; import {createServer} from 'node:http'; import {createHash} from 'node:crypto'; import {DatabaseSync} from 'node:sqlite'; import {tmpdir} from 'node:os'; import {join,dirname,basename} from 'node:path'; import {fileURLToPath,pathToFileURL} from 'node:url'; import {once} from 'node:events'; const ROOT=fileURLToPath(new URL('../../..',import.meta.url)).replace(/\/$/,''); const NODE=process.execPath; const MARKER='# mosaic-cli wrapper (Q81, agents/sage/work/q81/mosaic-cli.sh)'; const BUS='mosaic-bus@acme.service',CON='mosaic-webui.service'; const git=(cwd,...args)=>execFileSync('git',['-c','user.name=fixture','-c','user.email=fixture@invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','-C',cwd,...args],{encoding:'utf8'}).trim(); const script=(fn)=>`#!${NODE}\n(${fn})();\n`; // --- Stand-ins. Each is serialized into its own file, so it uses only process and its own scope. --- // systemctl --user: unit state in $SD_STATE/units.json, every call in calls.log. A bus start runs // holder.cjs (the store held open in WAL mode, writer.lock, host.json) and, with --pm in the unit's // ExecStart, writes the `launched` journal line and the launches entry unless a pm- fault // names another case. Faults are files in $SD_STATE/faults named for the copy they hit. function systemctl(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process'),crypto=process.getBuiltinModule('node:crypto'); const sd=process.env.SD_STATE,data=process.env.SD_DATA,unitDir=path.join(process.env.XDG_CONFIG_HOME,'systemd','user'); const args=process.argv.slice(2).filter((a)=>a!=='--user'),[verb,...rest]=args; fs.appendFileSync(path.join(sd,'calls.log'),args.join(' ')+'\n'); const faultFile=(n)=>path.join(sd,'faults',n),fault=(n)=>fs.existsSync(faultFile(n)); const stFile=path.join(sd,'units.json'); const load=()=>{try{return JSON.parse(fs.readFileSync(stFile,'utf8'))}catch{return {}}}; const save=(s)=>{fs.writeFileSync(stFile+'.tmp',JSON.stringify(s));fs.renameSync(stFile+'.tmp',stFile)}; const sleep=(ms)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms); const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}}; const isBus=(u)=>/^mosaic-bus@[a-z0-9-]+\.service$/.test(u),inst=(u)=>u.slice('mosaic-bus@'.length,-'.service'.length); const unitFile=(u)=>{ const main=isBus(u)?'mosaic-bus@.service':u,file=path.join(unitDir,main); if(!fs.existsSync(file))return null; const drops=[]; for(const d of isBus(u)?[main+'.d',u+'.d']:[u+'.d']){const dir=path.join(unitDir,d);if(fs.existsSync(dir))for(const f of fs.readdirSync(dir).sort())if(f.endsWith('.conf'))drops.push(path.join(dir,f))} const r={wd:'',exec:'',env:[],drops}; for(const f of [file,...drops])for(let line of fs.readFileSync(f,'utf8').split('\n')){ line=line.replace(/%i/g,isBus(u)?inst(u):'').replace(/%h/g,process.env.HOME); const m=/^(WorkingDirectory|ExecStart|Environment)=(.*)$/.exec(line);if(!m)continue; if(m[1]==='WorkingDirectory')r.wd=m[2];else if(m[1]==='ExecStart'){if(m[2])r.exec=m[2]}else r.env.push(m[2]); } return r; }; const snap=(ev)=>{ const dir=path.join(data,'bus'),files={}; try{for(const n of fs.readdirSync(dir).sort()){const p=path.join(dir,n),st=fs.statSync(p);if(st.isFile())files[n]=[st.size,st.mtimeMs,crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex')]}}catch{} fs.appendFileSync(path.join(sd,'snap.jsonl'),JSON.stringify({ev,files})+'\n'); }; const pm=(b,id,pid,off)=>{ let mode='';try{mode=fs.readFileSync(faultFile(`pm-${id}`),'utf8').trim()}catch{} const now=Date.now()-off*1000,epoch=Math.floor(now/1000),at=new Date(now).toISOString(),run=`run-${pid}`,unit=`mosaic-bus@${b}.service`; const line=(text)=>fs.appendFileSync(path.join(sd,'journal.jsonl'),JSON.stringify({epoch,unit,text})+'\n'); const ok=()=>line(`launched pm-${b}: run ${run}, claude opus`); const lf=path.join(sd,'launches.json');let list=[];try{list=JSON.parse(fs.readFileSync(lf,'utf8'))}catch{} const entry=(o)=>list.push({run,business:b,live:true,startedAt:at,...o}); line(`bus host up: business ${b}, socket /fixture/bus.sock, notifier off`); switch(mode){ case 'none':break; case 'two':ok();line(`launched pm-${b}: run ${run}b, claude opus`);entry({});break; case 'badword':line(`launched pm-${b}: job ${run}, claude opus`);entry({});break; case 'notlisted':ok();break; case 'early':ok();entry({startedAt:new Date(now-3600e3).toISOString()});break; case 'human':entry({run:`human-${pid}`});break; case 'other':ok();entry({business:'other'});break; case 'dead':ok();entry({live:false});break; default:ok();entry({}); } fs.writeFileSync(lf,JSON.stringify(list)); }; if(verb==='show'){ const u=rest[0],props=[];for(let i=1;i0)fs.writeFileSync(faultFile('bus-stops-after'),String(n-1));else s='inactive'} if(!q)console.log(s);process.exit(s==='active'?0:3); } if(verb==='is-enabled'){console.log('disabled');process.exit(1)} if(verb==='daemon-reload')process.exit(0); if(verb==='list-units'){ if(fault('list-fails'))process.exit(1); const pats=rest.filter((a)=>!a.startsWith('--')).map((p)=>new RegExp('^'+p.replace(/\./g,'\\.').replace(/\*/g,'.*')+'$')); for(const [u,s] of Object.entries(load()))if(pats.some((re)=>re.test(u)))console.log(`${u} loaded ${s.state} ${s.state==='active'?'running':'dead'} fixture`); process.exit(0); } if(verb==='start'){ const u=rest[0],f=unitFile(u);if(!f){console.error(`Unit ${u} not found.`);process.exit(5)} const s=load(),id=path.basename(f.wd); if(!isBus(u)){s[u]={state:'active',pid:null,wd:f.wd};save(s);process.exit(0)} if(fault(`start-dies-${id}`)){s[u]={state:'failed',pid:null,wd:f.wd};save(s);process.exit(0)} // Row 185 (T48): the copy's start writes one event before the holder opens the store, so the // store differs from a backup the run took before the start. The close leaves no -wal. if(fault(`start-writes-row-${id}`)){ const db=new (process.getBuiltinModule('node:sqlite').DatabaseSync)(path.join(data,'bus','bus.sqlite')); db.prepare('INSERT INTO events (at) VALUES (?)').run('2999-01-01T00:00:00.000Z');db.close(); } snap('start'); const off=process.env.SD_CLOCK_OFFSET||'0'; // The holder must not inherit stack-release.sh's lock (fd 9) or any other fd above 2. const close='for fd in /proc/self/fd/*; do n=${fd##*/}; [ "$n" -gt 2 ] && eval "exec $n>&-" 2>/dev/null; done; exec "$@"'; const ch=cp.spawn('bash',['-c',close,'holder',process.execPath,path.join(__dirname,'..','holder.cjs'),data,inst(u),sd,off],{detached:true,stdio:'ignore'}); ch.unref(); fs.appendFileSync(path.join(sd,'pids.log'),ch.pid+'\n'); const ready=path.join(sd,`ready-${ch.pid}`),end=Date.now()+10000; while(!fs.existsSync(ready)){if(Date.now()>end){console.error('fixture holder did not start');process.exit(1)}sleep(20)} s[u]={state:'active',pid:ch.pid,wd:f.wd};save(s); if(f.exec.split(' ').slice(1).includes('--pm'))pm(inst(u),id,ch.pid,Number(off)); process.exit(0); } if(verb==='stop'){ const u=rest[0],s=load(),st=s[u]; if(st&&st.pid){ try{process.kill(st.pid,'SIGTERM')}catch{} const end=Date.now()+10000;while(alive(st.pid)&&Date.now()JSON.parse(l))}catch{} for(const o of lines)if(o.unit===unit&&o.epoch>=from)console.log(o.text); } function systemdAnalyze(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'); if(process.argv[2]!=='--user'||process.argv[3]!=='unit-paths')process.exit(1); console.log(path.join(process.env.XDG_CONFIG_HOME,'systemd','user')); // The unit-path fault adds its text as one more path, as T28's relative path needs. const extra=path.join(process.env.SD_STATE,'faults','unit-path'); if(fs.existsSync(extra))console.log(fs.readFileSync(extra,'utf8')); } // The bus host: holds the store open in WAL mode until SIGTERM, as the broker does. function holder(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite'); const [data,business,sd,offset]=process.argv.slice(2),bus=path.join(data,'bus'); fs.mkdirSync(bus,{recursive:true,mode:0o700}); const db=new DatabaseSync(path.join(bus,'bus.sqlite')); if(!db.prepare("SELECT count(*) AS n FROM sqlite_master WHERE name='meta'").get().n){ db.exec('PRAGMA journal_mode=WAL'); db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)'); db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)'); db.prepare("INSERT INTO meta VALUES ('schema_version', '3g')").run(); } fs.writeFileSync(path.join(bus,'writer.lock'),String(process.pid)); const at=new Date(Date.now()-Number(offset||0)*1000).toISOString(),host=path.join(sd,'host.json'); fs.writeFileSync(host+'.tmp',JSON.stringify({host:{business,pid:process.pid,live:true,startedAt:at,notifier:'ok'},boot:{ok:true,business,at}})); fs.renameSync(host+'.tmp',host); process.on('SIGTERM',()=>{db.close();fs.rmSync(path.join(bus,'writer.lock'),{force:true});process.exit(0)}); fs.writeFileSync(path.join(sd,`ready-${process.pid}`),''); setInterval(()=>{},1<<30); } // A copy's scripts/mosaic: bus status and launches list from what the holder and systemctl wrote. function mosaicStub(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'); const id=path.basename(path.resolve(__dirname,'..')),sd=process.env.SD_STATE,a=process.argv.slice(2).join(' '); const fault=(n)=>fs.existsSync(path.join(sd,'faults',n)); const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}}; if(a==='bus status --json'){ if(fault(`status-${id}`))process.exit(1); let s={host:null,boot:null}; try{s=JSON.parse(fs.readFileSync(path.join(sd,'host.json'),'utf8'));s.host.live=alive(s.host.pid)}catch{} console.log(JSON.stringify(s)); }else if(a==='bus status')console.log(`bus: fixture status for ${id}`); else if(a==='launches list --json'){ if(fault(`launches-${id}`))process.exit(1); let l=[];try{l=JSON.parse(fs.readFileSync(path.join(sd,'launches.json'),'utf8'))}catch{} console.log(JSON.stringify(l)); }else{console.error(`fixture mosaic: ${a}`);process.exit(4)} } // A copy's migrate-schema.mjs: records the backups it saw, then sets 3h, or 3h-partial and exit 1. function migrateStub(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite'); const data=process.argv[2],sd=process.env.SD_STATE,dir=path.join(data,'backups','stack'); const backups=fs.existsSync(dir)?fs.readdirSync(dir).sort().map((name)=>({name,sums:fs.existsSync(path.join(dir,name,'SHA256SUMS'))})):[]; fs.writeFileSync(path.join(sd,'migrate.json'),JSON.stringify({backups})); const fail=fs.existsSync(path.join(sd,'faults','migrate-fails')); const db=new DatabaseSync(path.join(data,'bus','bus.sqlite')); db.prepare("UPDATE meta SET value=? WHERE key='schema_version'").run(fail?'3h-partial':'3h');db.close(); if(fail)process.exit(1); console.log('fixture migrate: 3h'); } // --- The module fixture: one repo, its copies and the stand-ins, shared by every test. --- const base=realpathSync(mkdtempSync(join(tmpdir(),'stack-release-'))); after(()=>{execFileSync('chmod',['-R','u+w',base]);rmSync(base,{recursive:true,force:true})}); const repo=join(base,'repo'),COPIES=join(base,'copies'),SBIN=join(base,'sbin'); mkdirSync(repo);mkdirSync(COPIES);mkdirSync(SBIN); writeFileSync(join(base,'package.json'),'{"type":"commonjs"}\n'); const tracked=execFileSync('git',['-C',ROOT,'ls-files','-co','--exclude-standard','--','scripts','packages','adapters','roles','RELEASE','package.json','package-lock.json',':!packages/*/tests',':!scripts/foundation'],{encoding:'utf8'}).trim().split('\n'); for(const f of tracked)if(existsSync(join(ROOT,f)))cpSync(join(ROOT,f),join(repo,f)); const pkg={name:'copy-fixture',version:'1.0.0'}; writeFileSync(join(repo,'package.json'),JSON.stringify(pkg)); writeFileSync(join(repo,'package-lock.json'),JSON.stringify({name:pkg.name,version:pkg.version,lockfileVersion:3,packages:{'':pkg}})); git(repo,'init','-q','-b','refactor');git(repo,'config','gc.auto','0');git(repo,'add','.');git(repo,'commit','-qm','fixture'); const commit=(msg)=>{git(repo,'commit','-q','--allow-empty','-m',msg);return git(repo,'rev-parse','HEAD')}; // refactor: F A B C1 C, then M merging side branch S (from A). U sits on a branch nothing reaches. const F=git(repo,'rev-parse','HEAD'),A=commit('row 901: A'),B=commit('row 902: B'),C1=commit('row 903: C1'),C=commit('row 904: C'); git(repo,'checkout','-q','-b','side',A);const S=commit('row 905: S'); git(repo,'checkout','-q','-b','unreached',A);const U=commit('row 906: U'); git(repo,'checkout','-q','refactor');git(repo,'merge','-q','--no-ff','-m','merge side','side'); const M=git(repo,'rev-parse','HEAD');git(repo,'update-ref','refs/remotes/origin/refactor',M); // `to` false leaves out the `to:` lines (C1); a string puts that value in each one (A). The --plan // steps line can't be read from either (T27). const storeText=(v,froms,to=true)=>`export const SCHEMA_VERSION = '${v}';\nexport const MIGRATIONS = [\n${froms.map((f,i)=>` {\n from: '${f}',\n${to===false?'':` to: '${to===true?froms[i+1]??v:to}',\n`} run() {},\n },\n`).join('')}];\n`; // Plug-in kinds (row 169): C1 and C know the harness kind, as a copy at or past c216f47db does. The // rest have it under another name, as ca15d73e9 has no harness kind. Every copy has the real business // loader and roles, which the tool imports to read the business file. const KINDS_NEW=readFileSync(join(repo,'packages','plugins','src','kinds.mjs'),'utf8'); const KINDS_OLD=KINDS_NEW.replace('\n harness: Object.freeze({','\n "harness-before-c216f47db": Object.freeze({'); assert.notEqual(KINDS_OLD,KINDS_NEW); for(const [id,store] of [[F,'// a copy from before SCHEMA_VERSION\n'],[A,storeText('3g',['3f'],'3x')],[B,storeText('3g',['3f'])],[C1,storeText('3g',['3f'],false)],[C,storeText('3h',['3f','3g'])],[S,storeText('3g',['3f'])],[U,storeText('3g',['3f'])]]){ const d=join(COPIES,id); for(const [f,text,mode] of [['scripts/mosaic',script(mosaicStub),0o755],['packages/webui/src/cli.mjs',[B,C1,C].includes(id)?"// fixture Console, never run\n// serve accepts '--tester-port'\n// serve accepts '--proof-header' (row 178)\n":'// fixture Console before tester listener, never run\n',0o644],['packages/bus/src/store.mjs',store,0o644],['packages/bus/src/migrate-schema.mjs',`(${migrateStub})();\n`,0o644],['packages/plugins/src/kinds.mjs',[C1,C].includes(id)?KINDS_NEW:KINDS_OLD,0o644]]){ mkdirSync(dirname(join(d,f)),{recursive:true});writeFileSync(join(d,f),text,{mode}); } for(const p of ['packages/business/src','roles'])cpSync(join(repo,p),join(d,p),{recursive:true}); execFileSync('chmod',['-R','a-w',d]); } for(const [name,fn] of [['systemctl',systemctl],['journalctl',journalctl],['systemd-analyze',systemdAnalyze]])writeFileSync(join(SBIN,name),script(fn),{mode:0o755}); writeFileSync(join(base,'holder.cjs'),`(${holder})();\n`); const SR=join(repo,'scripts','stack-release.sh'); // --- Per-test roots and helpers --- const sha=(f)=>createHash('sha256').update(readFileSync(f)).digest('hex'); const lines=(f)=>existsSync(f)?readFileSync(f,'utf8').split('\n').filter(Boolean):[]; const calls=(e)=>lines(join(e.sd,'calls.log')); const startsStops=(e,from)=>calls(e).slice(from).filter((c)=>/^(start|stop) /.test(c)); const logLines=(e)=>lines(e.log).map((l)=>JSON.parse(l)); const pointer=(e)=>existsSync(e.pointer)?JSON.parse(readFileSync(e.pointer,'utf8')):null; const units=(e)=>JSON.parse(readFileSync(join(e.sd,'units.json'),'utf8')); const snaps=(e)=>lines(join(e.sd,'snap.jsonl')).map((l)=>JSON.parse(l)); const fault=(e,name,text='')=>writeFileSync(join(e.sd,'faults',name),text); const sysctl=(e,...a)=>execFileSync(join(SBIN,'systemctl'),['--user',...a],{env:e.env,encoding:'utf8'}); const edit=(f,from,to)=>{const s=readFileSync(f,'utf8');assert.ok(s.includes(from),`${f} has no ${from}`);writeFileSync(f,s.replace(from,to))}; const posts=(e)=>readdirSync(e.data).filter((n)=>n.startsWith('bus.post-attempt-')); const backupSum=(e,act)=>readFileSync(join(e.data,'backups','stack',act,'SHA256SUMS'),'utf8').split(' ')[0]; const commitLines=(range)=>git(repo,'log','--oneline','--no-merges','--max-count=200',range).split('\n').map((l)=>` ${l}`); // The units, the wrapper, the pointer and the store, by sha256. const fp=(e)=>({bus:sha(join(e.units,'mosaic-bus@.service')),console:sha(join(e.units,CON)),wrapper:sha(join(e.bin,'mosaic')),pointer:existsSync(e.pointer)?sha(e.pointer):null,store:sha(e.store)}); // The schema version: readOnly while the holder has the store open, immutable when it is stopped. function storeVersion(file,held){ const db=held?new DatabaseSync(file,{readOnly:true}):new DatabaseSync(pathToFileURL(file).href+'?immutable=1',{readOnly:true}); try{return db.prepare("SELECT value FROM meta WHERE key='schema_version'").get().value}finally{db.close()} } // Every file and directory under p with size and mtime: T23 and T16 compare it before and after. function tree(p){ const out=[]; const walk=(q)=>{let st;try{st=statSync(q)}catch{return}out.push(st.isDirectory()?`${q}/`:`${q} ${st.size} ${st.mtimeMs}`);if(st.isDirectory())for(const n of readdirSync(q).sort())walk(join(q,n))}; walk(p);return out; } // T17: what a run may never remove. const kept=(e)=>[...tree(join(e.data,'backups','stack')),...tree(join(e.state,'stack-rollback')),...posts(e).flatMap((n)=>tree(join(e.data,n))),...tree(COPIES)].map((l)=>l.replace(/ .*/,'')); // The business file (row 169): the committed example made loadable, as acme, with `plugins` when given. const EXAMPLE=readFileSync(join(repo,'packages','business','examples','mosaic-stack.example.json'),'utf8'); function biz(e,plugins){ const b=JSON.parse(EXAMPLE);let n=0; const walk=(o)=>{for(const [k,v] of Object.entries(o)){if(k==='botId')o[k]=++n;else if(v==='YYYY-MM-DD')o[k]='2027-01-01';else if(v&&typeof v==='object')walk(v)}}; walk(b);b.id='acme';if(plugins!==undefined)b.plugins=plugins; const f=join(dirname(e.cfg),'businesses','acme.json');mkdirSync(dirname(f),{recursive:true}); writeFileSync(f,JSON.stringify(b,null,2)+'\n',{mode:0o600});chmodSync(f,0o600); } async function mk(t){ const root=realpathSync(mkdtempSync(join(tmpdir(),'sr-'))); const e={root,data:join(root,'data'),bin:join(root,'bin'),sd:join(root,'sd'),units:join(root,'xcfg','systemd','user'),cfg:join(root,'cfg','config.json'),claude:join(root,'claude-bin','claude')}; Object.assign(e,{state:join(e.data,'state'),pointer:join(e.data,'state','stack-active.json'),log:join(e.data,'state','stack-activation-log.jsonl'),store:join(e.data,'bus','bus.sqlite'),lock:join(e.data,'state','stack-release.lock')}); for(const d of [join(root,'home'),dirname(e.cfg),join(e.data,'bus'),e.bin,dirname(e.claude),join(e.sd,'faults'),e.units,join(root,'xdata')])mkdirSync(d,{recursive:true}); chmodSync(join(e.data,'bus'),0o700); writeFileSync(e.cfg,JSON.stringify({configVersion:1,environment:'development',dataRoot:e.data,execution:{backend:'docker',provider:'zai',model:'glm-5.3-flash',adapter:'mock'}})); biz(e); writeFileSync(join(e.bin,'mosaic'),`#!/usr/bin/env bash\n${MARKER}\n[ "\${1:-}" = __lease-capability ] && exec "${e.bin}/mosaic-old" "$@"\nexec "/old/scripts/mosaic" "$@"\n`,{mode:0o755}); writeFileSync(join(e.bin,'mosaic-old'),'#!/bin/sh\nexit 0\n',{mode:0o755}); writeFileSync(e.claude,'#!/bin/sh\necho "9.9.9 (Claude Code fixture)"\n',{mode:0o755}); e.env={PATH:`${SBIN}:${dirname(NODE)}:/usr/bin:/bin`,HOME:join(root,'home'),XDG_CONFIG_HOME:join(root,'xcfg'),XDG_DATA_HOME:join(root,'xdata'),MOSAIC_CONFIG:e.cfg,SD_STATE:e.sd,SD_DATA:e.data,TMPDIR:tmpdir()}; // The Console's /healthz: 200 while the stand-in has it active and no healthz- fault names its copy. const server=createServer((q,r)=>{ let st={};try{st=units(e)[CON]||{}}catch{} const ok=q.url==='/healthz'&&st.state==='active'&&!existsSync(join(e.sd,'faults',`healthz-${basename(st.wd||'')}`)); r.writeHead(ok?200:500);r.end(); }); server.listen(0,'127.0.0.1');await once(server,'listening');e.port=String(server.address().port); t.after(()=>{ for(const pid of lines(join(e.sd,'pids.log')).map(Number)){ try{if(readFileSync(`/proc/${pid}/cmdline`,'utf8').includes(join(base,'holder.cjs')))process.kill(pid,'SIGKILL')}catch{} } server.close();execFileSync('chmod',['-R','u+w',root]);rmSync(root,{recursive:true,force:true}); }); return e; } async function run(e,...args){ const before=existsSync(e.data)?kept(e):[]; const ch=spawn('bash',[e.sr??SR,...args,'--copies',COPIES,'--bin',e.bin],{env:e.env,stdio:['ignore','pipe','pipe']}); let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d}); const timer=setTimeout(()=>ch.kill('SIGKILL'),120000); const [code]=await once(ch,'close');clearTimeout(timer); const now=new Set(kept(e)); for(const p of before)assert.ok(now.has(p),`T17: ${args.join(' ')} removed ${p}`); return {code,out,err}; } // The stack on copy `copy` for business acme, both units active. pointer: adopt (a real adopt and // its log line), hand (written here, no log) or none. Snapshots start after the seed. async function seed(e,{copy=B,pm=false,store='3g',pointer:how='adopt'}={}){ const db=new DatabaseSync(e.store); db.exec('PRAGMA journal_mode=WAL'); db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)'); db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)'); db.prepare("INSERT INTO meta VALUES ('schema_version', ?)").run(store); for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`); db.close(); const sh=(name,...a)=>execFileSync('bash',[join(repo,'scripts',name),...a],{env:e.env,encoding:'utf8',stdio:['ignore','pipe','pipe']}); sh('bus-service.sh','install','--copy',copy,'--copies',COPIES,...(pm?['--pm']:[]),'--claude',e.claude); sh('webui-service.sh','install','--copy',copy,'--copies',COPIES,'--port',e.port,'--business','acme'); sh('stack-copy.sh','wrapper',copy,'--copies',COPIES,'--bin',e.bin); // The seeded host started two minutes ago, so its boot status and PM run predate any run's start. execFileSync(join(SBIN,'systemctl'),['--user','start',BUS],{env:{...e.env,SD_CLOCK_OFFSET:'120'}}); sysctl(e,'start',CON); if(how==='adopt'){const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err)} else if(how==='hand'){ mkdirSync(e.state,{recursive:true}); writeFileSync(e.pointer,JSON.stringify({copy,business:'acme',pm,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g',activation:'manual',at:'2026-10-18T00:00:00Z'})+'\n'); } writeFileSync(join(e.sd,'snap.jsonl'),''); } const rolledBack=(code,to=B)=>`stack-release: ${code}; rolled back to ${to}, which passed the gate\n`; const planBackup=(e)=>`backup: the run backs up the stopped store to ${e.data}/backups/stack// before any migration`; const planNoBackup='backup: none; the run rolls back before the backup'; const planTail=(e,to,pm=false)=>[`bus: scripts/bus-service.sh install --copy ${to} --copies ${COPIES}${pm?' --pm':''} --claude ${e.claude}`,'proof: off',`Console: scripts/webui-service.sh install --copy ${to} --copies ${COPIES} --port ${e.port} --business acme`,`CLI: scripts/stack-copy.sh wrapper ${to} --copies ${COPIES} --bin ${e.bin}`]; // --- The T table --- test('T1 adopt writes the pointer from the units once; a failing check refuses check-failed',async(t)=>{ const e=await mk(t);await seed(e,{pointer:'none'}); const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err); assert.equal(r.out,`adopted ${B} for acme (pm 0, schema 3g)\n`); const {at,activation,...p}=pointer(e); assert.deepEqual(p,{copy:B,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'}); assert.match(activation,new RegExp(`^\\d{8}T\\d{6}Z-${B.slice(0,12)}$`));assert.match(at,/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/); assert.deepEqual(logLines(e).map((l)=>[l.op,l.from,l.to,l.result]),[['adopt',null,B,'ok']]); const again=await run(e,'adopt','acme');assert.equal(again.code,3); assert.equal(again.err,`stack-release: refusing already-adopted: ${e.pointer} exists (copy ${B})\n`); assert.equal(logLines(e).at(-1).code,'already-adopted'); const f=await mk(t);await seed(f,{pointer:'none'});fault(f,`check-${B}`); const bad=await run(f,'adopt','acme');assert.equal(bad.code,3); assert.match(bad.err,/^stack-release: refusing check-failed: stack-copy.sh check did not pass:\nbus: /); assert.equal(pointer(f),null); }); test('T2 activate installs exactly the pointer\'s pm, claude and Console flags',async(t)=>{ const e=await mk(t);await seed(e); const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err+r.out); assert.ok(r.out.endsWith(`stack-release: activate ok: ${C1} is live for acme\n`),r.out); const bus=readFileSync(join(e.units,'mosaic-bus@.service'),'utf8').split('\n'),con=readFileSync(join(e.units,CON),'utf8').split('\n'); assert.ok(bus.includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i`),bus.join('\n')); assert.ok(bus.includes(`Environment=MOSAIC_CLAUDE_BIN=${e.claude}`)); const exec=con.find((l)=>l.startsWith('ExecStart=')); assert.ok(exec.endsWith(` ${COPIES}/${C1}/packages/webui/src/cli.mjs serve --checkout ${repo} --port ${e.port} --business acme`),exec); assert.equal(readFileSync(join(e.bin,'mosaic'),'utf8').trimEnd().split('\n').at(-1),`exec "${COPIES}/${C1}/scripts/mosaic" "$@"`); const {at,activation,...p}=pointer(e); assert.deepEqual(p,{copy:C1,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'}); }); test('T3 a hand-edited unit refuses unit-flags-drift, names the difference and changes nothing',async(t)=>{ const cases=[ ['pm',(e)=>edit(join(e.units,'mosaic-bus@.service'),' bus start %i\n',' bus start %i --pm\n'),()=>' pm: pointer 0, unit 1'], ['claude',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'claude.conf'),'[Service]\nEnvironment=MOSAIC_CLAUDE_BIN=/opt/other/claude\n')},(e)=>` claude: pointer ${e.claude}, unit /opt/other/claude`], ['console',(e)=>edit(join(e.units,CON),'--business acme\n','--business acme --board http://127.0.0.1:9\n'),(e)=>` console: pointer --port ${e.port} --business acme, unit --port ${e.port} --business acme --board http://127.0.0.1:9`], ]; for(const [name,change,line] of cases)await t.test(name,async(t)=>{ const e=await mk(t);await seed(e);change(e); const before=fp(e),n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',C1,'acme');assert.equal(r.code,3); assert.equal(r.err,`stack-release: refusing unit-flags-drift: the units differ from the pointer; settle it first:\n${line(e)}\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'unit-flags-drift'); }); }); test('T4 each item 3 refusal changes nothing and logs one line with its code',async(t)=>{ const setUnit=(e,u,state)=>{const s=units(e);s[u]={...(s[u]||{pid:null,wd:''}),state};writeFileSync(join(e.sd,'units.json'),JSON.stringify(s))}; const cases=[ ['copy-missing',()=>M,()=>`no copy ${M} in ${COPIES}`], ['copy-unreachable',()=>U,()=>`${U} is not reachable from refactor and origin/refactor`], ['copy-schema-unknown',()=>F,()=>`cannot read SCHEMA_VERSION and its chain from ${COPIES}/${F}/packages/bus/src/store.mjs`], ['already-active',()=>B,()=>`${B} is the pointer's copy`], ['instance-busy',(e)=>{setUnit(e,'mosaic-bus@other.service','active');return C1},()=>'mosaic-bus@other.service is active; the copy and wrapper switch reach it too','another business active'], ['instance-busy',(e)=>{setUnit(e,BUS,'activating');return C1},()=>`${BUS} is activating`,'this business activating'], ['drop-in-override',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'override.conf'),'[Service]\nWorkingDirectory=/elsewhere\n');return C1},(e)=>`${join(e.units,`${BUS}.d`,'override.conf')} overrides ExecStart or WorkingDirectory; the operator removes it as in TOOLS.md step 4`], ['no-pointer',()=>C1,(e)=>`no ${e.pointer}; run adopt first`,'',{pointer:'none'}], ]; for(const [code,setup,msg,label,opts] of cases)await t.test(label?`${code} (${label})`:code,async(t)=>{ const e=await mk(t);await seed(e,opts);const id=setup(e); const before=fp(e),n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',id,'acme');assert.equal(r.code,3,r.err); assert.equal(r.err,`stack-release: refusing ${code}: ${msg(e)}\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.equal(logLines(e).length,logs+1);assert.deepEqual([logLines(e).at(-1).result,logLines(e).at(-1).code],['refused',code]); }); }); test('T5 an activate with no migration stops the Console then the bus, installs, starts the bus then the Console',async(t)=>{ const e=await mk(t);await seed(e);const n=calls(e).length; const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err); assert.deepEqual(startsStops(e,n),[`stop ${CON}`,`stop ${BUS}`,`start ${BUS}`,`start ${CON}`]); const c=calls(e).slice(n),reload=c.indexOf('daemon-reload'); assert.ok(reload>c.indexOf(`stop ${BUS}`)&&reload[x.op,x.from,x.to,x.result]),[['activate',B,C1,'started'],['activate',B,C1,'ok']]); }); test('T6 a migration without --migrate rolls back with migration-needs-flag',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e); const r=await run(e,'activate',C,'acme','--timeout','3');assert.equal(r.code,1,r.out); assert.equal(r.err,rolledBack('migration-needs-flag')); assert.deepEqual(fp(e),before); assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd,units(e)[CON].state],['active',join(COPIES,B),'active']); const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','migration-needs-flag','ok']); assert.equal(storeVersion(e.store,true),'3g');assert.ok(!existsSync(join(e.data,'backups','stack'))); }); test('T7 --migrate takes the backup before the migration runs',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e); const r=await run(e,'activate',C,'acme','--migrate');assert.equal(r.code,0,r.err+r.out); const act=pointer(e).activation; assert.deepEqual(JSON.parse(readFileSync(join(e.sd,'migrate.json'),'utf8')),{backups:[{name:act,sums:true}]}); assert.equal(backupSum(e,act),before.store); assert.deepEqual([pointer(e).copy,pointer(e).schema],[C,'3h']);assert.equal(storeVersion(e.store,true),'3h'); assert.equal(logLines(e).at(-1).backup,act); }); test('T8 a failed migration restores the backup and keeps the store it replaced',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e);fault(e,'migrate-fails'); const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out); assert.equal(r.err,rolledBack('migrate-failed')); assert.equal(posts(e).length,1);assert.equal(storeVersion(join(e.data,posts(e)[0],'bus.sqlite'),false),'3h-partial'); const act=logLines(e).at(-1).activation; assert.equal(sha(e.store),backupSum(e,act));assert.equal(sha(e.store),before.store); assert.deepEqual(fp(e),before);assert.equal(units(e)[BUS].wd,join(COPIES,B)); }); test('T9 a store outside the target\'s chain rolls back with store-newer',async(t)=>{ const e=await mk(t);await seed(e,{store:'3z'});const before=fp(e); const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out); assert.equal(r.err,rolledBack('store-newer')); assert.deepEqual(fp(e),before);assert.equal(storeVersion(e.store,true),'3z');assert.ok(!existsSync(join(e.data,'backups','stack'))); }); test('T10 each gate check failing alone rolls back and puts every file back',async(t)=>{ const cases=[['units-inactive',`start-dies-${C1}`],['bus-status',`status-${C1}`],['healthz',`healthz-${C1}`],['copy-check',`check-${C1}`],['pm-launch-unconfirmed',`pm-${C1}`,'none']]; for(const [code,name,text] of cases)await t.test(code,async(t)=>{ const pm=code==='pm-launch-unconfirmed',e=await mk(t);await seed(e,{pm});fault(e,name,text); const before=fp(e); const r=await run(e,'activate',C1,'acme',...(pm?['--pm']:[]),'--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack(code));assert.ok(r.out.includes(`stack-release: gate failed on ${C1}: ${code}`),r.out); assert.deepEqual(fp(e),before); assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,B)]); const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed',code,'ok']); }); }); test('T10b the PM check needs exactly one launched line for a live run of this start',async(t)=>{ const cases=[ ['no launched line','none',/\(no launched line since the start\)/], ['two lines','two',/\(2 launched lines since the start\)/], ['a changed word','badword',/\(the launched line does not parse: launched pm-acme: job run-\d+, claude opus\)/], ['run not in launches list','notlisted',/\(run run-\d+ is not in launches list\)/], ['run started before the start','early',/\(run run-\d+ started before this start\)/], ['a human launch with no journal line','human',/\(no launched line since the start\)/], ['run for another business','other',/\(run run-\d+ is for other\)/], ['run not live','dead',/\(run run-\d+ is not live\)/], ]; for(const [name,mode,why] of cases)await t.test(name,async(t)=>{ const e=await mk(t);await seed(e,{pm:true});fault(e,`pm-${C1}`,mode); const r=await run(e,'activate',C1,'acme','--pm','--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('pm-launch-unconfirmed')); const gate=r.out.split('\n').find((l)=>l.startsWith(`stack-release: gate failed on ${C1}: pm-launch-unconfirmed`)); assert.ok(gate,r.out);assert.match(gate,why); }); await t.test('one well-formed line and a live run pass',async(t)=>{ const e=await mk(t);await seed(e,{pm:true}); const r=await run(e,'activate',C1,'acme','--pm');assert.equal(r.code,0,r.out+r.err); assert.ok(readFileSync(join(e.units,'mosaic-bus@.service'),'utf8').includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i --pm\n`)); assert.match(lines(join(e.sd,'journal-since.log')).at(-1),/^@\d+$/); assert.equal(pointer(e).pm,true); }); }); test('T11 a rollback whose gate fails too: rollback-failed, exit 2, both units stopped, no retry',async(t)=>{ const e=await mk(t);await seed(e);fault(e,`healthz-${C1}`);fault(e,`healthz-${B}`);const n=calls(e).length; const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,2,r.out+r.err); const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation); assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with healthz; both units are stopped. Rollback set: ${rs}, backup ${join(e.data,'backups','stack',l.activation)}\n`); assert.deepEqual([l.result,l.code,l.rollback,l.rollbackCode],['rollback-failed','healthz','failed','healthz']); // Row 185: the same-schema backup is named and was not restored, so no post-attempt store. assert.equal(l.backup,l.activation);assert.equal(posts(e).length,0); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']); const ss=startsStops(e,n); assert.deepEqual(ss.filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`,`start ${BUS}`,`start ${CON}`]); assert.deepEqual(ss.slice(-2),[`stop ${CON}`,`stop ${BUS}`]); assert.equal(pointer(e).copy,B); }); test('T12 rollback takes the previous copy from the log',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C1,'acme')).code,0); const r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.ok(r.out.endsWith(`stack-release: rollback ok: ${B} is live for acme\n`),r.out); assert.equal(pointer(e).copy,B); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.from,l.to,l.result],['rollback',C1,B,'ok']); }); test('T13 a rollback across a migration needs --restore, then restores and goes live',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store); const no=await run(e,'rollback','acme');assert.equal(no.code,3); assert.equal(no.err,`stack-release: refusing schema-differs: the store is at 3h and ${B} expects 3g; pass --restore ACTIVATION\n`); const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err); const o=join(e.data,'backups','stack',act); assert.match(r.out,new RegExp(`^restore: ${o}; the store now in place moves to ${e.data}/bus\\.post-attempt-\\d{8}T\\d{6}Z and is kept; 0 event\\(s\\) were written after the backup\\n`)); assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(storeVersion(e.store,true),'3g'); assert.equal(sha(e.store),backupSum(e,act)); assert.equal(posts(e).length,1);assert.equal(sha(join(e.data,posts(e)[0],'bus.sqlite')),storeC); assert.equal(logLines(e).at(-1).backup,act); }); test('T14 events written after the backup need --events N with the right N',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; const db=new DatabaseSync(e.store);for(let i=0;i<2;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run('2999-01-01T00:00:00.000Z');db.close(); const want='stack-release: refusing post-backup-events: 2 event(s) were written after the backup; Jason decides, then --events 2\n'; for(const extra of [[],['--events','1']]){ const before=fp(e),r=await run(e,'rollback','acme','--restore',act,...extra); assert.equal(r.code,3,extra.join(' '));assert.equal(r.err,want);assert.deepEqual(fp(e),before); } const r=await run(e,'rollback','acme','--restore',act,'--events','2');assert.equal(r.code,0,r.out+r.err); assert.match(r.out,/; 2 event\(s\) were written after the backup\n/); assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']); }); test('T14b a rollback --restore whose gate fails stops both units and skips item 6',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store),before=fp(e); fault(e,`healthz-${B}`);const n=calls(e).length; const r=await run(e,'rollback','acme','--restore',act,'--timeout','3');assert.equal(r.code,2,r.out+r.err); const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation),post=join(e.data,posts(e)[0]); assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with not-attempted; both units are stopped. Rollback set: ${rs}, backup ${join(e.data,'backups','stack',act)}, post-attempt store ${post}\n`); assert.deepEqual([l.result,l.code,l.rollbackCode],['rollback-failed','healthz','not-attempted']); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']); assert.equal(posts(e).length,1);assert.equal(sha(join(post,'bus.sqlite')),storeC);assert.equal(sha(e.store),backupSum(e,act)); assert.deepEqual(startsStops(e,n).filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`]); assert.equal(pointer(e).copy,C);assert.equal(sha(e.pointer),before.pointer); }); test('T14c a rollback without --restore whose gate fails runs item 6',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C1,'acme')).code,0);const before=fp(e); fault(e,`healthz-${B}`); const r=await run(e,'rollback','acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('healthz',C1)); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.code,l.rollback],['rollback','gate-failed','healthz','ok']); assert.deepEqual(fp(e),before);assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,C1)]); assert.deepEqual(readdirSync(join(e.data,'bus')).filter((n)=>n.startsWith('bus.sqlite')).sort(),['bus.sqlite','bus.sqlite-shm','bus.sqlite-wal']); }); test('T15 --plan names the migration and the restore and changes nothing',async(t)=>{ const e=await mk(t);await seed(e); const quiet=async(...args)=>{ const before=fp(e),n=calls(e).length,logs=logLines(e).length,r=await run(e,...args); assert.equal(r.code,0,r.err);assert.equal(r.err,''); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); return r.out; }; assert.ok((await quiet('activate',C,'acme','--plan')).includes('needs: --migrate (not given; the run would roll back with migration-needs-flag)\n')); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; assert.equal(await quiet('rollback','acme','--plan'),[ 'plan: rollback for acme',`from: ${C}`,`to: ${B}`, 'store schema: 3h (read from the store the running bus holds)','target schema: 3g', `restore: the store must go back to 3g; needs --restore ${act}`, 'commits: the run takes away 2 (newest first, at most 200):',...commitLines(`${B}..${C}`), ...planTail(e,B),''].join('\n')); }); test('T26 --plan lists the commits an activate adds, after the migration lines',async(t)=>{ const e=await mk(t);await seed(e); const head=(mig)=>['plan: activate for acme',`from: ${B}`,`to: ${C}`,'store schema: 3g (read from the store the running bus holds)','target schema: 3h', 'migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h',`needs: --migrate (${mig})`,'needs: Jason present for this run (decision 96 condition 2)']; const adds=commitLines(`${B}..${C}`); assert.equal(adds.length,2);assert.match(adds[0],/^ {2}[0-9a-f]{7,} row 904: C$/);assert.match(adds[1],/^ {2}[0-9a-f]{7,} row 903: C1$/); for(const [flags,mig,backup] of [[[],'not given; the run would roll back with migration-needs-flag',planNoBackup],[['--migrate'],'given',planBackup(e)]]){ const r=await run(e,'activate',C,'acme','--plan',...flags);assert.equal(r.code,0,r.err); assert.equal(r.out,[...head(mig),backup,'commits: the run adds 2 (newest first, at most 200):',...adds,...planTail(e,C),''].join('\n')); } const same=await run(e,'activate',C1,'acme','--plan');assert.equal(same.code,0,same.err); assert.ok(same.out.includes(`target schema: 3g\nmigration: none; the store is at the target's version\n${planBackup(e)}\ncommits: the run adds 1 (newest first, at most 200):\n`),same.out); }); test('T27 --plan names each migration step, or says the steps cannot be read',async(t)=>{ const e=await mk(t);await seed(e,{store:'3f',pointer:'hand'}); const two=await run(e,'activate',C,'acme','--plan');assert.equal(two.code,0,two.err); assert.ok(two.out.includes('\nmigration: 3f to 3h, after a backup of the stopped store\nmigration steps: 3f to 3g, 3g to 3h\nneeds: --migrate'),two.out); const none=await run(e,'activate',C1,'acme','--plan');assert.equal(none.code,0,none.err); assert.ok(none.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${C1}/packages/bus/src/store.mjs\nneeds: --migrate`),none.out); const short=await run(e,'activate',A,'acme','--plan');assert.equal(short.code,0,short.err); assert.ok(short.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${A}/packages/bus/src/store.mjs\nneeds: --migrate`),short.out); }); test('T25 --plan refuses copies-unrelated when neither copy is an ancestor of the other',async(t)=>{ const e=await mk(t);await seed(e,{pointer:'hand'}); const r=await run(e,'activate',S,'acme','--plan');assert.equal(r.code,3); assert.equal(r.err,`stack-release: refusing copies-unrelated: neither ${B} nor ${S} is an ancestor of the other, so the plan cannot list the commits\n`); assert.equal(r.out,'');assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock)); }); test('T23 --plan reads the store without changing a file, stopped or held',async(t)=>{ const want=['target schema: 3h','migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h','needs: --migrate (not given; the run would roll back with migration-needs-flag)','needs: Jason present for this run (decision 96 condition 2)']; for(const [name,stop,src] of [['stopped store',true,'read from the stopped store'],['held store',false,'read from the store the running bus holds']])await t.test(name,async(t)=>{ const e=await mk(t);await seed(e,{pointer:'hand'});if(stop)sysctl(e,'stop',BUS); const before=tree(e.data),units0=[sha(join(e.units,'mosaic-bus@.service')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],store0=sha(e.store); const r=await run(e,'activate',C,'acme','--plan');assert.equal(r.code,0,r.err); const out=r.out.split('\n'); assert.ok(out.includes(`store schema: 3g (${src})`),r.out); assert.deepEqual(out.slice(out.indexOf('target schema: 3h'),out.indexOf('target schema: 3h')+5),want); assert.deepEqual(tree(e.data),before);assert.equal(sha(e.store),store0); assert.deepEqual([sha(join(e.units,'mosaic-bus@.service')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],units0); assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock)); if(stop)assert.deepEqual(readdirSync(join(e.data,'bus')),['bus.sqlite']); }); }); test('T24 business-mismatch and capture-failed refuse with exit 3; a short id is a usage error',async(t)=>{ const e=await mk(t);await seed(e); let n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',C1,'other');assert.equal(r.code,3); assert.equal(r.err,'stack-release: refusing business-mismatch: the pointer is for acme\n'); assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'business-mismatch');assert.deepEqual(startsStops(e,n),[]); const sets=join(e.state,'stack-rollback');mkdirSync(sets);chmodSync(sets,0o555); n=calls(e).length;logs=logLines(e).length;const before=fp(e); const c=await run(e,'activate',C1,'acme');assert.equal(c.code,3); assert.match(c.err,new RegExp(`^stack-release: refusing capture-failed: could not write the rollback set ${sets}/\\d{8}T\\d{6}Z-${C1.slice(0,12)}; nothing was stopped\\n$`)); const l=logLines(e).slice(logs); assert.deepEqual(l.map((x)=>[x.result,x.code]),[['started',undefined],['refused','capture-failed']]);assert.equal(l[0].activation,l[1].activation); assert.deepEqual(startsStops(e,n),[]);assert.deepEqual(fp(e),before); const s=await run(e,'status','acme');assert.equal(s.code,0,s.err);assert.ok(!s.out.includes('interrupted run'),s.out); logs=logLines(e).length; const u=await run(e,'activate','abc123','acme');assert.equal(u.code,4);assert.match(u.err,/^usage: stack-release.sh adopt BUSINESS\n/); assert.equal(logLines(e).length,logs); }); test('T16 status prints the pointer, the log, the check, bus status and drift, and writes nothing',async(t)=>{ const e=await mk(t);await seed(e); const before=tree(e.data),fp0=fp(e); const r=await run(e,'status','acme');assert.equal(r.code,0,r.err); const copy=join(COPIES,B); assert.equal(r.out,[ `pointer: ${e.pointer}`,...readFileSync(e.pointer,'utf8').trimEnd().split('\n').map((l)=>` ${l}`), 'log (last five lines):',...lines(e.log).map((l)=>` ${l}`), 'check:',` bus: ${copy}`,` Console: ${copy}`,` CLI: ${copy}`,' result: pass', 'bus status:',` bus: fixture status for ${B}`, 'unit flags: match the pointer', 'drift: 4 commit(s) on refactor since the pointer\'s copy',''].join('\n')); assert.deepEqual(tree(e.data),before);assert.deepEqual(fp(e),fp0); }); test('T19 a started line with no final line: status shows it and every changing verb refuses',async(t)=>{ const e=await mk(t);await seed(e); const act=`20261018T000000Z-${C1.slice(0,12)}`; appendFileSync(e.log,JSON.stringify({at:'2026-10-18T00:00:00Z',op:'activate',from:B,to:C1,backup:null,activation:act,result:'started'})+'\n'); const s=await run(e,'status','acme');assert.equal(s.code,0); assert.ok(s.out.includes(`interrupted run: ${act} (started, no final line; Jason or Mos settles it)\n`),s.out); for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){ const before=fp(e),n=calls(e).length,r=await run(e,...args); assert.equal(r.code,3,args.join(' ')); assert.equal(r.err,`stack-release: refusing interrupted-run: activation ${act} started and has no final line; Jason or Mos settles it (status shows it)\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); } }); test('T20 a held run lock refuses run-locked without a log line; status and --plan still run',async(t)=>{ const e=await mk(t);await seed(e); const h=spawn('flock',[e.lock,'sleep','30'],{detached:true,stdio:'ignore'}); t.after(()=>{try{process.kill(-h.pid,'SIGKILL')}catch{}}); const end=Date.now()+10000; while(spawnSync('flock',['-n',e.lock,'true']).status!==1){assert.ok(Date.now()setTimeout(r,50))} for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){ const before=fp(e),logs=logLines(e).length,r=await run(e,...args); assert.equal(r.code,3,args.join(' ')); assert.equal(r.err,`stack-release: refusing run-locked: another stack-release run holds ${e.lock}\n`); assert.equal(logLines(e).length,logs);assert.deepEqual(fp(e),before); } assert.equal((await run(e,'status','acme')).code,0); const p=await run(e,'activate',C1,'acme','--plan');assert.equal(p.code,0,p.err); }); test('T21 a bus unit with no MOSAIC_CLAUDE_BIN: adopt and activate refuse no-claude-pin',async(t)=>{ const drop=(e)=>edit(join(e.units,'mosaic-bus@.service'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,''); const want=`stack-release: refusing no-claude-pin: ${BUS} has no MOSAIC_CLAUDE_BIN\n`; const a=await mk(t);await seed(a,{pointer:'none'});drop(a); const r=await run(a,'adopt','acme');assert.equal(r.code,3);assert.equal(r.err,want);assert.equal(pointer(a),null); const b=await mk(t);await seed(b);drop(b);const before=fp(b),n=calls(b).length; const s=await run(b,'activate',C1,'acme');assert.equal(s.code,3);assert.equal(s.err,want); assert.deepEqual(fp(b),before);assert.deepEqual(startsStops(b,n),[]); }); test('T22 the stopped store read: a -wal or -shm refuses store-unclean and is left alone; a clean read adds no file',async(t)=>{ for(const ext of ['-wal','-shm'])await t.test(`bus.sqlite${ext}`,async(t)=>{ const e=await mk(t);await seed(e);fault(e,`stop-leaves${ext}`); const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('store-unclean')); assert.ok(r.out.includes(`stack-release: ${e.store}${ext} is present; the store is left as it is for Jason or Mos\n`),r.out); const s=snaps(e),first=s[0],start=s.find((x)=>x.ev==='start'); assert.equal(first.ev,'stop');assert.ok(`bus.sqlite${ext}` in first.files,JSON.stringify(first)); assert.deepEqual(start.files,first.files); }); await t.test('clean',async(t)=>{ const e=await mk(t);await seed(e); const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); const s=snaps(e),stop=s.find((x)=>x.ev==='stop'),start=s.find((x)=>x.ev==='start'); assert.deepEqual(Object.keys(stop.files),['bus.sqlite']);assert.deepEqual(start.files,stop.files); }); }); // --- Row 166 (#1663): docs/plans/2026-10-18_stack-release-hardening.md items 1 to 4 --- test('T28 a unit path, drop-in directory or drop-in file that cannot be read, or a relative unit path, refuses drop-in-override before any stop',async(t)=>{ const cases=[ ['unit path',(e)=>[e.units,`cannot inspect unit path ${e.units}`]], ['drop-in directory',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'note.conf'),'[Service]\n');return [d,`cannot inspect drop-ins ${d}`]}], ['drop-in file',(e)=>{const d=join(e.units,`${BUS}.d`),f=join(d,'claude.conf');mkdirSync(d);writeFileSync(f,'[Service]\n');return [f,`cannot read drop-in ${f}`]}], ]; for(const [name,make] of cases)await t.test(name,async(t)=>{ const e=await mk(t);await seed(e); const [p,msg]=make(e),before=fp(e),n=calls(e).length,logs=logLines(e).length,mode=statSync(p).mode&0o7777; chmodSync(p,0); let r;try{r=await run(e,'activate',C1,'acme')}finally{chmodSync(p,mode)} assert.equal(r.code,3,r.out+r.err);assert.equal(r.err,`stack-release: refusing drop-in-override: ${msg}\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['active','active']); assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'drop-in-override'); }); // install_guard refuses a unit path that isn't absolute, so the precheck does too. await t.test('relative unit path',async(t)=>{ const e=await mk(t);await seed(e);fault(e,'unit-path','systemd/user'); const before=fp(e),n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',C1,'acme');assert.equal(r.code,3,r.out+r.err); assert.equal(r.err,'stack-release: refusing drop-in-override: invalid systemd unit path systemd/user\n'); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['active','active']); assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'drop-in-override'); }); }); test('T29 a bus that stops during the held-store read: --plan says so, rollback --restore refuses restore-mismatch',async(t)=>{ await t.test('--plan',async(t)=>{ const e=await mk(t);await seed(e);fault(e,'bus-stops-after','1'); const before=fp(e),logs=logLines(e).length; const r=await run(e,'activate',C,'acme','--plan');assert.equal(r.code,0,r.err);assert.equal(r.err,''); assert.equal(r.out,['plan: activate for acme',`from: ${B}`,`to: ${C}`, 'store schema: unknown (the bus stopped during the read, so the store state changed; run --plan again)','target schema: 3h',planBackup(e), 'commits: the run adds 2 (newest first, at most 200):',...commitLines(`${B}..${C}`),...planTail(e,C),''].join('\n')); assert.deepEqual(fp(e),before);assert.equal(logLines(e).length,logs); }); await t.test('rollback --restore',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; const stamp=readFileSync(join(e.data,'backups','stack',act,'STAMP'),'utf8').replace(/\n+$/,''); fault(e,'bus-stops-after','1');const before=fp(e),n=calls(e).length; const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,3,r.out+r.err); assert.equal(r.err,`stack-release: refusing restore-mismatch: the bus stopped while the events after ${stamp} were counted, so the store state changed; nothing was stopped\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(posts(e).length,0); assert.equal(logLines(e).at(-1).code,'restore-mismatch'); }); }); test('T30 a copy check that agrees on another copy fails the gate with copy-check and rolls back',async(t)=>{ const e=await mk(t);await seed(e); // A repo of its own, so the stand-in check replaces stack-copy.sh for this test only. Sourced, the // stand-in is the real script; run with `check`, it reports all three on the pointer's copy. const own=join(e.root,'repo');cpSync(repo,own,{recursive:true}); const sc=join(own,'scripts','stack-copy.sh'),at=join(COPIES,B);cpSync(sc,join(own,'scripts','stack-copy.real.sh')); writeFileSync(sc,`#!/usr/bin/env bash\nreal="$(dirname "\${BASH_SOURCE[0]}")/stack-copy.real.sh"\nif [ "\${BASH_SOURCE[0]}" != "$0" ]; then source "$real"; return 0; fi\nif [ "\${1:-}" = check ]; then printf 'bus: %s\\nConsole: %s\\nCLI: %s\\n' '${at}' '${at}' '${at}'; exit 0; fi\nexec bash "$real" "$@"\n`); e.sr=join(own,'scripts','stack-release.sh'); const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('copy-check')); assert.ok(r.out.includes(`stack-release: gate failed on ${C1}: copy-check (the bus, Console and CLI agree on ${at}, not ${join(COPIES,C1)})\n`),r.out); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.code,l.rollback],['activate','gate-failed','copy-check','ok']); assert.equal(pointer(e).copy,B);assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',at]); }); test('T31 a quoted or escaped Environment= or ExecStart= is unreadable: refusals by name, nothing read from /a',async(t)=>{ const forms=[ ['Environment double quotes',(e)=>edit(join(e.units,'mosaic-bus@.service'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,'Environment="MOSAIC_CLAUDE_BIN=/a b/claude"\n')], ['Environment single quotes',(e)=>edit(join(e.units,'mosaic-bus@.service'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,'Environment=\'MOSAIC_CLAUDE_BIN=/a b/claude\'\n')], ['Environment backslash',(e)=>edit(join(e.units,'mosaic-bus@.service'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,'Environment=MOSAIC_CLAUDE_BIN=/a\\ b/claude\n')], ['ExecStart quotes',(e)=>edit(join(e.units,'mosaic-bus@.service'),' bus start %i\n',' bus start %i "--pm"\n')], ['Console ExecStart quotes',(e)=>edit(join(e.units,CON),' --business acme\n',' --business "acme"\n')], ]; for(const [name,change] of forms)await t.test(name,async(t)=>{ const e=await mk(t);await seed(e);change(e);const before=fp(e),n=calls(e).length; const r=await run(e,'activate',C1,'acme');assert.equal(r.code,3,r.out+r.err); assert.equal(r.err,`stack-release: refusing unit-flags-drift: cannot read ${BUS} or ${CON}\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); const s=await run(e,'status','acme');assert.equal(s.code,0,s.err);assert.ok(s.out.includes('unit flags: cannot read the units\n'),s.out); const a=await mk(t);await seed(a,{pointer:'none'});change(a); const d=await run(a,'adopt','acme');assert.equal(d.code,3,d.out+d.err); assert.equal(d.err,`stack-release: refusing check-failed: cannot read ${BUS} or ${CON}\n`);assert.equal(pointer(a),null); for(const x of [r,d])assert.ok(!x.err.includes('/a'),x.err); }); }); // --- Row 169 (#1666): docs/plans/2026-10-19_pm-after-harness.md items 1 to 4 --- const HARNESS={harness:['claude-code']}; const busUnit=(e)=>readFileSync(join(e.units,'mosaic-bus@.service'),'utf8'); const execLine=(to,pm)=>`ExecStart=${COPIES}/${to}/scripts/mosaic bus start %i${pm?' --pm':''}\n`; const sinceLines=(e)=>lines(join(e.sd,'journal-since.log')).length; const PENDING_MSG='the pointer is pending (activate --pm-after-harness); run pm-on or rollback first'; // The stack after phase 1: seeded on B with --pm, then activate C1 --pm-after-harness. async function pending(e){ await seed(e,{pm:true}); const r=await run(e,'activate',C1,'acme','--pm-after-harness');assert.equal(r.code,0,r.out+r.err); } // A refusal before anything stops: exit 3, the message, no file changed, no start or stop, one log line. async function refused(e,args,code,msg){ const before=fp(e),n=calls(e).length,logs=logLines(e).length; const r=await run(e,...args);assert.equal(r.code,3,r.out+r.err);(e.outs??=[]).push(r.out,r.err); assert.equal(r.err,`stack-release: refusing ${code}: ${msg}\n`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.equal(logLines(e).length,logs+1);assert.deepEqual([logLines(e).at(-1).result,logLines(e).at(-1).code],['refused',code]); } test('T32 item 1: --pm-after-harness refuses a target without harness or a file that sets it, and passes without the PM',async(t)=>{ await t.test('usage: not with --pm, and only for activate',async(t)=>{ const e=await mk(t);await seed(e,{pm:true});const logs=logLines(e).length; for(const args of [['activate',C1,'acme','--pm','--pm-after-harness'],['activate',C1,'acme','--pm-after-harness','--pm'],['rollback','acme','--pm-after-harness'],['pm-on','acme','--pm-after-harness']]){ const r=await run(e,...args);assert.equal(r.code,4,args.join(' '));assert.match(r.err,/^usage: stack-release.sh adopt BUSINESS\n/); } assert.equal(logLines(e).length,logs); }); await t.test('pm-after-harness-target',async(t)=>{ const e=await mk(t);await seed(e,{pm:true}); await refused(e,['activate',A,'acme','--pm-after-harness'],'pm-after-harness-target',`${A} has no harness plug-in kind, so there is nothing to wait for; use --pm`); }); await t.test('pm-after-harness-set',async(t)=>{ const e=await mk(t);await seed(e,{pm:true}); for(const [h,shown] of [[['claude-code'],'["claude-code"]'],[[],'[]']]){ biz(e,{harness:h}); await refused(e,['activate',C1,'acme','--pm-after-harness'],'pm-after-harness-set',`the business file already sets harness ${shown}; use --pm`); } }); await t.test('a pointer without pm',async(t)=>{ const e=await mk(t);await seed(e); await refused(e,['activate',C1,'acme','--pm-after-harness'],'unit-flags-drift','the units differ from the pointer; settle it first:\n pm: pointer 0, --pm-after-harness needs a pointer with pm 1'); }); await t.test('phase 1 installs without --pm, skips pm_launched and writes the pending pointer',async(t)=>{ const e=await mk(t);await seed(e,{pm:true});const since=sinceLines(e),n=calls(e).length; const r=await run(e,'activate',C1,'acme','--pm-after-harness');assert.equal(r.code,0,r.out+r.err); assert.ok(r.out.endsWith(`stack-release: activate ok: ${C1} is live for acme without its PM; the PM is pending: Jason's plugins set of harness, then pm-on\n`),r.out); assert.ok(busUnit(e).includes(execLine(C1,false)),busUnit(e)); assert.equal(sinceLines(e),since); assert.deepEqual(startsStops(e,n),[`stop ${CON}`,`stop ${BUS}`,`start ${BUS}`,`start ${CON}`]); const {at,activation,...p}=pointer(e); assert.deepEqual(p,{copy:C1,business:'acme',pm:false,pmPending:true,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'}); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.from,l.to,l.result,l.pm],['activate',B,C1,'ok','pending']); }); }); test('T33 item 2: read_pointer takes pmPending only as true beside pm false; a pending pointer refuses activate and shows in status',async(t)=>{ await t.test('read_pointer',async(t)=>{ const e=await mk(t);await seed(e);const good=pointer(e); for(const bad of [{pmPending:false},{pmPending:'true'},{pmPending:1},{pmPending:null},{pm:true,pmPending:true}]){ writeFileSync(e.pointer,JSON.stringify({...good,...bad})+'\n'); const before=fp(e),n=calls(e).length,logs=logLines(e).length; for(const args of [['activate',C1,'acme'],['status','acme']]){ const r=await run(e,...args);assert.equal(r.code,1,`${JSON.stringify(bad)} ${args[0]}`); assert.ok(r.err.endsWith(`stack-release: ${e.pointer} is unreadable; Jason or Mos settles it\n`),r.err); } assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); } }); await t.test('pm-pending, already-adopted and the status line',async(t)=>{ const e=await mk(t);await pending(e); for(const args of [['activate',C,'acme','--migrate'],['activate',C,'acme','--pm','--migrate'],['activate',C1,'acme','--pm-after-harness']])await refused(e,args,'pm-pending',PENDING_MSG); await refused(e,['adopt','acme'],'already-adopted',`${e.pointer} exists (copy ${C1})`); const before=tree(e.data),r=await run(e,'status','acme');assert.equal(r.code,0,r.err); const out=r.out.split('\n'); assert.deepEqual(out.slice(0,3),[`pointer: ${e.pointer}`,` ${readFileSync(e.pointer,'utf8').trimEnd()}`,'pm: pending (activate --pm-after-harness); next: Jason\'s plugins set of harness, then stack-release.sh pm-on acme']); assert.ok(out.includes('unit flags: match the pointer'),r.out); assert.deepEqual(tree(e.data),before); }); }); test('T34 item 3: pm-on refuses before any stop, passes with the PM, or puts the phase-1 bus back',async(t)=>{ await t.test('not-pending',async(t)=>{ const e=await mk(t);await seed(e,{pm:true});biz(e,HARNESS); await refused(e,['pm-on','acme'],'not-pending','the pointer is not pending; pm-on follows activate --pm-after-harness'); }); await t.test('harness-unset',async(t)=>{ const e=await mk(t);await pending(e); await refused(e,['pm-on','acme'],'harness-unset','the business file sets no harness; Jason sets it with plugins set first'); biz(e,{harness:[]}); await refused(e,['pm-on','acme'],'harness-unset','the business file\'s harness is [], not a non-empty list'); }); await t.test('a launched line passes: the bus alone restarts with --pm',async(t)=>{ const e=await mk(t);await pending(e);biz(e,HARNESS); const since=sinceLines(e),n=calls(e).length,con=sha(join(e.units,CON)),act=pointer(e).activation; const r=await run(e,'pm-on','acme');assert.equal(r.code,0,r.out+r.err); assert.ok(r.out.endsWith(`stack-release: pm-on ok: the PM is live on ${C1} for acme\n`),r.out); assert.deepEqual(startsStops(e,n),[`stop ${BUS}`,`start ${BUS}`]); assert.ok(busUnit(e).includes(execLine(C1,true)),busUnit(e));assert.equal(sha(join(e.units,CON)),con); assert.equal(sinceLines(e),since+1);assert.match(lines(join(e.sd,'journal-since.log')).at(-1),/^@\d+$/); const {at,activation,...p}=pointer(e);assert.notEqual(activation,act); assert.deepEqual(p,{copy:C1,business:'acme',pm:true,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'}); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.to,l.result,l.pm,l.activation],['pm-on',C1,'ok','on',activation]); const s=await run(e,'status','acme');assert.equal(s.code,0,s.err);assert.ok(!s.out.includes('pm: pending'),s.out); }); await t.test('a failed pm_launched puts the phase-1 unit back: exit 1, still pending',async(t)=>{ const e=await mk(t);await pending(e);biz(e,HARNESS);fault(e,`pm-${C1}`,'none'); const before=fp(e),p0=readFileSync(e.pointer,'utf8'),n=calls(e).length; const r=await run(e,'pm-on','acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation); assert.equal(r.err,`stack-release: pm-on failed with pm-launch-unconfirmed (no launched line since the start); the bus runs ${C1} without its PM again and passed the phase-1 gate, and the pointer stays pending. Fix harness with plugins set, then run pm-on again. Rollback set: ${rs}\n`); assert.deepEqual([l.op,l.result,l.code,l.why],['pm-on','pm-on-failed','pm-launch-unconfirmed','no launched line since the start']); assert.deepEqual(fp(e),before);assert.ok(busUnit(e).includes(execLine(C1,false))); assert.equal(readFileSync(e.pointer,'utf8'),p0); assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd,units(e)[CON].state],['active',join(COPIES,C1),'active']); assert.deepEqual(startsStops(e,n),[`stop ${BUS}`,`start ${BUS}`,`stop ${BUS}`,`start ${BUS}`]); }); await t.test('a failed phase-1 gate too: pm-on-down, exit 2, both units stopped',async(t)=>{ const e=await mk(t);await pending(e);biz(e,HARNESS);fault(e,`healthz-${C1}`); const p0=readFileSync(e.pointer,'utf8'),store=sha(e.store); const r=await run(e,'pm-on','acme','--timeout','3');assert.equal(r.code,2,r.out+r.err); const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation); assert.equal(r.err,`stack-release: pm-on-down: pm-on failed with healthz and the phase-1 bus with healthz; both units are stopped. Rollback set: ${rs}\n`); assert.deepEqual([l.result,l.code,l.phase1],['pm-on-down','healthz','healthz']); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']); assert.equal(readFileSync(e.pointer,'utf8'),p0);assert.equal(sha(e.store),store); assert.ok(busUnit(e).includes(execLine(C1,false))); }); }); test('T35 item 4: rollback from pending installs --pm; a plug-in kind the target lacks refuses by name',async(t)=>{ await t.test('rollback from pending gates pm_launched on the old copy',async(t)=>{ const e=await mk(t);await pending(e);const since=sinceLines(e); const r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.ok(busUnit(e).includes(execLine(B,true)),busUnit(e)); assert.equal(sinceLines(e),since+1); const {at,activation,...p}=pointer(e); assert.deepEqual(p,{copy:B,business:'acme',pm:true,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'}); }); await t.test('after the harness set, rollback refuses until harness is gone',async(t)=>{ const e=await mk(t);await pending(e);biz(e,HARNESS); assert.equal((await run(e,'pm-on','acme')).code,0); await refused(e,['rollback','acme'],'unknown-kind-for-target',`the business file names plug-in kind(s) harness, which ${B} does not know; set each to null with plugins set through the live bus first`); biz(e); const r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.ok(busUnit(e).includes(execLine(B,true)));assert.deepEqual([pointer(e).copy,pointer(e).pm],[B,true]); }); await t.test('activate names every kind the target lacks',async(t)=>{ const e=await mk(t);await seed(e); biz(e,{...HARNESS,widget:{}}); await refused(e,['activate',A,'acme'],'unknown-kind-for-target',`the business file names plug-in kind(s) harness widget, which ${A} does not know`); await refused(e,['activate',C1,'acme'],'unknown-kind-for-target',`the business file names plug-in kind(s) widget, which ${C1} does not know`); }); }); test('T36 --plan shows the pending state and lists the row 169 refusals it can see',async(t)=>{ const e=await mk(t);await seed(e,{pm:true}); const quiet=async(code,...args)=>{ const before=fp(e),n=calls(e).length,logs=logLines(e).length,r=await run(e,...args); assert.equal(r.code,code,r.out+r.err);assert.equal(r.err,''); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); return r.out.split('\n'); }; let out=await quiet(0,'activate',C1,'acme','--pm-after-harness','--plan'); assert.deepEqual(out.slice(-6),['pm: the bus starts without --pm and the PM stays pending; next: Jason\'s plugins set of harness, then pm-on',...planTail(e,C1),'']); biz(e,HARNESS); out=await quiet(3,'activate',A,'acme','--pm-after-harness','--plan'); assert.deepEqual(out.slice(-7),['pm: the bus starts without --pm and the PM stays pending; next: Jason\'s plugins set of harness, then pm-on',...planTail(e,A),`refuses unknown-kind-for-target: the business file names plug-in kind(s) harness, which ${A} does not know`,'']); out=await quiet(3,'activate',C1,'acme','--pm-after-harness','--plan'); assert.equal(out.at(-2),'refuses pm-after-harness-set: the business file already sets harness ["claude-code"]; use --pm'); biz(e);assert.equal((await run(e,'activate',C1,'acme','--pm-after-harness')).code,0); out=await quiet(3,'activate',C,'acme','--plan'); assert.deepEqual(out.slice(-7),['pm: the pointer is pending (activate --pm-after-harness); the bus runs without its PM',...planTail(e,C,true),`refuses pm-pending: ${PENDING_MSG}`,'']); out=await quiet(0,'rollback','acme','--plan'); assert.deepEqual(out.slice(-6),['pm: the pointer is pending (activate --pm-after-harness); the bus runs without its PM',...planTail(e,B,true),'']); }); // Row 176 test 11: supported and older Console copies deliberately differ from the checkout. const testerFlags=(e,port='7333')=>['--port',e.port,'--business','acme','--tester-port',port]; const consoleText=(e)=>readFileSync(join(e.units,CON),'utf8'); const stepText=(e)=>readFileSync(join(e.state,'stack-rollback',pointer(e).activation,'steps.log'),'utf8'); test('T37 tester-port activate writes pointer and unit, replaces the port, and supported re-pin and rollback keep it',async(t)=>{ const e=await mk(t);await seed(e); let r=await run(e,'activate',C1,'acme','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,testerFlags(e));assert.match(consoleText(e),/--tester-port 7333(?:\n|$)/); r=await run(e,'status','acme');assert.equal(r.code,0,r.err);assert.match(r.out,/unit flags: match the pointer/); r=await run(e,'activate',B,'acme','--tester-port','7334');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,testerFlags(e,'7334')); assert.match(consoleText(e),/--tester-port 7334(?:\n|$)/);assert.doesNotMatch(consoleText(e),/--tester-port 7333/); assert.equal((consoleText(e).match(/--tester-port/g)||[]).length,1); r=await run(e,'activate',C1,'acme','--plan');assert.equal(r.code,0,r.out+r.err);assert.match(r.out,/Console: .*--tester-port 7334/); r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err);assert.deepEqual(pointer(e).console,testerFlags(e,'7334')); r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err);assert.equal(pointer(e).copy,B); assert.deepEqual(pointer(e).console,testerFlags(e,'7334'));assert.match(consoleText(e),/--tester-port 7334(?:\n|$)/); }); test('T38 tester-port rollback to unsupported target drops flag in plan, steps, unit and pointer',async(t)=>{ const e=await mk(t);await seed(e,{copy:A}); let r=await run(e,'activate',B,'acme','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); const before=fp(e),n=calls(e).length; r=await run(e,'rollback','acme','--plan');assert.equal(r.code,0,r.out+r.err); assert.match(r.out,/drop[^\n]*--tester-port 7333/i);assert.doesNotMatch(r.out.split('\n').find((l)=>l.startsWith('Console: ')),/--tester-port/); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err);assert.equal(pointer(e).copy,A); assert.deepEqual(pointer(e).console,['--port',e.port,'--business','acme']);assert.doesNotMatch(consoleText(e),/--tester-port/); assert.match(stepText(e),/drop[^\n]*--tester-port 7333/i); }); test('T39 tester-port explicit unsupported target reads target copy and refuses before stopping either unit',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e),n=calls(e).length; const r=await run(e,'activate',A,'acme','--tester-port','7333');assert.equal(r.code,3,r.out+r.err); assert.match(r.err,/refusing tester-port-target:/);assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); assert.equal(units(e)[BUS].state,'active');assert.equal(units(e)[CON].state,'active'); // The repository's Console accepts the flag while A's Console does not: probing REPO fails this test. assert.match(readFileSync(join(repo,'packages/webui/src/cli.mjs'),'utf8'),/--tester-port/); }); test('T40 tester-port invalid values refuse exit 4 before stopping, including tunnel and main ports',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e),n=calls(e).length; for(const value of ['7332',e.port,'1023','65536','x','7333.5']){ const r=await run(e,'activate',C1,'acme','--tester-port',value);assert.equal(r.code,4,`${value}: ${r.out}${r.err}`); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); } }); test('T41 tester-port inherited activate to unsupported target drops it and retains drift refusal and failed-run recovery',async(t)=>{ const e=await mk(t);await seed(e); let r=await run(e,'activate',C1,'acme','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); r=await run(e,'activate',A,'acme','--plan');assert.equal(r.code,0,r.out+r.err);assert.match(r.out,/drop[^\n]*--tester-port 7333/i); fault(e,`check-${A}`);const before=fp(e); r=await run(e,'activate',A,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err);assert.deepEqual(fp(e),before); rmSync(join(e.sd,'faults',`check-${A}`)); edit(join(e.units,CON),'--tester-port 7333','--tester-port 7334');const n=calls(e).length; r=await run(e,'activate',A,'acme');assert.equal(r.code,3,r.out+r.err);assert.match(r.err,/unit-flags-drift/);assert.deepEqual(startsStops(e,n),[]); edit(join(e.units,CON),'--tester-port 7334','--tester-port 7333'); r=await run(e,'activate',A,'acme');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,['--port',e.port,'--business','acme']);assert.doesNotMatch(consoleText(e),/--tester-port/); assert.match(stepText(e),/drop[^\n]*--tester-port 7333/i); }); test('T42 tester-port pm-on preserves pointer Console flags',async(t)=>{ const e=await mk(t);await seed(e,{pm:true}); let r=await run(e,'activate',C1,'acme','--pm-after-harness','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); biz(e,HARNESS); r=await run(e,'pm-on','acme');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,testerFlags(e));assert.match(consoleText(e),/--tester-port 7333(?:\n|$)/); }); // Row 180 item 1 (P19 391): a date stand-in prints one second for every call that names a time, // so each run below starts in that second, as two runs within one real second do. Calls for // +%s (the gate's start mark) pass through, so each gate still reads only its own start. function pinClock(e){ const dir=join(e.root,'clock'),second=Math.floor(Date.now()/1000);mkdirSync(dir); writeFileSync(join(dir,'date'),`#!/bin/sh\ncase "$*" in *%s*) exec /usr/bin/date "$@" ;; esac\nexec /usr/bin/date -d @${second} "$@"\n`,{mode:0o755}); e.env={...e.env,PATH:`${dir}:${e.env.PATH}`}; return execFileSync(join(dir,'date'),['-u','+%Y%m%dT%H%M%SZ'],{env:e.env,encoding:'utf8'}).trim(); } const rollbackSets=(e)=>readdirSync(join(e.state,'stack-rollback')).sort(); test('T43 row 180 item 1: runs in one second get distinct activation ids, rollback sets and post-attempt stores',async(t)=>{ await t.test('activate then pm-on on the same copy',async(t)=>{ const e=await mk(t);await seed(e,{pm:true});const sec=pinClock(e),id=`${sec}-${C1.slice(0,12)}`; let r=await run(e,'activate',C1,'acme','--pm-after-harness');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).activation,id); biz(e,HARNESS); r=await run(e,'pm-on','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).activation,`${id}-1`); assert.deepEqual(logLines(e).slice(-4).map((l)=>[l.op,l.result,l.activation]),[['activate','started',id],['activate','ok',id],['pm-on','started',`${id}-1`],['pm-on','ok',`${id}-1`]]); assert.deepEqual(rollbackSets(e),[id,`${id}-1`]); for(const a of [id,`${id}-1`])assert.ok(existsSync(join(e.state,'stack-rollback',a,'SHA256SUMS')),a); }); await t.test('two restores: an old-form id and a suffixed one, each with its own post-attempt store',async(t)=>{ const e=await mk(t);await seed(e);const sec=pinClock(e),c=`${sec}-${C.slice(0,12)}`,b=`${sec}-${B.slice(0,12)}`; for(const [act,restore] of [[c,b],[`${c}-1`,`${b}-1`]]){ let r=await run(e,'activate',C,'acme','--migrate');assert.equal(r.code,0,r.out+r.err);assert.equal(pointer(e).activation,act); assert.ok(existsSync(join(e.data,'backups','stack',act,'SHA256SUMS')),act); r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err); const post=join(e.data,`bus.post-attempt-${sec}${restore.slice(b.length)}`); assert.ok(r.out.startsWith(`restore: ${join(e.data,'backups','stack',act)}; the store now in place moves to ${post} and is kept; 0 event(s) were written after the backup\n`),r.out); assert.deepEqual([pointer(e).copy,pointer(e).activation,pointer(e).schema],[B,restore,'3g']); assert.equal(logLines(e).at(-1).backup,act); } assert.deepEqual(posts(e).sort(),[`bus.post-attempt-${sec}`,`bus.post-attempt-${sec}-1`]); assert.deepEqual(readdirSync(join(e.data,'backups','stack')).sort(),[c,`${c}-1`]); assert.deepEqual(rollbackSets(e),[c,`${c}-1`,b,`${b}-1`].sort()); }); await t.test('adopt takes the next free id too',async(t)=>{ const e=await mk(t);await seed(e,{pointer:'none'});const id=`${pinClock(e)}-${B.slice(0,12)}`; mkdirSync(join(e.state,'stack-rollback',id),{recursive:true}); const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).activation,`${id}-1`); assert.deepEqual(logLines(e).map((l)=>[l.op,l.result,l.activation]),[['adopt','ok',`${id}-1`]]); }); await t.test('all ten names taken: capture-failed before anything stops, and the log line names no activation',async(t)=>{ const e=await mk(t);await seed(e);const sec=pinClock(e),id=`${sec}-${C1.slice(0,12)}`; // A rollback set, a backup or a post-attempt store each takes a name. for(const s of ['','-1','-2','-3','-4','-5','-6','-7'])mkdirSync(join(e.state,'stack-rollback',`${id}${s}`),{recursive:true}); mkdirSync(join(e.data,'backups','stack',`${id}-8`),{recursive:true});writeFileSync(join(e.data,`bus.post-attempt-${sec}-9`),''); await refused(e,['activate',C1,'acme'],'capture-failed',`every activation id for ${id} is taken; nothing was stopped`); assert.equal(logLines(e).at(-1).activation,null); }); await t.test('--restore takes an id with or without -1 to -9 and nothing else',async(t)=>{ const e=await mk(t);await seed(e);const id=`20261020T013708Z-${B.slice(0,12)}`; for(const bad of [`${id}-0`,`${id}-10`,`${id}-a`,`${id}-`,`${id}1`]){ const r=await run(e,'rollback','acme','--restore',bad,'--plan');assert.equal(r.code,4,bad); } for(const good of [id,`${id}-1`,`${id}-9`]){ const r=await run(e,'rollback','acme','--restore',good,'--plan');assert.notEqual(r.code,4,`${good}: ${r.err}`); } }); }); test('T44 row 180 item 2: --plan lists tester-port-invalid and tester-port-target with its other refusals',async(t)=>{ const usageText=async(e)=>(await run(e,'--help')).err; await t.test('a bad tester port',async(t)=>{ const e=await mk(t);await seed(e,{pm:true});biz(e,HARNESS); const before=fp(e),n=calls(e).length,logs=logLines(e).length,p0=readFileSync(e.pointer,'utf8'); const rule=(port,main)=>`--tester-port ${port} is not a whole port from 1024 to 65535 other than 7332${main?` and the Console's --port ${main}`:''}`; // The fixture's Console has --port e.port; a pointer without --port has the default 7332, which // the reason names once. --plan reads the pointer's flags and not the units, so only it changes. const noPort=()=>{const q=JSON.parse(p0);q.console=q.console.filter((f,i,a)=>f!=='--port'&&a[i-1]!=='--port');writeFileSync(e.pointer,JSON.stringify(q)+'\n')}; for(const [port,reason,edit] of [['80',rule('80',e.port)],[e.port,rule(e.port,e.port)],['80',rule('80'),noPort],['7332',rule('7332'),noPort]]){ if(edit)edit(); const r=await run(e,'activate',C1,'acme','--pm-after-harness','--plan','--tester-port',port); assert.equal(r.code,3,r.out+r.err);assert.equal(r.err,''); assert.deepEqual(r.out.split('\n').filter((l)=>l.startsWith('refuses ')),[`refuses tester-port-invalid: ${reason}`,'refuses pm-after-harness-set: the business file already sets harness ["claude-code"]; use --pm']); const u=await run(e,'activate',C1,'acme','--pm-after-harness','--tester-port',port); assert.equal(u.code,4,u.out+u.err);assert.equal(u.out,'');assert.equal(u.err,await usageText(e)); writeFileSync(e.pointer,p0); } assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); }); await t.test('without --plan a bad port is usage before any pointer read',async(t)=>{ // parse_opts checks it, so a host with no pointer gets usage and not no-pointer (exit 3, a log line). const e=await mk(t);await seed(e,{pointer:'none'}); const n=calls(e).length,u=await run(e,'activate',C1,'acme','--tester-port','80'); assert.equal(u.code,4,u.out+u.err);assert.equal(u.out,'');assert.equal(u.err,await usageText(e)); assert.deepEqual(logLines(e),[]);assert.deepEqual(startsStops(e,n),[]); }); await t.test('a target Console without --tester-port',async(t)=>{ const e=await mk(t);await seed(e);biz(e,HARNESS); const before=fp(e),n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',A,'acme','--tester-port','7333','--plan'); assert.equal(r.code,3,r.out+r.err);assert.equal(r.err,''); assert.deepEqual(r.out.split('\n').filter((l)=>l.startsWith('refuses ')),[`refuses tester-port-target: ${A}'s Console does not support --tester-port`,`refuses unknown-kind-for-target: the business file names plug-in kind(s) harness, which ${A} does not know`]); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); await refused(e,['activate',A,'acme','--tester-port','7333'],'tester-port-target',`${A}'s Console does not support --tester-port`); }); }); // Row 181 (#1679, P19 390): COUNTED and STORE_JS as the script under test has them. The fixture // tables carry only seq and at, which is all the count reads. const srText=readFileSync(SR,'utf8'); const COUNTED=/^COUNTED="([^"]*)"$/m.exec(srText)[1].split(' '); const STORE_JS=/^STORE_JS='([\s\S]*?)'$/m.exec(srText)[1]; // sage-1163's list 2, written out so that a table dropped from the script's COUNTED fails T45 as // well as T46. const LISTED=['role_claims','thread_claims','decision_events','messages','deliveries','task_snapshots','projects','threads','thread_opens','instance_names','usage_records']; const LATE='2999-01-01T00:00:00.000Z'; const addRow=(file,table,at=LATE)=>{const db=new DatabaseSync(file);db.exec(`CREATE TABLE IF NOT EXISTS ${table} (seq INTEGER PRIMARY KEY, at TEXT NOT NULL)`);db.prepare(`INSERT INTO ${table} (at) VALUES (?)`).run(at);db.close()}; test('T45 row 181 items 2 and 3: the restore guard counts events and every COUNTED table',async(t)=>{ await t.test('both modes count events and each listed table, and a table the store lacks counts 0',async(t)=>{ const d=mkdtempSync(join(tmpdir(),'sr181-'));t.after(()=>rmSync(d,{recursive:true,force:true})); const stamp='2020-01-01T00:00:00.000Z',count=(f,mode)=>execFileSync(NODE,['-e',STORE_JS,'--',mode,f,stamp,COUNTED.join(' ')],{encoding:'utf8'}); const store=(name,rows,tables=rows)=>{ const f=join(d,name);for(const x of ['events',...tables])addRow(f,x,'2000-01-01T00:00:00.000Z');for(const x of rows)addRow(f,x);return f; }; const both=(f,want)=>{const s=sha(f);for(const mode of ['count','heldcount'])assert.equal(count(f,mode),want,`${basename(f)} ${mode}`);assert.equal(sha(f),s);assert.deepEqual(readdirSync(d).filter((n)=>n.startsWith(`${basename(f)}-`)),[])}; for(const x of LISTED)both(store(`${x}.sqlite`,['events',x]),`2\t1 event(s), 1 ${x} row(s)\n`); both(store('brief.sqlite',['events','usage_records','instance_names']),'3\t1 event(s), 1 instance_names row(s), 1 usage_records row(s)\n'); // A 3f store: no usage_records table, and nothing to refuse. both(store('3f.sqlite',['events'],['events','instance_names']),'1\t1 event(s)\n'); both(store('none.sqlite',[],['instance_names']),'0\t0 event(s)\n'); }); for(const x of ['instance_names','usage_records'])await t.test(`one ${x} row and no event refuses before any stop; --events 1 restores`,async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; addRow(e.store,x); const want=`stack-release: refusing post-backup-events: 0 event(s), 1 ${x} row(s) were written after the backup; Jason decides, then --events 1\n`; for(const extra of [[],['--events','2']]){ const before=fp(e),n=calls(e).length,r=await run(e,'rollback','acme','--restore',act,...extra); assert.equal(r.code,3,extra.join(' '));assert.equal(r.err,want);assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); } const r=await run(e,'rollback','acme','--restore',act,'--events','1');assert.equal(r.code,0,r.out+r.err); assert.ok(r.out.includes(`; 0 event(s), 1 ${x} row(s) were written after the backup\n`),r.out); assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(sha(e.store),backupSum(e,act)); }); await t.test('a listed table that cannot be read refuses restore-mismatch before any stop',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; const stamp=readFileSync(join(e.data,'backups','stack',act,'STAMP'),'utf8').replace(/\n+$/,''); const db=new DatabaseSync(e.store);db.exec('CREATE TABLE usage_records (seq INTEGER PRIMARY KEY)');db.close(); const before=fp(e),n=calls(e).length,r=await run(e,'rollback','acme','--restore',act); assert.equal(r.code,3,r.out+r.err);assert.ok(r.err.endsWith(`stack-release: refusing restore-mismatch: cannot count the rows after ${stamp}\n`),r.err); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).at(-1).code,'restore-mismatch'); }); await t.test('the in-run recount counts the listed tables too',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation; fault(e,'stop-writes-row','thread_opens'); const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('post-backup-events',C)); assert.ok(r.out.includes('; 0 event(s) were written after the backup\n'),r.out); assert.deepEqual([pointer(e).copy,pointer(e).schema],[C,'3h']);assert.equal(posts(e).length,0); const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','post-backup-events','ok']); }); }); test('T46 row 181 item 4: every table in the bus schema is in exactly one of four lists',()=>{ // The schema a target copy's store gets. Lists 1 and 2 come from the script, so a table added to // COUNTED or dropped from it shows here. Row 181's BUILD.md names each writer. const db=new DatabaseSync(':memory:');db.exec(readFileSync(join(ROOT,'packages','bus','schema.sql'),'utf8')); const tables=db.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all().map((r)=>r.name); const hasAt=(n)=>db.prepare(`SELECT count(*) AS n FROM pragma_table_info(?) WHERE name='at'`).get(n).n===1; const always={decisions:'decision.raise writes action.allowed in the same transaction'}; const untimed={ meta:'key and value only; its one UPDATE is the migration step', message_threads:'no time column; each row goes with a messages row, which is counted', sqlite_sequence:'SQLite\'s own AUTOINCREMENT table', task_comment_cursors:'no time column; a restore drops its advances uncounted (BUILD.md)', }; const lists=[['events'],COUNTED,Object.keys(always),Object.keys(untimed)],all=lists.flat(); assert.equal(new Set(all).size,all.length,`a table is in two lists: ${all.filter((n,i)=>all.indexOf(n)!==i)}`); assert.deepEqual(tables.filter((n)=>!all.includes(n)),[],'tables in no list'); assert.deepEqual(all.filter((n)=>!tables.includes(n)),[],'listed tables the schema lacks'); for(const n of lists.slice(0,3).flat())assert.ok(hasAt(n),`${n} has at`); for(const n of lists[3])assert.ok(!hasAt(n),`${n} has no at`); db.close(); }); // Row 185 (#1683): every activate with a store backs it up under its activation id, before any // migration. Only a backup taken before a migration (TAKEN) is restored after a gate failure. test('T47 row 185 test 1: a same-schema activate leaves a whole backup and the log line names it',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e); const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); const act=pointer(e).activation,o=join(e.data,'backups','stack',act); assert.ok(r.out.includes(`backup: ${o}\n`),r.out); for(const f of ['SHA256SUMS','DATA_ROOT','SCHEMA','STAMP'])assert.ok(existsSync(join(o,f)),f); assert.equal(readFileSync(join(o,'DATA_ROOT'),'utf8'),`${e.data}\n`); assert.equal(readFileSync(join(o,'SCHEMA'),'utf8'),`${storeVersion(e.store,true)}\n`);assert.equal(storeVersion(e.store,true),'3g'); assert.match(readFileSync(join(o,'STAMP'),'utf8'),/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\n$/); assert.equal(backupSum(e,act),before.store); const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.activation,l.backup],['activate','ok',act,act]); }); test('T48 row 185 test 2: a gate failure on a same-schema activate rolls back without a restore',async(t)=>{ const e=await mk(t);await seed(e);const before=fp(e); fault(e,`start-writes-row-${C1}`);fault(e,`healthz-${C1}`); const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); assert.equal(r.err,rolledBack('healthz')); const l=logLines(e).at(-1),act=l.activation; assert.deepEqual([l.result,l.code,l.rollback,l.backup],['gate-failed','healthz','ok',act]); // The store as the target's bus left it when the rollback stopped it, after the failed gate. const failed=snaps(e).filter((x)=>x.ev==='stop').at(-1).files['bus.sqlite'][2]; assert.notEqual(failed,backupSum(e,act),'the target start wrote a row after the backup'); assert.equal(sha(e.store),failed);assert.equal(posts(e).length,0); assert.deepEqual({...fp(e),store:before.store},before);assert.equal(units(e)[BUS].wd,join(COPIES,B)); }); test('T49 row 185 test 3: a same-schema rollback --restore restores, or refuses rows written after STAMP',async(t)=>{ await t.test('no rows after STAMP: restores and goes live',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C1,'acme')).code,0);const act=pointer(e).activation,storeC1=sha(e.store); const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err); assert.ok(r.out.startsWith(`restore: ${join(e.data,'backups','stack',act)}; the store now in place moves to `),r.out); assert.ok(r.out.includes('; 0 event(s) were written after the backup\n'),r.out); assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(sha(e.store),backupSum(e,act)); assert.equal(posts(e).length,1);assert.equal(sha(join(e.data,posts(e)[0],'bus.sqlite')),storeC1); assert.equal(logLines(e).at(-1).backup,act); }); await t.test('one instance_names row after STAMP refuses post-backup-events; --events 1 restores',async(t)=>{ const e=await mk(t);await seed(e); assert.equal((await run(e,'activate',C1,'acme')).code,0);const act=pointer(e).activation; addRow(e.store,'instance_names'); const before=fp(e),n=calls(e).length,no=await run(e,'rollback','acme','--restore',act); assert.equal(no.code,3,no.out+no.err); assert.equal(no.err,'stack-release: refusing post-backup-events: 0 event(s), 1 instance_names row(s) were written after the backup; Jason decides, then --events 1\n'); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); const r=await run(e,'rollback','acme','--restore',act,'--events','1');assert.equal(r.code,0,r.out+r.err); assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(sha(e.store),backupSum(e,act)); }); }); test('T50 row 185 test 4: activate --plan prints the backup line and writes nothing',async(t)=>{ const e=await mk(t);await seed(e); const before=fp(e),data=tree(e.data),n=calls(e).length,logs=logLines(e).length; const r=await run(e,'activate',C1,'acme','--plan');assert.equal(r.code,0,r.err);assert.equal(r.err,''); assert.deepEqual(r.out.split('\n').filter((x)=>x.startsWith('backup: ')),[planBackup(e)]); assert.deepEqual(fp(e),before);assert.deepEqual(tree(e.data),data);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); assert.ok(!existsSync(join(e.data,'backups'))); // Rollback is unchanged: it takes no backup, so its plan has no backup line. assert.equal((await run(e,'activate',C1,'acme')).code,0); const rb=await run(e,'rollback','acme','--plan');assert.equal(rb.code,0,rb.err); assert.deepEqual(rb.out.split('\n').filter((x)=>x.startsWith('backup')),[]); }); test('T51 row 185 test 5: an activate with no store file takes no backup',async(t)=>{ const e=await mk(t);await seed(e); sysctl(e,'stop',BUS);for(const f of readdirSync(join(e.data,'bus')))rmSync(join(e.data,'bus',f)); const p=await run(e,'activate',C1,'acme','--plan');assert.equal(p.code,0,p.err); assert.deepEqual(p.out.split('\n').filter((x)=>x.startsWith('backup')),['backup: none; there is no store file']); const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); assert.ok(!existsSync(join(e.data,'backups')));assert.ok(!r.out.includes('backup: '),r.out); const l=logLines(e).at(-1);assert.deepEqual([l.result,l.backup],['ok',null]); }); // Row 186 (#1684): the activate plan prints the backup line only when the run reaches backup_store. // Each case plans, checks nothing was written, then runs and checks a backup exists exactly when the plan said so. test('T52 row 186: the activate plan\'s backup line says what the run will do',async(t)=>{ const newer='migration: none possible; 3z is not in 3h\'s migration chain, so the run would roll back with store-newer'; const cases=[ ['same schema','3g',C1,[],'migration: none; the store is at the target\'s version',true,0], ['same schema with --migrate','3g',C1,['--migrate'],'migration: none; the store is at the target\'s version',true,0], ['in chain with --migrate','3g',C,['--migrate'],'needs: --migrate (given)',true,0], ['in chain without --migrate','3g',C,[],'needs: --migrate (not given; the run would roll back with migration-needs-flag)',false,1], ['outside the chain','3z',C,[],newer,false,1], ['outside the chain with --migrate','3z',C,['--migrate'],newer,false,1], ]; for(const [name,store,to,flags,why,backup,code] of cases)await t.test(name,async(t)=>{ const e=await mk(t);await seed(e,{store}); const before=fp(e),data=tree(e.data),n=calls(e).length,logs=logLines(e).length; const p=await run(e,'activate',to,'acme','--plan',...flags);assert.equal(p.code,0,p.err);assert.equal(p.err,''); const out=p.out.split('\n');assert.ok(out.includes(why),p.out); assert.deepEqual(out.filter((x)=>x.startsWith('backup')),[backup?planBackup(e):planNoBackup]); assert.deepEqual(fp(e),before);assert.deepEqual(tree(e.data),data);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); assert.ok(!existsSync(join(e.data,'backups'))); const r=await run(e,'activate',to,'acme',...flags,'--timeout','3');assert.equal(r.code,code,r.out+r.err); assert.equal(existsSync(join(e.data,'backups','stack')),backup);assert.equal(logLines(e).at(-1).backup??null,backup?logLines(e).at(-1).activation:null); }); }); // Row 178: proof metadata uses temp fixture files, never the installed slot. async function seedProof(e) { await seed(e); const file=join(e.root,'proof.fixture');writeFileSync(file,'fixture_proxy_proof_012345678901234567890123456789',{mode:0o600}); const extra=['--public-host','console.example','--auth-header','x-user','--allow-user','jason','--proof-header','x-console-proof','--proof-file',file]; const p=pointer(e);p.console.push(...extra);writeFileSync(e.pointer,JSON.stringify(p)+'\n'); const unit=join(e.units,CON);edit(unit,` --business acme\n`,` --business acme ${extra.join(' ')}\n`); return p.console; } test('T178 proxy proof header and path survive re-pin and rollback unchanged in the pointer and unit',async t=>{ const e=await mk(t),expected=await seedProof(e); let r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,expected);assert.ok(consoleText(e).includes(expected.join(' '))); r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,B);assert.deepEqual(pointer(e).console,expected);assert.ok(consoleText(e).includes(expected.join(' '))); }); test('T178 proxy proof pair survives a tester-port replacement as well',async t=>{ const e=await mk(t),expected=await seedProof(e); const r=await run(e,'activate',C1,'acme','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,[...expected,'--tester-port','7333']); assert.ok(consoleText(e).includes([...expected,'--tester-port','7333'].join(' '))); }); // Row 187 (#1685): activate sets, carries and drops the proof pair. Every proof file here holds // SENTINEL, which no output, unit, pointer, log or steps log may hold (brief test 8), and an // inotify watch counts each open or read of it, which must stay 0: the script checks the file // with lstat and stat only. Python's ctypes reaches inotify(7); node's fs.watch has no IN_ACCESS. const SENTINEL='row187_sentinel_proof_value_0123456789abcdef'; const PREVIEW=['--public-host','console.example','--auth-header','x-user','--allow-user','jason']; const WATCH=`import ctypes,os,select,struct,sys libc=ctypes.CDLL(None,use_errno=True) fd=libc.inotify_init1(0) if fd<0 or libc.inotify_add_watch(fd,os.fsencode(sys.argv[1]),0x21)<0:sys.exit(2) print('ready',flush=True) n=0 def drain(t): global n while select.select([fd],[],[],t)[0]: b=os.read(fd,65536);i=0 while i{for(const ch of WATCHERS)ch.kill('SIGKILL')}); // A proof file under dir (e.root, outside the data root, unless given), with its read watch running. async function proofFile(e,name='proof.187',mode=0o600,dir=e.root){ const f=join(dir,name);writeFileSync(f,SENTINEL+'\n',{mode});chmodSync(f,mode); const ch=spawn('python3',['-I','-c',WATCH,f],{stdio:['pipe','pipe','inherit']}); let out='';ch.stdout.on('data',(d)=>{out+=d}); await new Promise((res,rej)=>{ch.stdout.on('data',()=>{if(out.startsWith('ready\n'))res()});ch.on('error',rej);ch.on('close',(c)=>rej(new Error(`inotify watch exited ${c}`)))}); WATCHERS.add(ch);ch.on('close',()=>WATCHERS.delete(ch)); (e.watches??=[]).push(async()=>{ch.stdin.end();if(ch.exitCode===null)await once(ch,'close');return Number(out.split('\n')[1])}); return f; } async function prun(e,...args){const r=await run(e,...args);(e.outs??=[]).push(r.out,r.err);return r} // Brief test 8 and the read mutant, at the end of each case. async function noLeak(e){ const hits=[]; const walk=(q)=>{let st;try{st=lstatSync(q)}catch{return}if(st.isDirectory()){for(const n of readdirSync(q))walk(join(q,n))}else if(st.isFile()&&readFileSync(q).includes(SENTINEL))hits.push(q)}; for(const d of [e.data,e.units,e.sd,e.bin])walk(d); (e.outs??[]).forEach((o,i)=>{if(o.includes(SENTINEL))hits.push(`run output ${i}`)}); assert.deepEqual(hits,[]); for(const stop of e.watches??[])assert.equal(await stop(),0,'the run opened or read the proof file'); } // The seeded stack with the preview flags, and any flags in more, in the pointer and the Console unit. async function seedPreview(e,opts,more=[]){ await seed(e,opts); const p=pointer(e);p.console.push(...PREVIEW,...more);writeFileSync(e.pointer,JSON.stringify(p)+'\n'); edit(join(e.units,CON),' --business acme\n',` --business acme ${[...PREVIEW,...more].join(' ')}\n`); return p.console; } const pair=(f,h='x-console-proof')=>['--proof-header',h,'--proof-file',f]; const conExec=(e)=>consoleText(e).split('\n').find((l)=>l.startsWith('ExecStart=')); // --plan: exit code, nothing written, nothing started or stopped, no log line; the output lines. async function planOnly(e,code,...args){ const before=fp(e),data=tree(e.data),n=calls(e).length,logs=logLines(e).length; const r=await prun(e,...args,'--plan');assert.equal(r.code,code,r.out+r.err);assert.equal(r.err,''); assert.deepEqual(fp(e),before);assert.deepEqual(tree(e.data),data);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); return r.out.split('\n'); } const proofLines=(out)=>out.filter((l)=>l.startsWith('proof:')); const CARRY_MSG=(to)=>`${to}'s Console does not support --proof-header, which the pointer's Console flags carry; give --no-proof to drop the pair`; const HEADER_MSG=(h)=>`--proof-header ${h} must be a header name of a-z, 0-9 and - (64 at most, lowercased) other than host, origin, x-mosaic-tester and the --auth-header value`; test('T53 row 187: activate sets, carries and drops the proof pair',async(t)=>{ await t.test('tests 1 and 9: the pair goes after the preview flags, before --send and the kept --tester-port, as the unit renders it; a new pair replaces it',async(t)=>{ const e=await mk(t),base=await seedPreview(e,undefined,['--send']),f=await proofFile(e),g=await proofFile(e,'proof2.187',0o400); const pre=base.slice(0,-1); let r=await prun(e,'activate',C1,'acme','--tester-port','7333');assert.equal(r.code,0,r.out+r.err); const out=await planOnly(e,0,'activate',C1,'acme',...pair(f)); assert.deepEqual(proofLines(out),[`proof: on, header x-console-proof, file ${f}`]); assert.ok(out.find((l)=>l.startsWith('Console: ')).endsWith(` ${PREVIEW.join(' ')} ${pair(f).join(' ')} --send --tester-port 7333`)); r=await prun(e,'activate',C1,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,[...pre,...pair(f),'--send','--tester-port','7333']); assert.ok(conExec(e).endsWith(` --checkout ${repo} ${[...pre,...pair(f),'--send','--tester-port','7333'].join(' ')}`),conExec(e)); r=await prun(e,'status','acme');assert.equal(r.code,0,r.err);assert.match(r.out,/unit flags: match the pointer/); r=await prun(e,'activate',C1,'acme',...pair(g,'X-Proof-Two'));assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,[...pre,...pair(g,'X-Proof-Two'),'--send','--tester-port','7333']); r=await prun(e,'status','acme');assert.equal(r.code,0,r.err);assert.match(r.out,/unit flags: match the pointer/); assert.equal(conExec(e).match(/--proof-header/g).length,1);assert.ok(!conExec(e).includes(f)); await noLeak(e); }); await t.test('tests 2 and 3: an activate of the live copy writes the pair; a later activate and a rollback carry it',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); let r=await prun(e,'activate',B,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,B);assert.deepEqual(pointer(e).console,[...base,...pair(f)]);assert.ok(conExec(e).endsWith(pair(f).join(' '))); assert.deepEqual(proofLines(await planOnly(e,0,'activate',C1,'acme')),['proof: carried, header x-console-proof']); r=await prun(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,C1);assert.deepEqual(pointer(e).console,[...base,...pair(f)]);assert.ok(conExec(e).endsWith(pair(f).join(' '))); assert.deepEqual(proofLines(await planOnly(e,0,'rollback','acme')),['proof: carried, header x-console-proof']); r=await prun(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,B);assert.deepEqual(pointer(e).console,[...base,...pair(f)]);assert.ok(conExec(e).endsWith(pair(f).join(' '))); await noLeak(e); }); await t.test('test 2: only a pair change may name the live copy; the log keeps from and to, and rollback still goes to the earlier copy',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); await refused(e,['activate',B,'acme'],'already-active',`${B} is the pointer's copy`); await refused(e,['activate',B,'acme','--no-proof'],'already-active',`${B} is the pointer's copy`); await refused(e,['activate',B,'acme','--tester-port','7333'],'already-active',`${B} is the pointer's copy`); let r=await prun(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); r=await prun(e,'activate',C1,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); let l=logLines(e).at(-1);assert.deepEqual([l.op,l.from,l.to,l.result],['activate',C1,C1,'ok']);assert.ok(l.backup); assert.deepEqual(pointer(e).console,[...base,...pair(f)]); await refused(e,['activate',C1,'acme'],'already-active',`${C1} is the pointer's copy`); const out=await planOnly(e,0,'activate',C1,'acme','--no-proof'); assert.deepEqual(proofLines(out),['proof: dropped; --no-proof drops header x-console-proof']); assert.ok(out.includes('commits: the run adds 0 (newest first, at most 200):'),out.join('\n')); r=await prun(e,'activate',C1,'acme','--no-proof');assert.equal(r.code,0,r.out+r.err); l=logLines(e).at(-1);assert.deepEqual([l.from,l.to,l.result],[C1,C1,'ok']); assert.equal(pointer(e).copy,C1);assert.deepEqual(pointer(e).console,base);assert.doesNotMatch(conExec(e),/--proof-/); r=await prun(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err);assert.equal(pointer(e).copy,B); await noLeak(e); }); await t.test('test 2: a live-copy activate says only the Console flags change, and backs up the stopped store before it installs',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); const NOTE=`note: ${B} is the pointer's copy; the run re-pins it and only the Console flags change`; for(const args of [pair(f),['--no-proof']]){ const out=await planOnly(e,0,'activate',B,'acme',...args); assert.deepEqual(out.filter((l)=>l.startsWith('note: ')),[NOTE]);assert.equal(out[out.indexOf(`to: ${B}`)+1],NOTE); assert.deepEqual(out.filter((l)=>l.startsWith('backup')),[planBackup(e)]); const sum=createHash('sha256').update(readFileSync(e.store)).digest('hex'),n=calls(e).length; const r=await prun(e,'activate',B,'acme',...args);assert.equal(r.code,0,r.out+r.err); const act=pointer(e).activation,l=logLines(e).at(-1); assert.deepEqual([l.from,l.to,l.result,l.backup],[B,B,'ok',act]);assert.equal(backupSum(e,act),sum); assert.deepEqual(startsStops(e,n),[`stop ${CON}`,`stop ${BUS}`,`start ${BUS}`,`start ${CON}`]); const steps=stepText(e),at=(x)=>steps.indexOf(x); assert.ok(at('stopped both units')>=0&&at('stopped both units')l.startsWith('note: ')),[]); await noLeak(e); }); await t.test('test 3: pm-on keeps the pair',async(t)=>{ const e=await mk(t),base=await seedPreview(e,{pm:true}),f=await proofFile(e); let r=await prun(e,'activate',C1,'acme','--pm-after-harness',...pair(f));assert.equal(r.code,0,r.out+r.err); biz(e,HARNESS); r=await prun(e,'pm-on','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).pm,true);assert.deepEqual(pointer(e).console,[...base,...pair(f)]);assert.ok(conExec(e).endsWith(pair(f).join(' '))); await noLeak(e); }); await t.test('test 4: --no-proof drops the pair from the pointer and the unit and says so; with no pair it says so and passes',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); let r=await prun(e,'activate',B,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); assert.deepEqual(proofLines(await planOnly(e,0,'activate',C1,'acme','--no-proof')),['proof: dropped; --no-proof drops header x-console-proof']); r=await prun(e,'activate',C1,'acme','--no-proof');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,base);assert.doesNotMatch(conExec(e),/--proof-/); assert.match(stepText(e),new RegExp(`drop --proof-header x-console-proof --proof-file ${f}; --no-proof was given\\n`)); r=await prun(e,'status','acme');assert.equal(r.code,0,r.err);assert.match(r.out,/unit flags: match the pointer/); assert.deepEqual(proofLines(await planOnly(e,0,'activate',B,'acme','--no-proof')),['proof: off; --no-proof was given and the pointer holds no pair']); r=await prun(e,'activate',B,'acme','--no-proof');assert.equal(r.code,0,r.out+r.err); assert.deepEqual(pointer(e).console,base);assert.doesNotMatch(conExec(e),/--proof-/); assert.match(stepText(e),/--no-proof was given and the pointer holds no proof pair, so there is nothing to drop\n/); assert.deepEqual(proofLines(await planOnly(e,0,'activate',C1,'acme')),['proof: off']); await noLeak(e); }); await t.test('test 5: each refusal exits 3 before anything stops',async(t)=>{ const e=await mk(t);await seedPreview(e); const f=await proofFile(e),open=await proofFile(e,'open.187',0o644),loose=await proofFile(e,'loose.187',0o640); const link=join(e.root,'link.187');symlinkSync(f,link); const dir=join(e.root,'dir.187');mkdirSync(dir,{mode:0o700}); const odd=join(e.root,'odd 187');writeFileSync(odd,SENTINEL+'\n',{mode:0o600}); await refused(e,['activate',A,'acme',...pair(f)],'proof-target',`${A}'s Console does not support --proof-header`); for(const h of ['Host','origin','x-mosaic-tester','X-User','-lead','under_score','a'.repeat(65)]){ await refused(e,['activate',C1,'acme',...pair(f,h)],'proof-header-invalid',HEADER_MSG(h)); } const inCopies=join(COPIES,'proof-r187.187');writeFileSync(inCopies,SENTINEL+'\n',{mode:0o600});t.after(()=>rmSync(inCopies,{force:true})); await refused(e,['activate',C1,'acme',...pair(inCopies)],'proof-file-invalid',`--proof-file ${inCopies} is inside the repository or the copies`); rmSync(inCopies); for(const [p,why] of [['proof.187','is not an absolute path'],[odd,'holds a character outside [A-Za-z0-9/._+-]'],[link,'is a symlink'],[dir,'is not a regular file'],[join(e.root,'none.187'),'cannot be found'],[open,'has mode 644, not 600 or 400'],[loose,'has mode 640, not 600 or 400']]){ await refused(e,['activate',C1,'acme',...pair(p)],'proof-file-invalid',`--proof-file ${p} ${why}`); } const g=await mk(t);await seed(g);const h=await proofFile(g); await refused(g,['activate',C1,'acme',...pair(h)],'proof-needs-preview','the proof pair needs --public-host, --auth-header and --allow-user in the Console flags; the run\'s flags lack --public-host --auth-header --allow-user'); for(const x of [e,g])await noLeak(x); }); await t.test('tests 5 and 9: --plan lists every proof refusal with the others and writes nothing',async(t)=>{ const e=await mk(t);await seed(e);biz(e,HARNESS);const open=await proofFile(e,'open.187',0o644); const out=await planOnly(e,3,'activate',A,'acme',...pair(open,'host')); assert.deepEqual(proofLines(out),[`proof: on, header host, file ${open}`]); assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[ `refuses proof-target: ${A}'s Console does not support --proof-header`, 'refuses proof-needs-preview: the proof pair needs --public-host, --auth-header and --allow-user in the Console flags; the run\'s flags lack --public-host --auth-header --allow-user', `refuses proof-header-invalid: ${HEADER_MSG('host')}`, `refuses proof-file-invalid: --proof-file ${open} has mode 644, not 600 or 400`, `refuses unknown-kind-for-target: the business file names plug-in kind(s) harness, which ${A} does not know`]); await noLeak(e); }); await t.test('test 6: usage errors',async(t)=>{ const e=await mk(t);await seedPreview(e);const f=await proofFile(e); const usageText=(await prun(e,'--help')).err,n=calls(e).length,logs=logLines(e).length,before=fp(e); for(const args of [['activate',C1,'acme','--proof-header','x-console-proof'],['activate',C1,'acme','--proof-file',f],['activate',C1,'acme','--proof-header','','--proof-file',f], ['activate',C1,'acme','--no-proof',...pair(f)],['activate',C1,'acme',...pair(f),'--no-proof'],['activate',C1,'acme','--no-proof','--proof-header','x-console-proof'], ['rollback','acme',...pair(f)],['rollback','acme','--no-proof'],['rollback','acme','--proof-header','x-console-proof'],['pm-on','acme',...pair(f)],['pm-on','acme','--no-proof'],['pm-on','acme','--proof-file',f], ['activate',C1,'acme','--plan','--proof-file',f]]){ const r=await prun(e,...args);assert.equal(r.code,4,args.join(' '));assert.equal(r.out,'');assert.equal(r.err,usageText); } assert.match(usageText,/\[--proof-header NAME --proof-file PATH \| --no-proof\]/); assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); await noLeak(e); }); await t.test('test 7: a rollback to a copy without proof support drops the pair and says so',async(t)=>{ const e=await mk(t),base=await seedPreview(e,{copy:A}),f=await proofFile(e); let r=await prun(e,'activate',B,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); const out=await planOnly(e,0,'rollback','acme'); assert.deepEqual(proofLines(out),[`proof: dropped; ${A}'s Console does not support --proof-header, so the rollback drops header x-console-proof`]); assert.doesNotMatch(out.find((l)=>l.startsWith('Console: ')),/--proof-/); r=await prun(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,A);assert.deepEqual(pointer(e).console,base);assert.doesNotMatch(conExec(e),/--proof-/); assert.match(stepText(e),new RegExp(`drop --proof-header x-console-proof --proof-file ${f}; the target Console does not support it\\n`)); await noLeak(e); }); await t.test('test 7: an activate to a copy without proof support refuses proof-target without --no-proof and passes with it',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); let r=await prun(e,'activate',B,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); const out=await planOnly(e,3,'activate',A,'acme'); assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[`refuses proof-target: ${CARRY_MSG(A)}`]); await refused(e,['activate',A,'acme'],'proof-target',CARRY_MSG(A)); assert.deepEqual(proofLines(await planOnly(e,0,'activate',A,'acme','--no-proof')),['proof: dropped; --no-proof drops header x-console-proof']); r=await prun(e,'activate',A,'acme','--no-proof');assert.equal(r.code,0,r.out+r.err); assert.equal(pointer(e).copy,A);assert.deepEqual(pointer(e).console,base);assert.doesNotMatch(conExec(e),/--proof-/); assert.match(stepText(e),/drop --proof-header x-console-proof [^\n]*; --no-proof was given\n/); await noLeak(e); }); }); // Row 188 (#1686, P19 409 and 410): the checks on a carried pair, and a proof file inside REPO. // A file gone from its path keeps a hard link under e.root, so the inode and its read watch // outlive the path: inotify sends IN_IGNORED, which the watch would count, when a watched inode goes. const carried=['proof: carried, header x-console-proof']; test('T54 row 188: a carried pair gets the proof file checks, and a proof file inside the repository refuses',async(t)=>{ const changes=[ ['chmod 644',(e,f)=>{chmodSync(f,0o644);return 'has mode 644, not 600 or 400'}], ['a symlink to a mode-600 file',async(e,f)=>{const g=await proofFile(e,'swap.188');linkSync(f,join(e.root,'kept.188'));rmSync(f);symlinkSync(g,f);return 'is a symlink'}], ['removal',(e,f)=>{linkSync(f,join(e.root,'kept.188'));rmSync(f);return 'cannot be found'}], ]; for(const [name,change] of changes){ await t.test(`test 1: an activate that carries the pair refuses after ${name}, in the plan and the run, before anything stops`,async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); const r=await prun(e,'activate',B,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); const why=`--proof-file ${f} ${await change(e,f)}`; const out=await planOnly(e,3,'activate',C1,'acme'); assert.deepEqual(proofLines(out),carried);assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[`refuses proof-file-invalid: ${why}`]); await refused(e,['activate',C1,'acme'],'proof-file-invalid',why); assert.equal(pointer(e).copy,B);assert.deepEqual(pointer(e).console,[...base,...pair(f)]); await noLeak(e); }); } await t.test('test 2: a rollback that carries the pair refuses after chmod 644, in the plan and the run, before anything stops',async(t)=>{ const e=await mk(t),base=await seedPreview(e),f=await proofFile(e); const r=await prun(e,'activate',C1,'acme',...pair(f));assert.equal(r.code,0,r.out+r.err); chmodSync(f,0o644);const why=`--proof-file ${f} has mode 644, not 600 or 400`; const out=await planOnly(e,3,'rollback','acme'); assert.ok(out.includes(`to: ${B}`),out.join('\n')); assert.deepEqual(proofLines(out),carried);assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[`refuses proof-file-invalid: ${why}`]); await refused(e,['rollback','acme'],'proof-file-invalid',why); assert.equal(pointer(e).copy,C1);assert.deepEqual(pointer(e).console,[...base,...pair(f)]); await noLeak(e); }); // Tests 3 and 4: a mode-600 file of this uid, so only the realpath check can refuse it. const inRepo=async(t,e,dir)=>{t.after(()=>rmSync(dir,{recursive:true,force:true}));mkdirSync(dir);return proofFile(e,'proof.188',0o600,dir)}; await t.test('test 3: a proof file inside the repository refuses, in the plan and the run, before anything stops',async(t)=>{ const e=await mk(t);await seedPreview(e);const f=await inRepo(t,e,join(repo,'proof-r188')); const why=`--proof-file ${f} is inside the repository or the copies`; const out=await planOnly(e,3,'activate',C1,'acme',...pair(f)); assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[`refuses proof-file-invalid: ${why}`]); await refused(e,['activate',C1,'acme',...pair(f)],'proof-file-invalid',why); await noLeak(e); }); await t.test('test 4: a path outside the repository through a directory symlink into it refuses on its realpath',async(t)=>{ const e=await mk(t);await seedPreview(e);await inRepo(t,e,join(repo,'proof-r188')); const link=join(e.root,'repo-link.188');symlinkSync(join(repo,'proof-r188'),link); const f=join(link,'proof.188');assert.ok(!lstatSync(f).isSymbolicLink());assert.ok(!f.startsWith(`${repo}/`)); const why=`--proof-file ${f} is inside the repository or the copies`; const out=await planOnly(e,3,'activate',C1,'acme',...pair(f)); assert.deepEqual(out.filter((l)=>l.startsWith('refuses ')),[`refuses proof-file-invalid: ${why}`]); await refused(e,['activate',C1,'acme',...pair(f)],'proof-file-invalid',why); await noLeak(e); }); }); // --- rehearse (D2, row 163; the brief's "Stack rehearsal on a scratch root") --- // Copies of their own in RCOPIES: the real scripts/mosaic, config loader, business loader, roles and // mosaic-config.mjs, with stand-ins for the bus host (cli.mjs), the Console, the Store and // migrate-schema.mjs. A rehearsal starts its processes with only HOME, PATH and MOSAIC_CONFIG, so // the stand-ins take their faults from $HOME/faults and record what they did in $HOME. // The bus host. bus start writes the store's writer.lock, a meta row in the store, and its state // beside the scratch store, and prints the host-up line; without --no-launch it starts the PM, which here means a line in // $HOME/starts.log and a sessions.json. bus status --json answers from that state. function rHost(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'); const id=path.basename(path.resolve(import.meta.dirname,'..','..','..')),home=process.env.HOME,a=process.argv.slice(2); const fault=(n)=>fs.existsSync(path.join(home,'faults',n)); const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}}; const data=JSON.parse(fs.readFileSync(process.env.MOSAIC_CONFIG,'utf8')).dataRoot,state=path.join(data,'fixture-host.json'); const flags=["--pm","--no-launch"]; if(a.join(' ')==='bus status --json'){ if(fault(`status-${id}`))process.exit(1); let s={host:null,boot:null};try{s=JSON.parse(fs.readFileSync(state,'utf8'));s.host.live=alive(s.host.pid)}catch{} if(fault('status-other-host')&&s.host)s.host.pid=process.ppid; console.log(JSON.stringify(s));process.exit(0); } if(a[0]!=='bus'||a[1]!=='start'||!a.slice(3).every((f)=>flags.includes(f))){console.error(`fixture host: ${a.join(' ')}`);process.exit(4)} fs.appendFileSync(path.join(home,'host-args.log'),`${a.join(' ')}\n`); if(fault(`host-fails-${id}`)){console.error('fixture host: refused');process.exit(1)} const b=a[2],off=a.includes('--no-launch'),lock=path.join(data,'bus','writer.lock'),at=new Date().toISOString(); fs.writeFileSync(lock,String(process.pid),{flag:'wx'}); const db=new (process.getBuiltinModule('node:sqlite').DatabaseSync)(path.join(data,'bus','bus.sqlite')); db.prepare("INSERT OR REPLACE INTO meta(key,value) VALUES('fixture_host_start',?)").run(at);db.close(); fs.writeFileSync(state,JSON.stringify({host:{business:b,pid:process.pid,live:true,startedAt:at,notifier:'off'},boot:{ok:true,business:b,at}})); if(fault('host-launches-dir'))fs.mkdirSync(path.join(data,'launches')); // host-touches lists files a careless host would write: the live config, business file or backup. if(fault('host-touches'))for(const f of fs.readFileSync(path.join(home,'faults','host-touches'),'utf8').split('\n').filter(Boolean))fs.appendFileSync(f,'x'); if(!off||fault('host-launches')){ fs.appendFileSync(path.join(home,'starts.log'),`pm-${b}\n`); fs.mkdirSync(path.join(data,'bus-host'),{recursive:true}); fs.writeFileSync(path.join(data,'bus-host','sessions.json'),JSON.stringify({sessionsVersion:1,sessions:[{pid:process.pid}]})); } console.log(`bus host up: business ${b}, socket ${path.join(data,'bus','broker.sock')}, notifier off${off?', launches disabled':''}`); process.on('SIGTERM',()=>{if(fault('host-ignores-term'))return;if(!fault('host-keeps-lock'))fs.rmSync(lock,{force:true});process.exit(0)}); setInterval(()=>{},1<<30); } // The Console: serve on the given loopback port; /healthz answers 200 unless healthz- is set. function rConsole(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),http=process.getBuiltinModule('node:http'); const id=path.basename(path.resolve(import.meta.dirname,'..','..','..')),home=process.env.HOME,a=process.argv.slice(2); const fault=(n)=>fs.existsSync(path.join(home,'faults',n)); fs.appendFileSync(path.join(home,'console-args.log'),`${a.join(' ')}\n`); if(a[0]!=='serve'){console.error('fixture Console: refused');process.exit(1)} const s=http.createServer((q,r)=>{r.writeHead(q.url==='/healthz'&&!fault(`healthz-${id}`)?200:500);r.end()}); s.listen(Number(a[a.indexOf('--port')+1]),'127.0.0.1',()=>console.log(`Mosaic Console: http://127.0.0.1:${s.address().port}/\nBoard: fixture`)); process.on('SIGTERM',()=>process.exit(0)); } // migrate-schema.mjs: sets its copy's SCHEMA_VERSION, or exit 1 under migrate-fails. function rMigrate(){ const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite'); const home=process.env.HOME,data=process.argv[2]; const v=/^export const SCHEMA_VERSION = '([A-Za-z0-9]+)';$/m.exec(fs.readFileSync(path.join(import.meta.dirname,'store.mjs'),'utf8'))[1]; fs.appendFileSync(path.join(home,'migrate.log'),`${data}\n`); if(fs.existsSync(path.join(home,'faults','migrate-fails'))){console.error('fixture migrate: failed');process.exit(1)} const db=new DatabaseSync(path.join(data,'bus','bus.sqlite'));db.prepare("UPDATE meta SET value=? WHERE key='schema_version'").run(v);db.close(); console.log(`bus schema ${v} ready`); } // The copy's Store: refuses a store at another version with schema-mismatch, as the real one does. const rStore=(v,froms)=>`${storeText(v,froms)}import {DatabaseSync} from 'node:sqlite'; import {join} from 'node:path'; export class Store { constructor(root) { this.db = new DatabaseSync(join(root, 'bus', 'bus.sqlite')); if (this.db.prepare("SELECT value FROM meta WHERE key='schema_version'").get()?.value !== SCHEMA_VERSION) { this.db.close(); throw Error('schema-mismatch'); } } get(sql, ...args) { return this.db.prepare(sql).get(...args); } close() { this.db.close(); } } `; // B (3g) and C (3h) have --no-launch. A is a copy from before row 163, whose host has no such flag. // C1's scripts/mosaic runs B's host, as a copy-check failure needs. const RCOPIES=join(base,'rcopies');mkdirSync(RCOPIES); for(const [id,v,froms] of [[B,'3g',['3f']],[C,'3h',['3f','3g']],[A,'3g',['3f']],[C1,'3g',['3f']]]){ const d=join(RCOPIES,id); for(const p of ['scripts/mosaic','scripts/mosaic-config.mjs','packages/cli/src/config.mjs','packages/cli/src/errors.mjs','packages/bus/src/business.mjs','packages/plugins/src/kinds.mjs','packages/business/src','roles'])cpSync(join(repo,p),join(d,p),{recursive:true}); if(id===C1)writeFileSync(join(d,'scripts','mosaic'),`#!/usr/bin/env bash\nexec node "${join(RCOPIES,B,'packages','cli','src','cli.mjs')}" "$@"\n`,{mode:0o755}); for(const [f,text] of [['packages/cli/src/cli.mjs',id===A?'// fixture host from before row 163\nprocess.exit(1);\n':script(rHost)],['packages/webui/src/cli.mjs',script(rConsole)],['packages/bus/src/store.mjs',rStore(v,froms)],['packages/bus/src/migrate-schema.mjs',script(rMigrate)],['packages/bus/src/broker.mjs','export class BusError extends Error { constructor(code) { super(code); this.code = code; } }\n']]){mkdirSync(dirname(join(d,f)),{recursive:true});writeFileSync(join(d,f),text)} execFileSync('chmod',['-R','a-w',d]); } const TESTER='7499'; const freePort=async()=>{const s=createServer();s.listen(0,'127.0.0.1');await once(s,'listening');const p=String(s.address().port);await new Promise((r)=>s.close(r));return p}; const rh=(e,...p)=>join(e.env.HOME,...p); const rfault=(e,name)=>{mkdirSync(rh(e,'faults'),{recursive:true});writeFileSync(rh(e,'faults',name),'')}; // A root for one rehearsal: a stopped live store at `store`, a pointer by hand on `copy` with the // Console on e.port and the tester listener on TESTER, and a backup as backup_store writes one. // A rehearsal's leftovers, found by their scratch MOSAIC_CONFIG, are killed after the test. async function rmk(t,{copy=B,store='3g'}={}){ const e=await mk(t); t.after(()=>{ for(const pid of readdirSync('/proc').filter((n)=>/^\d+$/.test(n))) try{if(readFileSync(`/proc/${pid}/environ`,'utf8').includes(`MOSAIC_CONFIG=${e.data}/rehearsals/`))process.kill(Number(pid),'SIGKILL')}catch{} }); const db=new DatabaseSync(e.store); db.exec('PRAGMA journal_mode=WAL'); db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)'); db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)'); db.prepare("INSERT INTO meta VALUES ('schema_version', ?)").run(store); for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`); db.close(); mkdirSync(e.state,{recursive:true}); writeFileSync(e.pointer,JSON.stringify({copy,business:'acme',pm:true,claude:e.claude,console:['--port',e.port,'--business','acme','--tester-port',TESTER],schema:store,activation:'manual',at:'2026-10-20T00:00:00Z'})+'\n'); e.act=`20261020T000000Z-${copy.slice(0,12)}`;e.backup=join(e.data,'backups','stack',e.act); mkdirSync(join(e.backup,'bus'),{recursive:true,mode:0o700}); cpSync(e.store,join(e.backup,'bus','bus.sqlite')); writeFileSync(join(e.backup,'SHA256SUMS'),`${sha(e.store)} bus.sqlite\n`); for(const [f,text] of [['DATA_ROOT',e.data],['SCHEMA',store],['STAMP','2026-10-20T00:00:00.000Z']])writeFileSync(join(e.backup,f),`${text}\n`); e.biz=join(dirname(e.cfg),'businesses','acme.json'); return e; } // What a rehearsal may never change: the live store, config, business file, pointer and log, and // the backup, by sha256 of every file. const untouched=(e)=>JSON.stringify([...[e.store,e.cfg,e.biz,e.pointer].map((f)=>[f,sha(f)]),existsSync(e.log),...readdirSync(e.backup,{recursive:true}).sort().map((f)=>join(e.backup,f)).filter((f)=>statSync(f).isFile()).map((f)=>[f,sha(f)])]); async function rehearse(e,id,...args){ const ch=spawn('bash',[e.sr??SR,'rehearse',id,'acme',...args,'--copies',RCOPIES,'--bin',e.bin],{env:e.env,stdio:['ignore','pipe','pipe']}); let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d}); e.during?.(ch); const timer=setTimeout(()=>ch.kill('SIGKILL'),120000); const [code]=await once(ch,'close');clearTimeout(timer); const m=/\nrehearsal directory: (.*)\n$/.exec(out); return {code,out,err,dir:m?m[1]:null,names:out.split('\n').filter((l)=>/^(PASS|FAIL) /.test(l)).map((l)=>l.replace(/^(PASS|FAIL) ([^:]*).*/,'$1 $2'))}; } // A script of its own whose text has `from` replaced, for the guard tests. The replacement is a // function so a shell "$$" in `to` stays as written. function ownScript(e,from,to){ const own=join(e.root,'repo');cpSync(repo,own,{recursive:true}); e.sr=join(own,'scripts','stack-release.sh'); const s=readFileSync(e.sr,'utf8');assert.ok(s.includes(from),`${e.sr} has no ${from}`);writeFileSync(e.sr,s.replace(from,()=>to)); } const rDirs=(e)=>existsSync(join(e.data,'rehearsals'))?readdirSync(join(e.data,'rehearsals')):[]; const PRE=['PASS copy','PASS port','PASS backup','PASS scratch-root','PASS data-root-live','PASS sessions-live','PASS data-root-scratch','PASS notify-off','PASS tracker-configured','PASS credential-ref-live']; const POST=['PASS console-stopped','PASS host-stopped','PASS host-released-store','PASS backup-unchanged','PASS live-config-unchanged']; test('R1 rehearse passes on the same schema; the scratch root holds its files and the live files stay byte-identical',async(t)=>{ const e=await rmk(t),before=untouched(e),port=await freePort(); const r=await rehearse(e,B,'--backup',e.act,'--port',port,'--timeout','10'); assert.equal(r.code,0,r.out+r.err); assert.deepEqual(r.names,[...PRE,'PASS migration','PASS gate','PASS no-session',...POST]); assert.ok(r.out.endsWith(`REHEARSAL_PASS\nrehearsal directory: ${r.dir}\n`),r.out); assert.equal(r.err,''); assert.equal(untouched(e),before); const R=r.dir;assert.equal(dirname(R),join(e.data,'rehearsals'));assert.match(basename(R),new RegExp(`^\\d{8}T\\d{6}Z-${B.slice(0,12)}$`)); for(const d of [dirname(R),R,join(R,'placeholders'),join(R,'config')])assert.equal(statSync(d).mode&0o777,0o700,d); for(const f of ['steps.log','backup.before','live.before','business.diff','host.log','console.log','config/config.json','data/bus/bus.sqlite'])assert.ok(existsSync(join(R,f)),f); // The host wrote the scratch store, a copy: the backup has no such row. const metaOf=(f)=>{const db=new DatabaseSync(f,{readOnly:true});try{return db.prepare("SELECT value FROM meta WHERE key='fixture_host_start'").get()?.value}finally{db.close()}}; assert.match(metaOf(join(R,'data','bus','bus.sqlite')),/^\d{4}-/);assert.equal(metaOf(join(e.backup,'bus','bus.sqlite')),undefined); // The scratch config differs from the live one only in dataRoot. assert.deepEqual(JSON.parse(readFileSync(join(R,'config','config.json'),'utf8')),{...JSON.parse(readFileSync(e.cfg,'utf8')),dataRoot:join(R,'data')}); // The notifier config: binding null at 0600. const nf=join(R,'data','notify','acme','notify.json'); assert.deepEqual(JSON.parse(readFileSync(nf,'utf8')),{notifyVersion:1,binding:null});assert.equal(statSync(nf).mode&0o777,0o600); // The scratch business: no tracker.* variable, and every reference a 0600 placeholder with its date kept. const live=JSON.parse(readFileSync(e.biz,'utf8')),sb=JSON.parse(readFileSync(join(R,'config','businesses','acme.json'),'utf8')); assert.ok(Object.keys(live.vars).some((k)=>k.startsWith('tracker.'))); assert.deepEqual(Object.keys(sb.vars).filter((k)=>k.startsWith('tracker.')),[]); const refs=[...Object.entries(sb.roles).flatMap(([n,r])=>Object.entries(r.credentials??{}).map(([s,ref])=>[`${n}-${s}`,ref,live.roles[n].credentials[s]])),...Object.entries(sb.tracker.sync.credentials).map(([s,ref])=>[`@sync-${s}`,ref,live.tracker.sync.credentials[s]])]; assert.ok(refs.length>=2); for(const [name,ref,old] of refs){ const f=join(R,'placeholders',name); assert.deepEqual(ref,{...old,file:f});assert.equal(readFileSync(f,'utf8'),'rehearsal-placeholder-token');assert.equal(statSync(f).mode&0o777,0o600); } assert.deepEqual(readdirSync(join(R,'placeholders')).sort(),refs.map(([n])=>n).sort()); // Everything else in the business is as it was. const strip=(b)=>{const c=structuredClone(b);for(const k of Object.keys(c.vars))if(k.startsWith('tracker.'))delete c.vars[k];for(const r of Object.values(c.roles))delete r.credentials;delete c.tracker.sync.credentials;return c}; assert.deepEqual(strip(sb),strip(live)); const diff=readFileSync(join(R,'business.diff'),'utf8');assert.match(diff,/^-.*"tracker\.baseUrl"/m);assert.match(diff,/^\+.*placeholders\/pm-/m); // The host ran with launches disabled and started no PM; the Console got no --send. assert.deepEqual(lines(rh(e,'host-args.log')),['bus start acme --no-launch']); assert.deepEqual(lines(rh(e,'console-args.log')),[`serve --port ${port} --business acme`]); assert.equal(existsSync(rh(e,'starts.log')),false); assert.equal(existsSync(join(R,'data','bus','writer.lock')),false); assert.deepEqual(lines(join(R,'steps.log')).map((l)=>l.replace(/^\S+ /,'')),r.out.split('\n').filter((l)=>/^(PASS|FAIL) |^REHEARSAL_/.test(l))); }); test('R2 rehearse runs the migration checks when the schema differs; the live store stays at 3g',async(t)=>{ const e=await rmk(t),before=untouched(e),port=await freePort(); const r=await rehearse(e,C,'--backup',e.backup,'--port',port,'--timeout','10'); assert.equal(r.code,0,r.out+r.err); assert.deepEqual(r.names,[...PRE,'PASS new-refuses-old-store','PASS migrate','PASS new-opens-migrated','PASS migrate-again','PASS old-refuses-migrated','PASS gate','PASS no-session',...POST]); assert.ok(r.out.includes('PASS migrate: 3g to 3h\n'),r.out); assert.equal(untouched(e),before);assert.equal(storeVersion(e.store,false),'3g'); assert.equal(storeVersion(join(r.dir,'data','bus','bus.sqlite'),false),'3h'); assert.equal(storeVersion(join(r.dir,'refuse','bus','bus.sqlite'),false),'3g'); assert.deepEqual(lines(rh(e,'migrate.log')),[join(r.dir,'data'),join(r.dir,'data')]); }); test('R3 a failing migration gives REHEARSAL_FAIL migrate and starts nothing',async(t)=>{ const e=await rmk(t),before=untouched(e);rfault(e,'migrate-fails'); const r=await rehearse(e,C,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.deepEqual(r.names,[...PRE,'PASS new-refuses-old-store','FAIL migrate','PASS backup-unchanged','PASS live-config-unchanged']); assert.ok(r.out.includes('FAIL migrate: migrate-schema.mjs exit 1: fixture migrate: failed\n'),r.out); assert.ok(r.out.endsWith(`REHEARSAL_FAIL\nrehearsal directory: ${r.dir}\n`),r.out); assert.equal(existsSync(rh(e,'host-args.log')),false);assert.equal(existsSync(rh(e,'console-args.log')),false); assert.equal(untouched(e),before); }); test('R4 a host that does not start, does not answer status, reports another pid, launches anyway, makes a launches directory or keeps its lock each fails by name',async(t)=>{ for(const [f,name,why] of [[`host-fails-${B}`,'host-up','exited'],[`status-${B}`,'bus-status','bus status failed'],['status-other-host','bus-status','the host this run started'],['host-launches','no-session','bus-host/sessions.json'],['host-launches-dir','no-session','found launches under'],['host-keeps-lock','host-released-store','writer.lock is still there']]){ const e=await rmk(t),before=untouched(e);rfault(e,f); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,f+r.out+r.err); assert.ok(r.names.includes(`FAIL ${name}`),f+r.out);assert.equal(r.names.filter((n)=>n.startsWith('FAIL')).length,1,f+r.out); assert.ok(r.out.split('\n').some((l)=>l.startsWith(`FAIL ${name}: `)&&l.includes(why)),f+r.out); assert.ok(r.names.includes('PASS console-stopped')&&r.names.includes('PASS backup-unchanged'),f+r.out); assert.ok(r.out.endsWith(`REHEARSAL_FAIL\nrehearsal directory: ${r.dir}\n`),r.out); assert.equal(untouched(e),before); } }); test('R5 a host that ignores SIGTERM gets SIGKILL and FAIL host-stopped',async(t)=>{ const e=await rmk(t);rfault(e,'host-ignores-term'); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','2'); assert.equal(r.code,1,r.out+r.err); assert.ok(/^FAIL host-stopped: pid \d+ was still running 2s after SIGTERM; sent SIGKILL$/m.test(r.out),r.out); }); test('R6 a Console that does not answer /healthz gives REHEARSAL_FAIL healthz',async(t)=>{ const e=await rmk(t);rfault(e,`healthz-${B}`); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes('FAIL healthz: no answer within 3s\n'),r.out); assert.deepEqual(r.names.filter((n)=>n.startsWith('FAIL')),['FAIL healthz']); }); test('R7 a host on another copy fails copy-match',async(t)=>{ const e=await rmk(t); const r=await rehearse(e,C1,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes(`FAIL copy-match: the host's command line is not ${join(RCOPIES,C1)}'s cli.mjs bus start acme --no-launch\n`),r.out); }); test('R8 a scratch port in use or reserved refuses, and the rehearsal takes no other port',async(t)=>{ const e=await rmk(t),before=untouched(e); const held=createServer();held.listen(0,'127.0.0.1');await once(held,'listening');t.after(()=>held.close()); const p=String(held.address().port); const r=await rehearse(e,B,'--backup',e.act,'--port',p); assert.equal(r.code,1,r.out+r.err); assert.equal(r.out,`PASS copy: ${B}, schema 3g\nFAIL port-in-use: 127.0.0.1:${p} is in use; the rehearsal takes no other port\nREHEARSAL_FAIL\n`); for(const q of ['7332','7333',e.port,TESTER]){ const s=await rehearse(e,B,'--backup',e.act,'--port',q); assert.equal(s.code,1,s.out+s.err); assert.equal(s.out,`PASS copy: ${B}, schema 3g\nFAIL port-reserved: --port ${q} is a live Console port (7332, 7333 and the pointer's: ${e.port} ${TESTER})\nREHEARSAL_FAIL\n`); } assert.deepEqual(rDirs(e),[]);assert.equal(existsSync(rh(e,'host-args.log')),false);assert.equal(existsSync(rh(e,'console-args.log')),false); assert.equal(untouched(e),before); }); test('R9 mos-0963: a role reference left outside placeholders/, or left on an env variable, fails credential-ref-live before the host starts',async(t)=>{ const e=await rmk(t); // The live pm gitea reference names a file that is there, as a live token would be. const tok=join(e.root,'live-gitea.token');writeFileSync(tok,'live-token-value',{mode:0o600}); const b=JSON.parse(readFileSync(e.biz,'utf8'));b.roles.pm.credentials.gitea.file=tok;writeFileSync(e.biz,JSON.stringify(b,null,2)+'\n',{mode:0o600}); const before=untouched(e); ownScript(e,'for(const [name,r] of Object.entries(b.roles??{})){strip(r?.vars);swap(name,r?.credentials)}','for(const [name,r] of Object.entries(b.roles??{})){strip(r?.vars);if(name!=="pm")swap(name,r?.credentials)}'); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes(`FAIL credential-ref-live: pm gitea is ${tok}, not a file in ${join(r.dir,'placeholders')}\n`),r.out); assert.deepEqual(r.names.slice(-3),['FAIL credential-ref-live','PASS backup-unchanged','PASS live-config-unchanged']); assert.equal(existsSync(rh(e,'host-args.log')),false);assert.equal(existsSync(rh(e,'console-args.log')),false); assert.equal(untouched(e),before); // An env reference names no file at all; the guard says which variable it is. const f=await rmk(t); const fb=JSON.parse(readFileSync(f.biz,'utf8'));const {file,...rest}=fb.roles.pm.credentials.gitea;fb.roles.pm.credentials.gitea={...rest,env:'REHEARSAL_GITEA_TOKEN'};writeFileSync(f.biz,JSON.stringify(fb,null,2)+'\n',{mode:0o600}); ownScript(f,'for(const [name,r] of Object.entries(b.roles??{})){strip(r?.vars);swap(name,r?.credentials)}','for(const [name,r] of Object.entries(b.roles??{})){strip(r?.vars);if(name!=="pm")swap(name,r?.credentials)}'); const s=await rehearse(f,B,'--backup',f.act,'--port',await freePort(),'--timeout','3'); assert.equal(s.code,1,s.out+s.err); assert.ok(s.out.includes('FAIL credential-ref-live: pm gitea is env REHEARSAL_GITEA_TOKEN\n'),s.out); assert.equal(existsSync(rh(f,'host-args.log')),false); }); test('R10 mos-0962: a scratch business that still sets a tracker fails tracker-configured',async(t)=>{ const e=await rmk(t); const proj=join(e.root,'project');mkdirSync(join(proj,'.mosaic'),{recursive:true}); writeFileSync(join(proj,'.mosaic','project.json'),JSON.stringify({projectVersion:1,id:'stack',vars:{'tracker.project':7}}),{mode:0o600}); const b=JSON.parse(readFileSync(e.biz,'utf8'));b.projects.stack.root=proj;writeFileSync(e.biz,JSON.stringify(b,null,2)+'\n',{mode:0o600}); ownScript(e,'if(k.startsWith("tracker."))delete vars[k]','if(k.startsWith("tracker-kept."))delete vars[k]'); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes('FAIL tracker-configured: bootConfig sets trackers for acme\n'),r.out); assert.equal(existsSync(rh(e,'host-args.log')),false); }); test('R11 mos-0961: a scratch config on the live data root or on any root but the scratch one, or a sessions.json naming a running pid, refuses by name',async(t)=>{ // mosaic-config already refuses the live path itself, which contains the scratch config directory; // the guard catches a path that reaches the live data root through a link. const e=await rmk(t),link=join(e.root,'live-link');symlinkSync(e.data,link); ownScript(e,'c.dataRoot=data;',`c.dataRoot=${JSON.stringify(link)};`); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes(`FAIL data-root-live: the scratch config names dataRoot ${link}, which resolves to the live data root ${e.data}\n`),r.out); assert.equal(existsSync(rh(e,'host-args.log')),false); // A root that is neither live nor this rehearsal's, and one that is not there. for(const [dir,made,msg] of [['other-root',true,(d,R)=>`the scratch config resolves to ${d}, not ${join(R,'data')}`],['no-root',false,(d)=>`${d} does not exist`]]){ const g=await rmk(t),d=join(g.root,dir);if(made)mkdirSync(d); ownScript(g,'c.dataRoot=data;',`c.dataRoot=${JSON.stringify(d)};`); const q=await rehearse(g,B,'--backup',g.act,'--port',await freePort(),'--timeout','3'); assert.equal(q.code,1,dir+q.out+q.err); assert.ok(q.out.includes(`FAIL data-root-scratch: ${msg(d,q.dir)}\n`),dir+q.out); assert.equal(existsSync(rh(g,'host-args.log')),false); } const f=await rmk(t); ownScript(f,`printf '{"notifyVersion":1,"binding":null}\\n' > "$R/data/notify/$BUSINESS/notify.json"`,`printf '{"notifyVersion":1,"binding":null}\\n' > "$R/data/notify/$BUSINESS/notify.json" && mkdir "$R/data/bus-host" && printf '{"sessionsVersion":1,"sessions":[{"pid":%s}]}\\n' "$$" > "$R/data/bus-host/sessions.json"`); const s=await rehearse(f,B,'--backup',f.act,'--port',await freePort(),'--timeout','3'); assert.equal(s.code,1,s.out+s.err); assert.ok(/^FAIL sessions-live: \S+\/bus-host\/sessions\.json names running pid\(s\) \d+$/m.test(s.out),s.out); // The file is left as it was: the rehearsal never deletes it and retries. assert.ok(existsSync(join(s.dir,'data','bus-host','sessions.json'))); assert.equal(existsSync(rh(f,'host-args.log')),false); }); test('R12 mos-0962: a notifier config with a binding fails notify-off',async(t)=>{ const e=await rmk(t); ownScript(e,`printf '{"notifyVersion":1,"binding":null}\\n'`,`printf '{"notifyVersion":1,"binding":"discord"}\\n'`); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','3'); assert.equal(r.code,1,r.out+r.err); assert.ok(r.out.includes('FAIL notify-off: '),r.out); assert.equal(existsSync(rh(e,'host-args.log')),false); }); test('R13 the pre-checks: an old copy, a backup that fails its sums, a missing backup or pointer, and usage',async(t)=>{ const e=await rmk(t),before=untouched(e),port=await freePort(); const old=await rehearse(e,A,'--backup',e.act,'--port',port); assert.equal(old.code,1);assert.equal(old.out,`FAIL copy-without-no-launch: ${A}'s bus host has no --no-launch flag\nREHEARSAL_FAIL\n`); const none=await rehearse(e,B,'--backup','20261020T000000Z-000000000000','--port',port); assert.equal(none.code,1);assert.ok(none.out.includes(`FAIL backup-missing: ${join(e.data,'backups','stack','20261020T000000Z-000000000000')} has no bus/bus.sqlite and SHA256SUMS\n`),none.out); const f=await rmk(t);appendFileSync(join(f.backup,'bus','bus.sqlite'),'x'); const bad=await rehearse(f,B,'--backup',f.act,'--port',port); assert.equal(bad.code,1);assert.ok(bad.out.includes(`FAIL backup-sums: ${f.backup}/bus does not match its SHA256SUMS\n`),bad.out); const g=await rmk(t);rmSync(g.pointer); const np=await rehearse(g,B,'--backup',g.act,'--port',port); assert.equal(np.code,1);assert.match(np.out,/^FAIL no-pointer: /); for(const args of [['--port',port],['--backup',e.act],['--backup',e.act,'--port','80'],['--backup',e.act,'--port','70000'],['--backup','relative/dir','--port',port],['--backup',e.act,'--port',port,'--pm'],['--backup',e.act,'--port',port,'--plan']]){ const u=await rehearse(e,B,...args);assert.equal(u.code,4,args.join(' ')+u.out+u.err);assert.match(u.err,/^usage: /); } assert.deepEqual([...rDirs(e),...rDirs(f),...rDirs(g)],[]); assert.equal(untouched(e),before); }); test('R14 a host that writes the live config, the live business file or the backup fails the end check by name',async(t)=>{ for(const [pick,name] of [[(e)=>e.cfg,'live-config-unchanged'],[(e)=>e.biz,'live-config-unchanged'],[(e)=>join(e.backup,'bus','bus.sqlite'),'backup-unchanged']]){ const e=await rmk(t),f=pick(e);rfault(e,'host-touches');writeFileSync(rh(e,'faults','host-touches'),`${f}\n`); const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','10'); assert.equal(r.code,1,f+r.out+r.err); assert.deepEqual(r.names.filter((n)=>n.startsWith('FAIL')),[`FAIL ${name}`],f+r.out); assert.ok(r.names.includes('PASS gate')&&r.names.includes('PASS host-stopped'),f+r.out); assert.ok(r.out.endsWith(`REHEARSAL_FAIL\nrehearsal directory: ${r.dir}\n`),r.out); } }); test('R15 a rehearsal stopped by TERM, INT or HUP fails signal and still stops the host and Console and runs the end checks',async(t)=>{ for(const sig of ['SIGTERM','SIGINT','SIGHUP']){ // The Console never answers /healthz, so the gate is still waiting when the signal comes. const e=await rmk(t);rfault(e,`healthz-${B}`); e.during=async(ch)=>{for(let i=0;i<200&&!existsSync(rh(e,'console-args.log'));i++)await new Promise((r)=>setTimeout(r,50));ch.kill(sig)}; const r=await rehearse(e,B,'--backup',e.act,'--port',await freePort(),'--timeout','30'); assert.equal(r.code,1,sig+r.out+r.err); assert.deepEqual(r.names,[...PRE,'PASS migration','FAIL signal','PASS console-stopped','PASS host-stopped','PASS host-released-store','PASS backup-unchanged','PASS live-config-unchanged'],sig+r.out); assert.ok(r.out.includes('FAIL signal: the rehearsal was interrupted\n'),r.out); assert.ok(r.out.endsWith(`REHEARSAL_FAIL\nrehearsal directory: ${r.dir}\n`),r.out); const pid=JSON.parse(readFileSync(join(r.dir,'data','fixture-host.json'),'utf8')).host.pid; assert.equal(existsSync(`/proc/${pid}`),false,`${sig}: host pid ${pid} is still there`); } });