Files
stack/packages/cli/tests/stack-release.test.mjs
T
jason.woltjeandClaude Opus 5.5 7e7f8e208e scripts: health-gated stack activate and rollback, stack-release.sh (row 162, D1, #1659)
New scripts/stack-release.sh with four verbs. adopt records the running
copy, the unit flags and the store schema in state/stack-active.json; it
stops nothing. activate ID BUSINESS [--pm] [--migrate] stops the units,
checks the stopped store, backs it up, migrates when the target's schema
differs, installs and starts the target, then gates on both units active,
a live bus host whose pid is the bus unit's MainPID with boot ok, Console
/healthz, stack-copy.sh check and one launched journal line. A failed
gate rolls back inside the same run and keeps the attempted store as
bus.post-attempt-<stamp>. rollback [--restore ACT [--events N]] goes back
to the log's previous copy and counts the events written after the
backup. status prints the pointer, the log tail and the copy check.
--plan prints what a run would do and writes nothing; it reads the store
schema from the store the running bus holds, or from a stopped store.
Exit codes: refusal 3, gate failure with rollback 1, rollback failed 2,
usage 4. Nothing is pruned.

stack-copy.sh gets a reachable function split from copy_commit and a
guard so its helpers can be sourced. TOOLS.md "Pinned stack update" and
the cli README pinned-stack paragraph describe the verbs.

Owner Darkwing, brief docs/plans/2026-10-18_stack-release.md (D1).
Reviewer Filbert: r1 approve (comment 28997) on 091693c5, r2 confirm-only
approve (29004) on candidate manifest c213a801...bd15c after row 125
moved the base under TOOLS.md and the cli README. Sage gate on 23622980f:
25 of 25 steps rc=0 on c96a95320 (15 packages + 10 scripts), node pass 2353 fail 0, selftests 422 passed 0 failed, 2026-10-18T20:32:13Z to 20:38:55Z, network sage-r162_default removed, 0 containers.

Nothing live. The 8e7ea89a1 window (repin-8e7ea89a1.md) runs these verbs
on Mos's go with Jason present.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 15:39:09 -05:00

733 lines
51 KiB
JavaScript

// scripts/stack-release.sh (D1, #1659). Design, the T table and the mutants:
// agents/darkwing/work/queue-162/BUILD.md. Every run uses a temp data root, temp units and stand-in
// systemctl, journalctl and systemd-analyze on PATH; nothing here reads ~/.mosaic-dev or a live unit.
import test, {after} from 'node:test';
import assert from 'node:assert/strict';
import {execFileSync,spawn,spawnSync} from 'node:child_process';
import {mkdtempSync,mkdirSync,writeFileSync,readFileSync,appendFileSync,cpSync,rmSync,chmodSync,existsSync,statSync,readdirSync,realpathSync} from 'node:fs';
import {createServer} from 'node:http';
import {createHash} from 'node:crypto';
import {DatabaseSync} from 'node:sqlite';
import {tmpdir} from 'node:os';
import {join,dirname,basename} from 'node:path';
import {fileURLToPath,pathToFileURL} from 'node:url';
import {once} from 'node:events';
const ROOT=fileURLToPath(new URL('../../..',import.meta.url)).replace(/\/$/,'');
const NODE=process.execPath;
const MARKER='# mosaic-cli wrapper (Q81, agents/sage/work/q81/mosaic-cli.sh)';
const BUS='[email protected]',CON='mosaic-webui.service';
const git=(cwd,...args)=>execFileSync('git',['-c','user.name=fixture','-c','user.email=fixture@invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','-C',cwd,...args],{encoding:'utf8'}).trim();
const script=(fn)=>`#!${NODE}\n(${fn})();\n`;
// --- Stand-ins. Each is serialized into its own file, so it uses only process and its own scope. ---
// systemctl --user: unit state in $SD_STATE/units.json, every call in calls.log. A bus start runs
// holder.cjs (the store held open in WAL mode, writer.lock, host.json) and, with --pm in the unit's
// ExecStart, writes the `launched` journal line and the launches entry unless a pm-<copy> fault
// names another case. Faults are files in $SD_STATE/faults named for the copy they hit.
function systemctl(){
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process'),crypto=process.getBuiltinModule('node:crypto');
const sd=process.env.SD_STATE,data=process.env.SD_DATA,unitDir=path.join(process.env.XDG_CONFIG_HOME,'systemd','user');
const args=process.argv.slice(2).filter((a)=>a!=='--user'),[verb,...rest]=args;
fs.appendFileSync(path.join(sd,'calls.log'),args.join(' ')+'\n');
const faultFile=(n)=>path.join(sd,'faults',n),fault=(n)=>fs.existsSync(faultFile(n));
const stFile=path.join(sd,'units.json');
const load=()=>{try{return JSON.parse(fs.readFileSync(stFile,'utf8'))}catch{return {}}};
const save=(s)=>{fs.writeFileSync(stFile+'.tmp',JSON.stringify(s));fs.renameSync(stFile+'.tmp',stFile)};
const sleep=(ms)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms);
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}};
const isBus=(u)=>/^mosaic-bus@[a-z0-9-]+\.service$/.test(u),inst=(u)=>u.slice('mosaic-bus@'.length,-'.service'.length);
const unitFile=(u)=>{
const main=isBus(u)?'[email protected]':u,file=path.join(unitDir,main);
if(!fs.existsSync(file))return null;
const drops=[];
for(const d of isBus(u)?[main+'.d',u+'.d']:[u+'.d']){const dir=path.join(unitDir,d);if(fs.existsSync(dir))for(const f of fs.readdirSync(dir).sort())if(f.endsWith('.conf'))drops.push(path.join(dir,f))}
const r={wd:'',exec:'',env:[],drops};
for(const f of [file,...drops])for(let line of fs.readFileSync(f,'utf8').split('\n')){
line=line.replace(/%i/g,isBus(u)?inst(u):'').replace(/%h/g,process.env.HOME);
const m=/^(WorkingDirectory|ExecStart|Environment)=(.*)$/.exec(line);if(!m)continue;
if(m[1]==='WorkingDirectory')r.wd=m[2];else if(m[1]==='ExecStart'){if(m[2])r.exec=m[2]}else r.env.push(m[2]);
}
return r;
};
const snap=(ev)=>{
const dir=path.join(data,'bus'),files={};
try{for(const n of fs.readdirSync(dir).sort()){const p=path.join(dir,n),st=fs.statSync(p);if(st.isFile())files[n]=[st.size,st.mtimeMs,crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex')]}}catch{}
fs.appendFileSync(path.join(sd,'snap.jsonl'),JSON.stringify({ev,files})+'\n');
};
const pm=(b,id,pid,off)=>{
let mode='';try{mode=fs.readFileSync(faultFile(`pm-${id}`),'utf8').trim()}catch{}
const now=Date.now()-off*1000,epoch=Math.floor(now/1000),at=new Date(now).toISOString(),run=`run-${pid}`,unit=`mosaic-bus@${b}.service`;
const line=(text)=>fs.appendFileSync(path.join(sd,'journal.jsonl'),JSON.stringify({epoch,unit,text})+'\n');
const ok=()=>line(`launched pm-${b}: run ${run}, claude opus`);
const lf=path.join(sd,'launches.json');let list=[];try{list=JSON.parse(fs.readFileSync(lf,'utf8'))}catch{}
const entry=(o)=>list.push({run,business:b,live:true,startedAt:at,...o});
line(`bus host up: business ${b}, socket /fixture/bus.sock, notifier off`);
switch(mode){
case 'none':break;
case 'two':ok();line(`launched pm-${b}: run ${run}b, claude opus`);entry({});break;
case 'badword':line(`launched pm-${b}: job ${run}, claude opus`);entry({});break;
case 'notlisted':ok();break;
case 'early':ok();entry({startedAt:new Date(now-3600e3).toISOString()});break;
case 'human':entry({run:`human-${pid}`});break;
case 'other':ok();entry({business:'other'});break;
case 'dead':ok();entry({live:false});break;
default:ok();entry({});
}
fs.writeFileSync(lf,JSON.stringify(list));
};
if(verb==='show'){
const u=rest[0],props=[];for(let i=1;i<rest.length;i++)if(rest[i]==='-p')props.push(rest[++i]);
const f=unitFile(u),st=load()[u]||{},id=f?path.basename(f.wd):'';
for(const p of props){
if(p==='LoadState')console.log(`LoadState=${f?'loaded':'not-found'}`);
else if(p==='WorkingDirectory')console.log(`WorkingDirectory=${f?f.wd+(!isBus(u)&&fault(`check-${id}`)?'-moved':''):''}`);
else if(p==='ExecStart')console.log(f&&f.exec?`ExecStart={ path=${f.exec.split(' ')[0]} ; argv[]=${f.exec} ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }`:'ExecStart=');
else if(p==='Environment')console.log(`Environment=${f?f.env.join(' '):''}`);
else if(p==='MainPID')console.log(`MainPID=${st.pid||0}`);
else if(p==='DropInPaths')console.log(`DropInPaths=${f?f.drops.join(' '):''}`);
}
process.exit(0);
}
if(verb==='is-active'){const q=rest[0]==='--quiet',u=q?rest[1]:rest[0],s=(load()[u]||{}).state||'inactive';if(!q)console.log(s);process.exit(s==='active'?0:3)}
if(verb==='is-enabled'){console.log('disabled');process.exit(1)}
if(verb==='daemon-reload')process.exit(0);
if(verb==='list-units'){
if(fault('list-fails'))process.exit(1);
const pats=rest.filter((a)=>!a.startsWith('--')).map((p)=>new RegExp('^'+p.replace(/\./g,'\\.').replace(/\*/g,'.*')+'$'));
for(const [u,s] of Object.entries(load()))if(pats.some((re)=>re.test(u)))console.log(`${u} loaded ${s.state} ${s.state==='active'?'running':'dead'} fixture`);
process.exit(0);
}
if(verb==='start'){
const u=rest[0],f=unitFile(u);if(!f){console.error(`Unit ${u} not found.`);process.exit(5)}
const s=load(),id=path.basename(f.wd);
if(!isBus(u)){s[u]={state:'active',pid:null,wd:f.wd};save(s);process.exit(0)}
if(fault(`start-dies-${id}`)){s[u]={state:'failed',pid:null,wd:f.wd};save(s);process.exit(0)}
snap('start');
const off=process.env.SD_CLOCK_OFFSET||'0';
// The holder must not inherit stack-release.sh's lock (fd 9) or any other fd above 2.
const close='for fd in /proc/self/fd/*; do n=${fd##*/}; [ "$n" -gt 2 ] && eval "exec $n>&-" 2>/dev/null; done; exec "$@"';
const ch=cp.spawn('bash',['-c',close,'holder',process.execPath,path.join(__dirname,'..','holder.cjs'),data,inst(u),sd,off],{detached:true,stdio:'ignore'});
ch.unref();
fs.appendFileSync(path.join(sd,'pids.log'),ch.pid+'\n');
const ready=path.join(sd,`ready-${ch.pid}`),end=Date.now()+10000;
while(!fs.existsSync(ready)){if(Date.now()>end){console.error('fixture holder did not start');process.exit(1)}sleep(20)}
s[u]={state:'active',pid:ch.pid,wd:f.wd};save(s);
if(f.exec.split(' ').slice(1).includes('--pm'))pm(inst(u),id,ch.pid,Number(off));
process.exit(0);
}
if(verb==='stop'){
const u=rest[0],s=load(),st=s[u];
if(st&&st.pid){
try{process.kill(st.pid,'SIGTERM')}catch{}
const end=Date.now()+10000;while(alive(st.pid)&&Date.now()<end)sleep(20);
if(alive(st.pid))try{process.kill(st.pid,'SIGKILL')}catch{}
}
if(st){s[u]={...st,state:'inactive',pid:null};save(s)}
if(isBus(u)){
// One-shot: the stop leaves an empty -wal or -shm beside the store, as an unclean close would.
for(const [name,ext] of [['stop-leaves-wal','-wal'],['stop-leaves-shm','-shm']])if(fault(name)){fs.rmSync(faultFile(name));fs.writeFileSync(path.join(data,'bus','bus.sqlite'+ext),'')}
snap('stop');
}
process.exit(0);
}
console.error(`fixture systemctl: unsupported: ${args.join(' ')}`);process.exit(1);
}
// journalctl --user -u UNIT --since @EPOCH -o cat. A bare local time gets exit 1, as M35 needs.
function journalctl(){
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path');
const sd=process.env.SD_STATE,a=process.argv.slice(2);let unit='',since='';
for(let i=0;i<a.length;i++){if(a[i]==='-u')unit=a[++i];else if(a[i]==='--since')since=a[++i]}
fs.appendFileSync(path.join(sd,'journal-since.log'),since+'\n');
if(fs.existsSync(path.join(sd,'faults','journal-fails')))process.exit(1);
let from;
if(/^@\d+$/.test(since))from=Number(since.slice(1));
else{const m=/^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d) UTC$/.exec(since);if(!m)process.exit(1);from=Date.parse(`${m[1]}T${m[2]}Z`)/1000}
let lines=[];try{lines=fs.readFileSync(path.join(sd,'journal.jsonl'),'utf8').split('\n').filter(Boolean).map((l)=>JSON.parse(l))}catch{}
for(const o of lines)if(o.unit===unit&&o.epoch>=from)console.log(o.text);
}
function systemdAnalyze(){
const path=process.getBuiltinModule('node:path');
if(process.argv[2]!=='--user'||process.argv[3]!=='unit-paths')process.exit(1);
console.log(path.join(process.env.XDG_CONFIG_HOME,'systemd','user'));
}
// The bus host: holds the store open in WAL mode until SIGTERM, as the broker does.
function holder(){
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite');
const [data,business,sd,offset]=process.argv.slice(2),bus=path.join(data,'bus');
fs.mkdirSync(bus,{recursive:true,mode:0o700});
const db=new DatabaseSync(path.join(bus,'bus.sqlite'));
if(!db.prepare("SELECT count(*) AS n FROM sqlite_master WHERE name='meta'").get().n){
db.exec('PRAGMA journal_mode=WAL');
db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
db.prepare("INSERT INTO meta VALUES ('schema_version', '3g')").run();
}
fs.writeFileSync(path.join(bus,'writer.lock'),String(process.pid));
const at=new Date(Date.now()-Number(offset||0)*1000).toISOString(),host=path.join(sd,'host.json');
fs.writeFileSync(host+'.tmp',JSON.stringify({host:{business,pid:process.pid,live:true,startedAt:at,notifier:'ok'},boot:{ok:true,business,at}}));
fs.renameSync(host+'.tmp',host);
process.on('SIGTERM',()=>{db.close();fs.rmSync(path.join(bus,'writer.lock'),{force:true});process.exit(0)});
fs.writeFileSync(path.join(sd,`ready-${process.pid}`),'');
setInterval(()=>{},1<<30);
}
// A copy's scripts/mosaic: bus status and launches list from what the holder and systemctl wrote.
function mosaicStub(){
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path');
const id=path.basename(path.resolve(__dirname,'..')),sd=process.env.SD_STATE,a=process.argv.slice(2).join(' ');
const fault=(n)=>fs.existsSync(path.join(sd,'faults',n));
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}};
if(a==='bus status --json'){
if(fault(`status-${id}`))process.exit(1);
let s={host:null,boot:null};
try{s=JSON.parse(fs.readFileSync(path.join(sd,'host.json'),'utf8'));s.host.live=alive(s.host.pid)}catch{}
console.log(JSON.stringify(s));
}else if(a==='bus status')console.log(`bus: fixture status for ${id}`);
else if(a==='launches list --json'){
if(fault(`launches-${id}`))process.exit(1);
let l=[];try{l=JSON.parse(fs.readFileSync(path.join(sd,'launches.json'),'utf8'))}catch{}
console.log(JSON.stringify(l));
}else{console.error(`fixture mosaic: ${a}`);process.exit(4)}
}
// A copy's migrate-schema.mjs: records the backups it saw, then sets 3h, or 3h-partial and exit 1.
function migrateStub(){
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite');
const data=process.argv[2],sd=process.env.SD_STATE,dir=path.join(data,'backups','stack');
const backups=fs.existsSync(dir)?fs.readdirSync(dir).sort().map((name)=>({name,sums:fs.existsSync(path.join(dir,name,'SHA256SUMS'))})):[];
fs.writeFileSync(path.join(sd,'migrate.json'),JSON.stringify({backups}));
const fail=fs.existsSync(path.join(sd,'faults','migrate-fails'));
const db=new DatabaseSync(path.join(data,'bus','bus.sqlite'));
db.prepare("UPDATE meta SET value=? WHERE key='schema_version'").run(fail?'3h-partial':'3h');db.close();
if(fail)process.exit(1);
console.log('fixture migrate: 3h');
}
// --- The module fixture: one repo, its copies and the stand-ins, shared by every test. ---
const base=realpathSync(mkdtempSync(join(tmpdir(),'stack-release-')));
after(()=>{execFileSync('chmod',['-R','u+w',base]);rmSync(base,{recursive:true,force:true})});
const repo=join(base,'repo'),COPIES=join(base,'copies'),SBIN=join(base,'sbin');
mkdirSync(repo);mkdirSync(COPIES);mkdirSync(SBIN);
writeFileSync(join(base,'package.json'),'{"type":"commonjs"}\n');
const tracked=execFileSync('git',['-C',ROOT,'ls-files','-co','--exclude-standard','--','scripts','packages','adapters','roles','RELEASE','package.json','package-lock.json',':!packages/*/tests',':!scripts/foundation'],{encoding:'utf8'}).trim().split('\n');
for(const f of tracked)if(existsSync(join(ROOT,f)))cpSync(join(ROOT,f),join(repo,f));
const pkg={name:'copy-fixture',version:'1.0.0'};
writeFileSync(join(repo,'package.json'),JSON.stringify(pkg));
writeFileSync(join(repo,'package-lock.json'),JSON.stringify({name:pkg.name,version:pkg.version,lockfileVersion:3,packages:{'':pkg}}));
git(repo,'init','-q','-b','refactor');git(repo,'config','gc.auto','0');git(repo,'add','.');git(repo,'commit','-qm','fixture');
const commit=(msg)=>{git(repo,'commit','-q','--allow-empty','-m',msg);return git(repo,'rev-parse','HEAD')};
// refactor: F A B C1 C, then M merging side branch S (from A). U sits on a branch nothing reaches.
const F=git(repo,'rev-parse','HEAD'),A=commit('row 901: A'),B=commit('row 902: B'),C1=commit('row 903: C1'),C=commit('row 904: C');
git(repo,'checkout','-q','-b','side',A);const S=commit('row 905: S');
git(repo,'checkout','-q','-b','unreached',A);const U=commit('row 906: U');
git(repo,'checkout','-q','refactor');git(repo,'merge','-q','--no-ff','-m','merge side','side');
const M=git(repo,'rev-parse','HEAD');git(repo,'update-ref','refs/remotes/origin/refactor',M);
// `to` false leaves out the `to:` lines (C1); a string puts that value in each one (A). The --plan
// steps line can't be read from either (T27).
const storeText=(v,froms,to=true)=>`export const SCHEMA_VERSION = '${v}';\nexport const MIGRATIONS = [\n${froms.map((f,i)=>` {\n from: '${f}',\n${to===false?'':` to: '${to===true?froms[i+1]??v:to}',\n`} run() {},\n },\n`).join('')}];\n`;
for(const [id,store] of [[F,'// a copy from before SCHEMA_VERSION\n'],[A,storeText('3g',['3f'],'3x')],[B,storeText('3g',['3f'])],[C1,storeText('3g',['3f'],false)],[C,storeText('3h',['3f','3g'])],[S,storeText('3g',['3f'])],[U,storeText('3g',['3f'])]]){
const d=join(COPIES,id);
for(const [f,text,mode] of [['scripts/mosaic',script(mosaicStub),0o755],['packages/webui/src/cli.mjs','// fixture Console, never run\n',0o644],['packages/bus/src/store.mjs',store,0o644],['packages/bus/src/migrate-schema.mjs',`(${migrateStub})();\n`,0o644]]){
mkdirSync(dirname(join(d,f)),{recursive:true});writeFileSync(join(d,f),text,{mode});
}
execFileSync('chmod',['-R','a-w',d]);
}
for(const [name,fn] of [['systemctl',systemctl],['journalctl',journalctl],['systemd-analyze',systemdAnalyze]])writeFileSync(join(SBIN,name),script(fn),{mode:0o755});
writeFileSync(join(base,'holder.cjs'),`(${holder})();\n`);
const SR=join(repo,'scripts','stack-release.sh');
// --- Per-test roots and helpers ---
const sha=(f)=>createHash('sha256').update(readFileSync(f)).digest('hex');
const lines=(f)=>existsSync(f)?readFileSync(f,'utf8').split('\n').filter(Boolean):[];
const calls=(e)=>lines(join(e.sd,'calls.log'));
const startsStops=(e,from)=>calls(e).slice(from).filter((c)=>/^(start|stop) /.test(c));
const logLines=(e)=>lines(e.log).map((l)=>JSON.parse(l));
const pointer=(e)=>existsSync(e.pointer)?JSON.parse(readFileSync(e.pointer,'utf8')):null;
const units=(e)=>JSON.parse(readFileSync(join(e.sd,'units.json'),'utf8'));
const snaps=(e)=>lines(join(e.sd,'snap.jsonl')).map((l)=>JSON.parse(l));
const fault=(e,name,text='')=>writeFileSync(join(e.sd,'faults',name),text);
const sysctl=(e,...a)=>execFileSync(join(SBIN,'systemctl'),['--user',...a],{env:e.env,encoding:'utf8'});
const edit=(f,from,to)=>{const s=readFileSync(f,'utf8');assert.ok(s.includes(from),`${f} has no ${from}`);writeFileSync(f,s.replace(from,to))};
const posts=(e)=>readdirSync(e.data).filter((n)=>n.startsWith('bus.post-attempt-'));
const backupSum=(e,act)=>readFileSync(join(e.data,'backups','stack',act,'SHA256SUMS'),'utf8').split(' ')[0];
const commitLines=(range)=>git(repo,'log','--oneline','--no-merges','--max-count=200',range).split('\n').map((l)=>` ${l}`);
// The units, the wrapper, the pointer and the store, by sha256.
const fp=(e)=>({bus:sha(join(e.units,'[email protected]')),console:sha(join(e.units,CON)),wrapper:sha(join(e.bin,'mosaic')),pointer:existsSync(e.pointer)?sha(e.pointer):null,store:sha(e.store)});
// The schema version: readOnly while the holder has the store open, immutable when it is stopped.
function storeVersion(file,held){
const db=held?new DatabaseSync(file,{readOnly:true}):new DatabaseSync(pathToFileURL(file).href+'?immutable=1',{readOnly:true});
try{return db.prepare("SELECT value FROM meta WHERE key='schema_version'").get().value}finally{db.close()}
}
// Every file and directory under p with size and mtime: T23 and T16 compare it before and after.
function tree(p){
const out=[];
const walk=(q)=>{let st;try{st=statSync(q)}catch{return}out.push(st.isDirectory()?`${q}/`:`${q} ${st.size} ${st.mtimeMs}`);if(st.isDirectory())for(const n of readdirSync(q).sort())walk(join(q,n))};
walk(p);return out;
}
// T17: what a run may never remove.
const kept=(e)=>[...tree(join(e.data,'backups','stack')),...tree(join(e.state,'stack-rollback')),...posts(e).flatMap((n)=>tree(join(e.data,n))),...tree(COPIES)].map((l)=>l.replace(/ .*/,''));
async function mk(t){
const root=realpathSync(mkdtempSync(join(tmpdir(),'sr-')));
const e={root,data:join(root,'data'),bin:join(root,'bin'),sd:join(root,'sd'),units:join(root,'xcfg','systemd','user'),cfg:join(root,'cfg','config.json'),claude:join(root,'claude-bin','claude')};
Object.assign(e,{state:join(e.data,'state'),pointer:join(e.data,'state','stack-active.json'),log:join(e.data,'state','stack-activation-log.jsonl'),store:join(e.data,'bus','bus.sqlite'),lock:join(e.data,'state','stack-release.lock')});
for(const d of [join(root,'home'),dirname(e.cfg),join(e.data,'bus'),e.bin,dirname(e.claude),join(e.sd,'faults'),e.units,join(root,'xdata')])mkdirSync(d,{recursive:true});
chmodSync(join(e.data,'bus'),0o700);
writeFileSync(e.cfg,JSON.stringify({configVersion:1,environment:'development',dataRoot:e.data,execution:{backend:'docker',provider:'zai',model:'glm-5.3-flash',adapter:'mock'}}));
writeFileSync(join(e.bin,'mosaic'),`#!/usr/bin/env bash\n${MARKER}\n[ "\${1:-}" = __lease-capability ] && exec "${e.bin}/mosaic-old" "$@"\nexec "/old/scripts/mosaic" "$@"\n`,{mode:0o755});
writeFileSync(join(e.bin,'mosaic-old'),'#!/bin/sh\nexit 0\n',{mode:0o755});
writeFileSync(e.claude,'#!/bin/sh\necho "9.9.9 (Claude Code fixture)"\n',{mode:0o755});
e.env={PATH:`${SBIN}:${dirname(NODE)}:/usr/bin:/bin`,HOME:join(root,'home'),XDG_CONFIG_HOME:join(root,'xcfg'),XDG_DATA_HOME:join(root,'xdata'),MOSAIC_CONFIG:e.cfg,SD_STATE:e.sd,SD_DATA:e.data,TMPDIR:tmpdir()};
// The Console's /healthz: 200 while the stand-in has it active and no healthz-<copy> fault names its copy.
const server=createServer((q,r)=>{
let st={};try{st=units(e)[CON]||{}}catch{}
const ok=q.url==='/healthz'&&st.state==='active'&&!existsSync(join(e.sd,'faults',`healthz-${basename(st.wd||'')}`));
r.writeHead(ok?200:500);r.end();
});
server.listen(0,'127.0.0.1');await once(server,'listening');e.port=String(server.address().port);
t.after(()=>{
for(const pid of lines(join(e.sd,'pids.log')).map(Number)){
try{if(readFileSync(`/proc/${pid}/cmdline`,'utf8').includes(join(base,'holder.cjs')))process.kill(pid,'SIGKILL')}catch{}
}
server.close();execFileSync('chmod',['-R','u+w',root]);rmSync(root,{recursive:true,force:true});
});
return e;
}
async function run(e,...args){
const before=existsSync(e.data)?kept(e):[];
const ch=spawn('bash',[SR,...args,'--copies',COPIES,'--bin',e.bin],{env:e.env,stdio:['ignore','pipe','pipe']});
let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d});
const timer=setTimeout(()=>ch.kill('SIGKILL'),120000);
const [code]=await once(ch,'close');clearTimeout(timer);
const now=new Set(kept(e));
for(const p of before)assert.ok(now.has(p),`T17: ${args.join(' ')} removed ${p}`);
return {code,out,err};
}
// The stack on copy `copy` for business acme, both units active. pointer: adopt (a real adopt and
// its log line), hand (written here, no log) or none. Snapshots start after the seed.
async function seed(e,{copy=B,pm=false,store='3g',pointer:how='adopt'}={}){
const db=new DatabaseSync(e.store);
db.exec('PRAGMA journal_mode=WAL');
db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
db.prepare("INSERT INTO meta VALUES ('schema_version', ?)").run(store);
for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`);
db.close();
const sh=(name,...a)=>execFileSync('bash',[join(repo,'scripts',name),...a],{env:e.env,encoding:'utf8',stdio:['ignore','pipe','pipe']});
sh('bus-service.sh','install','--copy',copy,'--copies',COPIES,...(pm?['--pm']:[]),'--claude',e.claude);
sh('webui-service.sh','install','--copy',copy,'--copies',COPIES,'--port',e.port,'--business','acme');
sh('stack-copy.sh','wrapper',copy,'--copies',COPIES,'--bin',e.bin);
// The seeded host started two minutes ago, so its boot status and PM run predate any run's start.
execFileSync(join(SBIN,'systemctl'),['--user','start',BUS],{env:{...e.env,SD_CLOCK_OFFSET:'120'}});
sysctl(e,'start',CON);
if(how==='adopt'){const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err)}
else if(how==='hand'){
mkdirSync(e.state,{recursive:true});
writeFileSync(e.pointer,JSON.stringify({copy,business:'acme',pm,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g',activation:'manual',at:'2026-10-18T00:00:00Z'})+'\n');
}
writeFileSync(join(e.sd,'snap.jsonl'),'');
}
const rolledBack=(code,to=B)=>`stack-release: ${code}; rolled back to ${to}, which passed the gate\n`;
const planTail=(e,to,pm=false)=>[`bus: scripts/bus-service.sh install --copy ${to} --copies ${COPIES}${pm?' --pm':''} --claude ${e.claude}`,`Console: scripts/webui-service.sh install --copy ${to} --copies ${COPIES} --port ${e.port} --business acme`,`CLI: scripts/stack-copy.sh wrapper ${to} --copies ${COPIES} --bin ${e.bin}`];
// --- The T table ---
test('T1 adopt writes the pointer from the units once; a failing check refuses check-failed',async(t)=>{
const e=await mk(t);await seed(e,{pointer:'none'});
const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err);
assert.equal(r.out,`adopted ${B} for acme (pm 0, schema 3g)\n`);
const {at,activation,...p}=pointer(e);
assert.deepEqual(p,{copy:B,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'});
assert.match(activation,new RegExp(`^\\d{8}T\\d{6}Z-${B.slice(0,12)}$`));assert.match(at,/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/);
assert.deepEqual(logLines(e).map((l)=>[l.op,l.from,l.to,l.result]),[['adopt',null,B,'ok']]);
const again=await run(e,'adopt','acme');assert.equal(again.code,3);
assert.equal(again.err,`stack-release: refusing already-adopted: ${e.pointer} exists (copy ${B})\n`);
assert.equal(logLines(e).at(-1).code,'already-adopted');
const f=await mk(t);await seed(f,{pointer:'none'});fault(f,`check-${B}`);
const bad=await run(f,'adopt','acme');assert.equal(bad.code,3);
assert.match(bad.err,/^stack-release: refusing check-failed: stack-copy.sh check did not pass:\nbus: /);
assert.equal(pointer(f),null);
});
test('T2 activate installs exactly the pointer\'s pm, claude and Console flags',async(t)=>{
const e=await mk(t);await seed(e);
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err+r.out);
assert.ok(r.out.endsWith(`stack-release: activate ok: ${C1} is live for acme\n`),r.out);
const bus=readFileSync(join(e.units,'[email protected]'),'utf8').split('\n'),con=readFileSync(join(e.units,CON),'utf8').split('\n');
assert.ok(bus.includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i`),bus.join('\n'));
assert.ok(bus.includes(`Environment=MOSAIC_CLAUDE_BIN=${e.claude}`));
const exec=con.find((l)=>l.startsWith('ExecStart='));
assert.ok(exec.endsWith(` ${COPIES}/${C1}/packages/webui/src/cli.mjs serve --checkout ${repo} --port ${e.port} --business acme`),exec);
assert.equal(readFileSync(join(e.bin,'mosaic'),'utf8').trimEnd().split('\n').at(-1),`exec "${COPIES}/${C1}/scripts/mosaic" "$@"`);
const {at,activation,...p}=pointer(e);
assert.deepEqual(p,{copy:C1,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'});
});
test('T3 a hand-edited unit refuses unit-flags-drift, names the difference and changes nothing',async(t)=>{
const cases=[
['pm',(e)=>edit(join(e.units,'[email protected]'),' bus start %i\n',' bus start %i --pm\n'),()=>' pm: pointer 0, unit 1'],
['claude',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'claude.conf'),'[Service]\nEnvironment=MOSAIC_CLAUDE_BIN=/opt/other/claude\n')},(e)=>` claude: pointer ${e.claude}, unit /opt/other/claude`],
['console',(e)=>edit(join(e.units,CON),'--business acme\n','--business acme --board http://127.0.0.1:9\n'),(e)=>` console: pointer --port ${e.port} --business acme, unit --port ${e.port} --business acme --board http://127.0.0.1:9`],
];
for(const [name,change,line] of cases)await t.test(name,async(t)=>{
const e=await mk(t);await seed(e);change(e);
const before=fp(e),n=calls(e).length,logs=logLines(e).length;
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,3);
assert.equal(r.err,`stack-release: refusing unit-flags-drift: the units differ from the pointer; settle it first:\n${line(e)}\n`);
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'unit-flags-drift');
});
});
test('T4 each item 3 refusal changes nothing and logs one line with its code',async(t)=>{
const setUnit=(e,u,state)=>{const s=units(e);s[u]={...(s[u]||{pid:null,wd:''}),state};writeFileSync(join(e.sd,'units.json'),JSON.stringify(s))};
const cases=[
['copy-missing',()=>M,()=>`no copy ${M} in ${COPIES}`],
['copy-unreachable',()=>U,()=>`${U} is not reachable from refactor and origin/refactor`],
['copy-schema-unknown',()=>F,()=>`cannot read SCHEMA_VERSION and its chain from ${COPIES}/${F}/packages/bus/src/store.mjs`],
['already-active',()=>B,()=>`${B} is the pointer's copy`],
['instance-busy',(e)=>{setUnit(e,'[email protected]','active');return C1},()=>'[email protected] is active; the copy and wrapper switch reach it too','another business active'],
['instance-busy',(e)=>{setUnit(e,BUS,'activating');return C1},()=>`${BUS} is activating`,'this business activating'],
['drop-in-override',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'override.conf'),'[Service]\nWorkingDirectory=/elsewhere\n');return C1},(e)=>`${join(e.units,`${BUS}.d`,'override.conf')} overrides ExecStart or WorkingDirectory; the operator removes it as in TOOLS.md step 4`],
['no-pointer',()=>C1,(e)=>`no ${e.pointer}; run adopt first`,'',{pointer:'none'}],
];
for(const [code,setup,msg,label,opts] of cases)await t.test(label?`${code} (${label})`:code,async(t)=>{
const e=await mk(t);await seed(e,opts);const id=setup(e);
const before=fp(e),n=calls(e).length,logs=logLines(e).length;
const r=await run(e,'activate',id,'acme');assert.equal(r.code,3,r.err);
assert.equal(r.err,`stack-release: refusing ${code}: ${msg(e)}\n`);
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
assert.equal(logLines(e).length,logs+1);assert.deepEqual([logLines(e).at(-1).result,logLines(e).at(-1).code],['refused',code]);
});
});
test('T5 an activate with no migration stops the Console then the bus, installs, starts the bus then the Console',async(t)=>{
const e=await mk(t);await seed(e);const n=calls(e).length;
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err);
assert.deepEqual(startsStops(e,n),[`stop ${CON}`,`stop ${BUS}`,`start ${BUS}`,`start ${CON}`]);
const c=calls(e).slice(n),reload=c.indexOf('daemon-reload');
assert.ok(reload>c.indexOf(`stop ${BUS}`)&&reload<c.indexOf(`start ${BUS}`),'daemon-reload between the stops and the starts');
assert.equal(pointer(e).copy,C1);assert.equal(units(e)[BUS].wd,join(COPIES,C1));
assert.ok(!existsSync(join(e.data,'backups','stack')));
const l=logLines(e).slice(-2);assert.deepEqual(l.map((x)=>[x.op,x.from,x.to,x.result]),[['activate',B,C1,'started'],['activate',B,C1,'ok']]);
});
test('T6 a migration without --migrate rolls back with migration-needs-flag',async(t)=>{
const e=await mk(t);await seed(e);const before=fp(e);
const r=await run(e,'activate',C,'acme','--timeout','3');assert.equal(r.code,1,r.out);
assert.equal(r.err,rolledBack('migration-needs-flag'));
assert.deepEqual(fp(e),before);
assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd,units(e)[CON].state],['active',join(COPIES,B),'active']);
const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','migration-needs-flag','ok']);
assert.equal(storeVersion(e.store,true),'3g');assert.ok(!existsSync(join(e.data,'backups','stack')));
});
test('T7 --migrate takes the backup before the migration runs',async(t)=>{
const e=await mk(t);await seed(e);const before=fp(e);
const r=await run(e,'activate',C,'acme','--migrate');assert.equal(r.code,0,r.err+r.out);
const act=pointer(e).activation;
assert.deepEqual(JSON.parse(readFileSync(join(e.sd,'migrate.json'),'utf8')),{backups:[{name:act,sums:true}]});
assert.equal(backupSum(e,act),before.store);
assert.deepEqual([pointer(e).copy,pointer(e).schema],[C,'3h']);assert.equal(storeVersion(e.store,true),'3h');
assert.equal(logLines(e).at(-1).backup,act);
});
test('T8 a failed migration restores the backup and keeps the store it replaced',async(t)=>{
const e=await mk(t);await seed(e);const before=fp(e);fault(e,'migrate-fails');
const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out);
assert.equal(r.err,rolledBack('migrate-failed'));
assert.equal(posts(e).length,1);assert.equal(storeVersion(join(e.data,posts(e)[0],'bus.sqlite'),false),'3h-partial');
const act=logLines(e).at(-1).activation;
assert.equal(sha(e.store),backupSum(e,act));assert.equal(sha(e.store),before.store);
assert.deepEqual(fp(e),before);assert.equal(units(e)[BUS].wd,join(COPIES,B));
});
test('T9 a store outside the target\'s chain rolls back with store-newer',async(t)=>{
const e=await mk(t);await seed(e,{store:'3z'});const before=fp(e);
const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out);
assert.equal(r.err,rolledBack('store-newer'));
assert.deepEqual(fp(e),before);assert.equal(storeVersion(e.store,true),'3z');assert.ok(!existsSync(join(e.data,'backups','stack')));
});
test('T10 each gate check failing alone rolls back and puts every file back',async(t)=>{
const cases=[['units-inactive',`start-dies-${C1}`],['bus-status',`status-${C1}`],['healthz',`healthz-${C1}`],['copy-check',`check-${C1}`],['pm-launch-unconfirmed',`pm-${C1}`,'none']];
for(const [code,name,text] of cases)await t.test(code,async(t)=>{
const pm=code==='pm-launch-unconfirmed',e=await mk(t);await seed(e,{pm});fault(e,name,text);
const before=fp(e);
const r=await run(e,'activate',C1,'acme',...(pm?['--pm']:[]),'--timeout','3');assert.equal(r.code,1,r.out+r.err);
assert.equal(r.err,rolledBack(code));assert.ok(r.out.includes(`stack-release: gate failed on ${C1}: ${code}`),r.out);
assert.deepEqual(fp(e),before);
assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,B)]);
const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed',code,'ok']);
});
});
test('T10b the PM check needs exactly one launched line for a live run of this start',async(t)=>{
const cases=[
['no launched line','none',/\(no launched line since the start\)/],
['two lines','two',/\(2 launched lines since the start\)/],
['a changed word','badword',/\(the launched line does not parse: launched pm-acme: job run-\d+, claude opus\)/],
['run not in launches list','notlisted',/\(run run-\d+ is not in launches list\)/],
['run started before the start','early',/\(run run-\d+ started before this start\)/],
['a human launch with no journal line','human',/\(no launched line since the start\)/],
['run for another business','other',/\(run run-\d+ is for other\)/],
['run not live','dead',/\(run run-\d+ is not live\)/],
];
for(const [name,mode,why] of cases)await t.test(name,async(t)=>{
const e=await mk(t);await seed(e,{pm:true});fault(e,`pm-${C1}`,mode);
const r=await run(e,'activate',C1,'acme','--pm','--timeout','3');assert.equal(r.code,1,r.out+r.err);
assert.equal(r.err,rolledBack('pm-launch-unconfirmed'));
const gate=r.out.split('\n').find((l)=>l.startsWith(`stack-release: gate failed on ${C1}: pm-launch-unconfirmed`));
assert.ok(gate,r.out);assert.match(gate,why);
});
await t.test('one well-formed line and a live run pass',async(t)=>{
const e=await mk(t);await seed(e,{pm:true});
const r=await run(e,'activate',C1,'acme','--pm');assert.equal(r.code,0,r.out+r.err);
assert.ok(readFileSync(join(e.units,'[email protected]'),'utf8').includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i --pm\n`));
assert.match(lines(join(e.sd,'journal-since.log')).at(-1),/^@\d+$/);
assert.equal(pointer(e).pm,true);
});
});
test('T11 a rollback whose gate fails too: rollback-failed, exit 2, both units stopped, no retry',async(t)=>{
const e=await mk(t);await seed(e);fault(e,`healthz-${C1}`);fault(e,`healthz-${B}`);const n=calls(e).length;
const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,2,r.out+r.err);
const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation);
assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with healthz; both units are stopped. Rollback set: ${rs}\n`);
assert.deepEqual([l.result,l.code,l.rollback,l.rollbackCode],['rollback-failed','healthz','failed','healthz']);
assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']);
const ss=startsStops(e,n);
assert.deepEqual(ss.filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`,`start ${BUS}`,`start ${CON}`]);
assert.deepEqual(ss.slice(-2),[`stop ${CON}`,`stop ${BUS}`]);
assert.equal(pointer(e).copy,B);
});
test('T12 rollback takes the previous copy from the log',async(t)=>{
const e=await mk(t);await seed(e);
assert.equal((await run(e,'activate',C1,'acme')).code,0);
const r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err);
assert.ok(r.out.endsWith(`stack-release: rollback ok: ${B} is live for acme\n`),r.out);
assert.equal(pointer(e).copy,B);
const l=logLines(e).at(-1);assert.deepEqual([l.op,l.from,l.to,l.result],['rollback',C1,B,'ok']);
});
test('T13 a rollback across a migration needs --restore, then restores and goes live',async(t)=>{
const e=await mk(t);await seed(e);
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store);
const no=await run(e,'rollback','acme');assert.equal(no.code,3);
assert.equal(no.err,`stack-release: refusing schema-differs: the store is at 3h and ${B} expects 3g; pass --restore ACTIVATION\n`);
const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err);
const o=join(e.data,'backups','stack',act);
assert.match(r.out,new RegExp(`^restore: ${o}; the store now in place moves to ${e.data}/bus\\.post-attempt-\\d{8}T\\d{6}Z and is kept; 0 event\\(s\\) were written after the backup\\n`));
assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(storeVersion(e.store,true),'3g');
assert.equal(sha(e.store),backupSum(e,act));
assert.equal(posts(e).length,1);assert.equal(sha(join(e.data,posts(e)[0],'bus.sqlite')),storeC);
assert.equal(logLines(e).at(-1).backup,act);
});
test('T14 events written after the backup need --events N with the right N',async(t)=>{
const e=await mk(t);await seed(e);
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation;
const db=new DatabaseSync(e.store);for(let i=0;i<2;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run('2999-01-01T00:00:00.000Z');db.close();
const want='stack-release: refusing post-backup-events: 2 event(s) were written after the backup; Jason decides, then --events 2\n';
for(const extra of [[],['--events','1']]){
const before=fp(e),r=await run(e,'rollback','acme','--restore',act,...extra);
assert.equal(r.code,3,extra.join(' '));assert.equal(r.err,want);assert.deepEqual(fp(e),before);
}
const r=await run(e,'rollback','acme','--restore',act,'--events','2');assert.equal(r.code,0,r.out+r.err);
assert.match(r.out,/; 2 event\(s\) were written after the backup\n/);
assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);
});
test('T14b a rollback --restore whose gate fails stops both units and skips item 6',async(t)=>{
const e=await mk(t);await seed(e);
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store),before=fp(e);
fault(e,`healthz-${B}`);const n=calls(e).length;
const r=await run(e,'rollback','acme','--restore',act,'--timeout','3');assert.equal(r.code,2,r.out+r.err);
const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation),post=join(e.data,posts(e)[0]);
assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with not-attempted; both units are stopped. Rollback set: ${rs}, backup ${join(e.data,'backups','stack',act)}, post-attempt store ${post}\n`);
assert.deepEqual([l.result,l.code,l.rollbackCode],['rollback-failed','healthz','not-attempted']);
assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']);
assert.equal(posts(e).length,1);assert.equal(sha(join(post,'bus.sqlite')),storeC);assert.equal(sha(e.store),backupSum(e,act));
assert.deepEqual(startsStops(e,n).filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`]);
assert.equal(pointer(e).copy,C);assert.equal(sha(e.pointer),before.pointer);
});
test('T14c a rollback without --restore whose gate fails runs item 6',async(t)=>{
const e=await mk(t);await seed(e);
assert.equal((await run(e,'activate',C1,'acme')).code,0);const before=fp(e);
fault(e,`healthz-${B}`);
const r=await run(e,'rollback','acme','--timeout','3');assert.equal(r.code,1,r.out+r.err);
assert.equal(r.err,rolledBack('healthz',C1));
const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.code,l.rollback],['rollback','gate-failed','healthz','ok']);
assert.deepEqual(fp(e),before);assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,C1)]);
assert.deepEqual(readdirSync(join(e.data,'bus')).filter((n)=>n.startsWith('bus.sqlite')).sort(),['bus.sqlite','bus.sqlite-shm','bus.sqlite-wal']);
});
test('T15 --plan names the migration and the restore and changes nothing',async(t)=>{
const e=await mk(t);await seed(e);
const quiet=async(...args)=>{
const before=fp(e),n=calls(e).length,logs=logLines(e).length,r=await run(e,...args);
assert.equal(r.code,0,r.err);assert.equal(r.err,'');
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs);
return r.out;
};
assert.ok((await quiet('activate',C,'acme','--plan')).includes('needs: --migrate (not given; the run would roll back with migration-needs-flag)\n'));
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation;
assert.equal(await quiet('rollback','acme','--plan'),[
'plan: rollback for acme',`from: ${C}`,`to: ${B}`,
'store schema: 3h (read from the store the running bus holds)','target schema: 3g',
`restore: the store must go back to 3g; needs --restore ${act}`,
'commits: the run takes away 2 (newest first, at most 200):',...commitLines(`${B}..${C}`),
...planTail(e,B),''].join('\n'));
});
test('T26 --plan lists the commits an activate adds, after the migration lines',async(t)=>{
const e=await mk(t);await seed(e);
const head=(mig)=>['plan: activate for acme',`from: ${B}`,`to: ${C}`,'store schema: 3g (read from the store the running bus holds)','target schema: 3h',
'migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h',`needs: --migrate (${mig})`,'needs: Jason present for this run (decision 96 condition 2)'];
const adds=commitLines(`${B}..${C}`);
assert.equal(adds.length,2);assert.match(adds[0],/^ {2}[0-9a-f]{7,} row 904: C$/);assert.match(adds[1],/^ {2}[0-9a-f]{7,} row 903: C1$/);
for(const [flags,mig] of [[[],'not given; the run would roll back with migration-needs-flag'],[['--migrate'],'given']]){
const r=await run(e,'activate',C,'acme','--plan',...flags);assert.equal(r.code,0,r.err);
assert.equal(r.out,[...head(mig),'commits: the run adds 2 (newest first, at most 200):',...adds,...planTail(e,C),''].join('\n'));
}
const same=await run(e,'activate',C1,'acme','--plan');assert.equal(same.code,0,same.err);
assert.ok(same.out.includes('target schema: 3g\nmigration: none; the store is at the target\'s version\ncommits: the run adds 1 (newest first, at most 200):\n'),same.out);
});
test('T27 --plan names each migration step, or says the steps cannot be read',async(t)=>{
const e=await mk(t);await seed(e,{store:'3f',pointer:'hand'});
const two=await run(e,'activate',C,'acme','--plan');assert.equal(two.code,0,two.err);
assert.ok(two.out.includes('\nmigration: 3f to 3h, after a backup of the stopped store\nmigration steps: 3f to 3g, 3g to 3h\nneeds: --migrate'),two.out);
const none=await run(e,'activate',C1,'acme','--plan');assert.equal(none.code,0,none.err);
assert.ok(none.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${C1}/packages/bus/src/store.mjs\nneeds: --migrate`),none.out);
const short=await run(e,'activate',A,'acme','--plan');assert.equal(short.code,0,short.err);
assert.ok(short.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${A}/packages/bus/src/store.mjs\nneeds: --migrate`),short.out);
});
test('T25 --plan refuses copies-unrelated when neither copy is an ancestor of the other',async(t)=>{
const e=await mk(t);await seed(e,{pointer:'hand'});
const r=await run(e,'activate',S,'acme','--plan');assert.equal(r.code,3);
assert.equal(r.err,`stack-release: refusing copies-unrelated: neither ${B} nor ${S} is an ancestor of the other, so the plan cannot list the commits\n`);
assert.equal(r.out,'');assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock));
});
test('T23 --plan reads the store without changing a file, stopped or held',async(t)=>{
const want=['target schema: 3h','migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h','needs: --migrate (not given; the run would roll back with migration-needs-flag)','needs: Jason present for this run (decision 96 condition 2)'];
for(const [name,stop,src] of [['stopped store',true,'read from the stopped store'],['held store',false,'read from the store the running bus holds']])await t.test(name,async(t)=>{
const e=await mk(t);await seed(e,{pointer:'hand'});if(stop)sysctl(e,'stop',BUS);
const before=tree(e.data),units0=[sha(join(e.units,'[email protected]')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],store0=sha(e.store);
const r=await run(e,'activate',C,'acme','--plan');assert.equal(r.code,0,r.err);
const out=r.out.split('\n');
assert.ok(out.includes(`store schema: 3g (${src})`),r.out);
assert.deepEqual(out.slice(out.indexOf('target schema: 3h'),out.indexOf('target schema: 3h')+5),want);
assert.deepEqual(tree(e.data),before);assert.equal(sha(e.store),store0);
assert.deepEqual([sha(join(e.units,'[email protected]')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],units0);
assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock));
if(stop)assert.deepEqual(readdirSync(join(e.data,'bus')),['bus.sqlite']);
});
});
test('T24 business-mismatch and capture-failed refuse with exit 3; a short id is a usage error',async(t)=>{
const e=await mk(t);await seed(e);
let n=calls(e).length,logs=logLines(e).length;
const r=await run(e,'activate',C1,'other');assert.equal(r.code,3);
assert.equal(r.err,'stack-release: refusing business-mismatch: the pointer is for acme\n');
assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'business-mismatch');assert.deepEqual(startsStops(e,n),[]);
const sets=join(e.state,'stack-rollback');mkdirSync(sets);chmodSync(sets,0o555);
n=calls(e).length;logs=logLines(e).length;const before=fp(e);
const c=await run(e,'activate',C1,'acme');assert.equal(c.code,3);
assert.match(c.err,new RegExp(`^stack-release: refusing capture-failed: could not write the rollback set ${sets}/\\d{8}T\\d{6}Z-${C1.slice(0,12)}; nothing was stopped\\n$`));
const l=logLines(e).slice(logs);
assert.deepEqual(l.map((x)=>[x.result,x.code]),[['started',undefined],['refused','capture-failed']]);assert.equal(l[0].activation,l[1].activation);
assert.deepEqual(startsStops(e,n),[]);assert.deepEqual(fp(e),before);
const s=await run(e,'status','acme');assert.equal(s.code,0,s.err);assert.ok(!s.out.includes('interrupted run'),s.out);
logs=logLines(e).length;
const u=await run(e,'activate','abc123','acme');assert.equal(u.code,4);assert.match(u.err,/^usage: stack-release.sh adopt BUSINESS\n/);
assert.equal(logLines(e).length,logs);
});
test('T16 status prints the pointer, the log, the check, bus status and drift, and writes nothing',async(t)=>{
const e=await mk(t);await seed(e);
const before=tree(e.data),fp0=fp(e);
const r=await run(e,'status','acme');assert.equal(r.code,0,r.err);
const copy=join(COPIES,B);
assert.equal(r.out,[
`pointer: ${e.pointer}`,...readFileSync(e.pointer,'utf8').trimEnd().split('\n').map((l)=>` ${l}`),
'log (last five lines):',...lines(e.log).map((l)=>` ${l}`),
'check:',` bus: ${copy}`,` Console: ${copy}`,` CLI: ${copy}`,' result: pass',
'bus status:',` bus: fixture status for ${B}`,
'unit flags: match the pointer',
'drift: 4 commit(s) on refactor since the pointer\'s copy',''].join('\n'));
assert.deepEqual(tree(e.data),before);assert.deepEqual(fp(e),fp0);
});
test('T19 a started line with no final line: status shows it and every changing verb refuses',async(t)=>{
const e=await mk(t);await seed(e);
const act=`20261018T000000Z-${C1.slice(0,12)}`;
appendFileSync(e.log,JSON.stringify({at:'2026-10-18T00:00:00Z',op:'activate',from:B,to:C1,backup:null,activation:act,result:'started'})+'\n');
const s=await run(e,'status','acme');assert.equal(s.code,0);
assert.ok(s.out.includes(`interrupted run: ${act} (started, no final line; Jason or Mos settles it)\n`),s.out);
for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){
const before=fp(e),n=calls(e).length,r=await run(e,...args);
assert.equal(r.code,3,args.join(' '));
assert.equal(r.err,`stack-release: refusing interrupted-run: activation ${act} started and has no final line; Jason or Mos settles it (status shows it)\n`);
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
}
});
test('T20 a held run lock refuses run-locked without a log line; status and --plan still run',async(t)=>{
const e=await mk(t);await seed(e);
const h=spawn('flock',[e.lock,'sleep','30'],{detached:true,stdio:'ignore'});
t.after(()=>{try{process.kill(-h.pid,'SIGKILL')}catch{}});
const end=Date.now()+10000;
while(spawnSync('flock',['-n',e.lock,'true']).status!==1){assert.ok(Date.now()<end,'the lock holder did not take the lock');await new Promise((r)=>setTimeout(r,50))}
for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){
const before=fp(e),logs=logLines(e).length,r=await run(e,...args);
assert.equal(r.code,3,args.join(' '));
assert.equal(r.err,`stack-release: refusing run-locked: another stack-release run holds ${e.lock}\n`);
assert.equal(logLines(e).length,logs);assert.deepEqual(fp(e),before);
}
assert.equal((await run(e,'status','acme')).code,0);
const p=await run(e,'activate',C1,'acme','--plan');assert.equal(p.code,0,p.err);
});
test('T21 a bus unit with no MOSAIC_CLAUDE_BIN: adopt and activate refuse no-claude-pin',async(t)=>{
const drop=(e)=>edit(join(e.units,'[email protected]'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,'');
const want=`stack-release: refusing no-claude-pin: ${BUS} has no MOSAIC_CLAUDE_BIN\n`;
const a=await mk(t);await seed(a,{pointer:'none'});drop(a);
const r=await run(a,'adopt','acme');assert.equal(r.code,3);assert.equal(r.err,want);assert.equal(pointer(a),null);
const b=await mk(t);await seed(b);drop(b);const before=fp(b),n=calls(b).length;
const s=await run(b,'activate',C1,'acme');assert.equal(s.code,3);assert.equal(s.err,want);
assert.deepEqual(fp(b),before);assert.deepEqual(startsStops(b,n),[]);
});
test('T22 the stopped store read: a -wal or -shm refuses store-unclean and is left alone; a clean read adds no file',async(t)=>{
for(const ext of ['-wal','-shm'])await t.test(`bus.sqlite${ext}`,async(t)=>{
const e=await mk(t);await seed(e);fault(e,`stop-leaves${ext}`);
const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err);
assert.equal(r.err,rolledBack('store-unclean'));
assert.ok(r.out.includes(`stack-release: ${e.store}${ext} is present; the store is left as it is for Jason or Mos\n`),r.out);
const s=snaps(e),first=s[0],start=s.find((x)=>x.ev==='start');
assert.equal(first.ev,'stop');assert.ok(`bus.sqlite${ext}` in first.files,JSON.stringify(first));
assert.deepEqual(start.files,first.files);
});
await t.test('clean',async(t)=>{
const e=await mk(t);await seed(e);
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err);
const s=snaps(e),stop=s.find((x)=>x.ev==='stop'),start=s.find((x)=>x.ev==='start');
assert.deepEqual(Object.keys(stop.files),['bus.sqlite']);assert.deepEqual(start.files,stop.files);
});
});