New scripts/stack-release.sh with four verbs. adopt records the running copy, the unit flags and the store schema in state/stack-active.json; it stops nothing. activate ID BUSINESS [--pm] [--migrate] stops the units, checks the stopped store, backs it up, migrates when the target's schema differs, installs and starts the target, then gates on both units active, a live bus host whose pid is the bus unit's MainPID with boot ok, Console /healthz, stack-copy.sh check and one launched journal line. A failed gate rolls back inside the same run and keeps the attempted store as bus.post-attempt-<stamp>. rollback [--restore ACT [--events N]] goes back to the log's previous copy and counts the events written after the backup. status prints the pointer, the log tail and the copy check. --plan prints what a run would do and writes nothing; it reads the store schema from the store the running bus holds, or from a stopped store. Exit codes: refusal 3, gate failure with rollback 1, rollback failed 2, usage 4. Nothing is pruned. stack-copy.sh gets a reachable function split from copy_commit and a guard so its helpers can be sourced. TOOLS.md "Pinned stack update" and the cli README pinned-stack paragraph describe the verbs. Owner Darkwing, brief docs/plans/2026-10-18_stack-release.md (D1). Reviewer Filbert: r1 approve (comment 28997) on 091693c5, r2 confirm-only approve (29004) on candidate manifest c213a801...bd15c after row 125 moved the base under TOOLS.md and the cli README. Sage gate on23622980f: 25 of 25 steps rc=0 on c96a95320 (15 packages + 10 scripts), node pass 2353 fail 0, selftests 422 passed 0 failed, 2026-10-18T20:32:13Z to 20:38:55Z, network sage-r162_default removed, 0 containers. Nothing live. The8e7ea89a1window (repin-8e7ea89a1.md) runs these verbs on Mos's go with Jason present. Co-Authored-By: Claude Opus 5.5 <[email protected]>
733 lines
51 KiB
JavaScript
733 lines
51 KiB
JavaScript
// scripts/stack-release.sh (D1, #1659). Design, the T table and the mutants:
|
|
// agents/darkwing/work/queue-162/BUILD.md. Every run uses a temp data root, temp units and stand-in
|
|
// systemctl, journalctl and systemd-analyze on PATH; nothing here reads ~/.mosaic-dev or a live unit.
|
|
import test, {after} from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {execFileSync,spawn,spawnSync} from 'node:child_process';
|
|
import {mkdtempSync,mkdirSync,writeFileSync,readFileSync,appendFileSync,cpSync,rmSync,chmodSync,existsSync,statSync,readdirSync,realpathSync} from 'node:fs';
|
|
import {createServer} from 'node:http';
|
|
import {createHash} from 'node:crypto';
|
|
import {DatabaseSync} from 'node:sqlite';
|
|
import {tmpdir} from 'node:os';
|
|
import {join,dirname,basename} from 'node:path';
|
|
import {fileURLToPath,pathToFileURL} from 'node:url';
|
|
import {once} from 'node:events';
|
|
|
|
const ROOT=fileURLToPath(new URL('../../..',import.meta.url)).replace(/\/$/,'');
|
|
const NODE=process.execPath;
|
|
const MARKER='# mosaic-cli wrapper (Q81, agents/sage/work/q81/mosaic-cli.sh)';
|
|
const BUS='[email protected]',CON='mosaic-webui.service';
|
|
const git=(cwd,...args)=>execFileSync('git',['-c','user.name=fixture','-c','user.email=fixture@invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','-C',cwd,...args],{encoding:'utf8'}).trim();
|
|
const script=(fn)=>`#!${NODE}\n(${fn})();\n`;
|
|
|
|
// --- Stand-ins. Each is serialized into its own file, so it uses only process and its own scope. ---
|
|
|
|
// systemctl --user: unit state in $SD_STATE/units.json, every call in calls.log. A bus start runs
|
|
// holder.cjs (the store held open in WAL mode, writer.lock, host.json) and, with --pm in the unit's
|
|
// ExecStart, writes the `launched` journal line and the launches entry unless a pm-<copy> fault
|
|
// names another case. Faults are files in $SD_STATE/faults named for the copy they hit.
|
|
function systemctl(){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process'),crypto=process.getBuiltinModule('node:crypto');
|
|
const sd=process.env.SD_STATE,data=process.env.SD_DATA,unitDir=path.join(process.env.XDG_CONFIG_HOME,'systemd','user');
|
|
const args=process.argv.slice(2).filter((a)=>a!=='--user'),[verb,...rest]=args;
|
|
fs.appendFileSync(path.join(sd,'calls.log'),args.join(' ')+'\n');
|
|
const faultFile=(n)=>path.join(sd,'faults',n),fault=(n)=>fs.existsSync(faultFile(n));
|
|
const stFile=path.join(sd,'units.json');
|
|
const load=()=>{try{return JSON.parse(fs.readFileSync(stFile,'utf8'))}catch{return {}}};
|
|
const save=(s)=>{fs.writeFileSync(stFile+'.tmp',JSON.stringify(s));fs.renameSync(stFile+'.tmp',stFile)};
|
|
const sleep=(ms)=>Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms);
|
|
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}};
|
|
const isBus=(u)=>/^mosaic-bus@[a-z0-9-]+\.service$/.test(u),inst=(u)=>u.slice('mosaic-bus@'.length,-'.service'.length);
|
|
const unitFile=(u)=>{
|
|
const main=isBus(u)?'[email protected]':u,file=path.join(unitDir,main);
|
|
if(!fs.existsSync(file))return null;
|
|
const drops=[];
|
|
for(const d of isBus(u)?[main+'.d',u+'.d']:[u+'.d']){const dir=path.join(unitDir,d);if(fs.existsSync(dir))for(const f of fs.readdirSync(dir).sort())if(f.endsWith('.conf'))drops.push(path.join(dir,f))}
|
|
const r={wd:'',exec:'',env:[],drops};
|
|
for(const f of [file,...drops])for(let line of fs.readFileSync(f,'utf8').split('\n')){
|
|
line=line.replace(/%i/g,isBus(u)?inst(u):'').replace(/%h/g,process.env.HOME);
|
|
const m=/^(WorkingDirectory|ExecStart|Environment)=(.*)$/.exec(line);if(!m)continue;
|
|
if(m[1]==='WorkingDirectory')r.wd=m[2];else if(m[1]==='ExecStart'){if(m[2])r.exec=m[2]}else r.env.push(m[2]);
|
|
}
|
|
return r;
|
|
};
|
|
const snap=(ev)=>{
|
|
const dir=path.join(data,'bus'),files={};
|
|
try{for(const n of fs.readdirSync(dir).sort()){const p=path.join(dir,n),st=fs.statSync(p);if(st.isFile())files[n]=[st.size,st.mtimeMs,crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex')]}}catch{}
|
|
fs.appendFileSync(path.join(sd,'snap.jsonl'),JSON.stringify({ev,files})+'\n');
|
|
};
|
|
const pm=(b,id,pid,off)=>{
|
|
let mode='';try{mode=fs.readFileSync(faultFile(`pm-${id}`),'utf8').trim()}catch{}
|
|
const now=Date.now()-off*1000,epoch=Math.floor(now/1000),at=new Date(now).toISOString(),run=`run-${pid}`,unit=`mosaic-bus@${b}.service`;
|
|
const line=(text)=>fs.appendFileSync(path.join(sd,'journal.jsonl'),JSON.stringify({epoch,unit,text})+'\n');
|
|
const ok=()=>line(`launched pm-${b}: run ${run}, claude opus`);
|
|
const lf=path.join(sd,'launches.json');let list=[];try{list=JSON.parse(fs.readFileSync(lf,'utf8'))}catch{}
|
|
const entry=(o)=>list.push({run,business:b,live:true,startedAt:at,...o});
|
|
line(`bus host up: business ${b}, socket /fixture/bus.sock, notifier off`);
|
|
switch(mode){
|
|
case 'none':break;
|
|
case 'two':ok();line(`launched pm-${b}: run ${run}b, claude opus`);entry({});break;
|
|
case 'badword':line(`launched pm-${b}: job ${run}, claude opus`);entry({});break;
|
|
case 'notlisted':ok();break;
|
|
case 'early':ok();entry({startedAt:new Date(now-3600e3).toISOString()});break;
|
|
case 'human':entry({run:`human-${pid}`});break;
|
|
case 'other':ok();entry({business:'other'});break;
|
|
case 'dead':ok();entry({live:false});break;
|
|
default:ok();entry({});
|
|
}
|
|
fs.writeFileSync(lf,JSON.stringify(list));
|
|
};
|
|
if(verb==='show'){
|
|
const u=rest[0],props=[];for(let i=1;i<rest.length;i++)if(rest[i]==='-p')props.push(rest[++i]);
|
|
const f=unitFile(u),st=load()[u]||{},id=f?path.basename(f.wd):'';
|
|
for(const p of props){
|
|
if(p==='LoadState')console.log(`LoadState=${f?'loaded':'not-found'}`);
|
|
else if(p==='WorkingDirectory')console.log(`WorkingDirectory=${f?f.wd+(!isBus(u)&&fault(`check-${id}`)?'-moved':''):''}`);
|
|
else if(p==='ExecStart')console.log(f&&f.exec?`ExecStart={ path=${f.exec.split(' ')[0]} ; argv[]=${f.exec} ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }`:'ExecStart=');
|
|
else if(p==='Environment')console.log(`Environment=${f?f.env.join(' '):''}`);
|
|
else if(p==='MainPID')console.log(`MainPID=${st.pid||0}`);
|
|
else if(p==='DropInPaths')console.log(`DropInPaths=${f?f.drops.join(' '):''}`);
|
|
}
|
|
process.exit(0);
|
|
}
|
|
if(verb==='is-active'){const q=rest[0]==='--quiet',u=q?rest[1]:rest[0],s=(load()[u]||{}).state||'inactive';if(!q)console.log(s);process.exit(s==='active'?0:3)}
|
|
if(verb==='is-enabled'){console.log('disabled');process.exit(1)}
|
|
if(verb==='daemon-reload')process.exit(0);
|
|
if(verb==='list-units'){
|
|
if(fault('list-fails'))process.exit(1);
|
|
const pats=rest.filter((a)=>!a.startsWith('--')).map((p)=>new RegExp('^'+p.replace(/\./g,'\\.').replace(/\*/g,'.*')+'$'));
|
|
for(const [u,s] of Object.entries(load()))if(pats.some((re)=>re.test(u)))console.log(`${u} loaded ${s.state} ${s.state==='active'?'running':'dead'} fixture`);
|
|
process.exit(0);
|
|
}
|
|
if(verb==='start'){
|
|
const u=rest[0],f=unitFile(u);if(!f){console.error(`Unit ${u} not found.`);process.exit(5)}
|
|
const s=load(),id=path.basename(f.wd);
|
|
if(!isBus(u)){s[u]={state:'active',pid:null,wd:f.wd};save(s);process.exit(0)}
|
|
if(fault(`start-dies-${id}`)){s[u]={state:'failed',pid:null,wd:f.wd};save(s);process.exit(0)}
|
|
snap('start');
|
|
const off=process.env.SD_CLOCK_OFFSET||'0';
|
|
// The holder must not inherit stack-release.sh's lock (fd 9) or any other fd above 2.
|
|
const close='for fd in /proc/self/fd/*; do n=${fd##*/}; [ "$n" -gt 2 ] && eval "exec $n>&-" 2>/dev/null; done; exec "$@"';
|
|
const ch=cp.spawn('bash',['-c',close,'holder',process.execPath,path.join(__dirname,'..','holder.cjs'),data,inst(u),sd,off],{detached:true,stdio:'ignore'});
|
|
ch.unref();
|
|
fs.appendFileSync(path.join(sd,'pids.log'),ch.pid+'\n');
|
|
const ready=path.join(sd,`ready-${ch.pid}`),end=Date.now()+10000;
|
|
while(!fs.existsSync(ready)){if(Date.now()>end){console.error('fixture holder did not start');process.exit(1)}sleep(20)}
|
|
s[u]={state:'active',pid:ch.pid,wd:f.wd};save(s);
|
|
if(f.exec.split(' ').slice(1).includes('--pm'))pm(inst(u),id,ch.pid,Number(off));
|
|
process.exit(0);
|
|
}
|
|
if(verb==='stop'){
|
|
const u=rest[0],s=load(),st=s[u];
|
|
if(st&&st.pid){
|
|
try{process.kill(st.pid,'SIGTERM')}catch{}
|
|
const end=Date.now()+10000;while(alive(st.pid)&&Date.now()<end)sleep(20);
|
|
if(alive(st.pid))try{process.kill(st.pid,'SIGKILL')}catch{}
|
|
}
|
|
if(st){s[u]={...st,state:'inactive',pid:null};save(s)}
|
|
if(isBus(u)){
|
|
// One-shot: the stop leaves an empty -wal or -shm beside the store, as an unclean close would.
|
|
for(const [name,ext] of [['stop-leaves-wal','-wal'],['stop-leaves-shm','-shm']])if(fault(name)){fs.rmSync(faultFile(name));fs.writeFileSync(path.join(data,'bus','bus.sqlite'+ext),'')}
|
|
snap('stop');
|
|
}
|
|
process.exit(0);
|
|
}
|
|
console.error(`fixture systemctl: unsupported: ${args.join(' ')}`);process.exit(1);
|
|
}
|
|
|
|
// journalctl --user -u UNIT --since @EPOCH -o cat. A bare local time gets exit 1, as M35 needs.
|
|
function journalctl(){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path');
|
|
const sd=process.env.SD_STATE,a=process.argv.slice(2);let unit='',since='';
|
|
for(let i=0;i<a.length;i++){if(a[i]==='-u')unit=a[++i];else if(a[i]==='--since')since=a[++i]}
|
|
fs.appendFileSync(path.join(sd,'journal-since.log'),since+'\n');
|
|
if(fs.existsSync(path.join(sd,'faults','journal-fails')))process.exit(1);
|
|
let from;
|
|
if(/^@\d+$/.test(since))from=Number(since.slice(1));
|
|
else{const m=/^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d) UTC$/.exec(since);if(!m)process.exit(1);from=Date.parse(`${m[1]}T${m[2]}Z`)/1000}
|
|
let lines=[];try{lines=fs.readFileSync(path.join(sd,'journal.jsonl'),'utf8').split('\n').filter(Boolean).map((l)=>JSON.parse(l))}catch{}
|
|
for(const o of lines)if(o.unit===unit&&o.epoch>=from)console.log(o.text);
|
|
}
|
|
|
|
function systemdAnalyze(){
|
|
const path=process.getBuiltinModule('node:path');
|
|
if(process.argv[2]!=='--user'||process.argv[3]!=='unit-paths')process.exit(1);
|
|
console.log(path.join(process.env.XDG_CONFIG_HOME,'systemd','user'));
|
|
}
|
|
|
|
// The bus host: holds the store open in WAL mode until SIGTERM, as the broker does.
|
|
function holder(){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite');
|
|
const [data,business,sd,offset]=process.argv.slice(2),bus=path.join(data,'bus');
|
|
fs.mkdirSync(bus,{recursive:true,mode:0o700});
|
|
const db=new DatabaseSync(path.join(bus,'bus.sqlite'));
|
|
if(!db.prepare("SELECT count(*) AS n FROM sqlite_master WHERE name='meta'").get().n){
|
|
db.exec('PRAGMA journal_mode=WAL');
|
|
db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
|
|
db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
|
|
db.prepare("INSERT INTO meta VALUES ('schema_version', '3g')").run();
|
|
}
|
|
fs.writeFileSync(path.join(bus,'writer.lock'),String(process.pid));
|
|
const at=new Date(Date.now()-Number(offset||0)*1000).toISOString(),host=path.join(sd,'host.json');
|
|
fs.writeFileSync(host+'.tmp',JSON.stringify({host:{business,pid:process.pid,live:true,startedAt:at,notifier:'ok'},boot:{ok:true,business,at}}));
|
|
fs.renameSync(host+'.tmp',host);
|
|
process.on('SIGTERM',()=>{db.close();fs.rmSync(path.join(bus,'writer.lock'),{force:true});process.exit(0)});
|
|
fs.writeFileSync(path.join(sd,`ready-${process.pid}`),'');
|
|
setInterval(()=>{},1<<30);
|
|
}
|
|
|
|
// A copy's scripts/mosaic: bus status and launches list from what the holder and systemctl wrote.
|
|
function mosaicStub(){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path');
|
|
const id=path.basename(path.resolve(__dirname,'..')),sd=process.env.SD_STATE,a=process.argv.slice(2).join(' ');
|
|
const fault=(n)=>fs.existsSync(path.join(sd,'faults',n));
|
|
const alive=(pid)=>{try{const s=fs.readFileSync(`/proc/${pid}/stat`,'utf8');return s[s.lastIndexOf(')')+2]!=='Z'}catch{return false}};
|
|
if(a==='bus status --json'){
|
|
if(fault(`status-${id}`))process.exit(1);
|
|
let s={host:null,boot:null};
|
|
try{s=JSON.parse(fs.readFileSync(path.join(sd,'host.json'),'utf8'));s.host.live=alive(s.host.pid)}catch{}
|
|
console.log(JSON.stringify(s));
|
|
}else if(a==='bus status')console.log(`bus: fixture status for ${id}`);
|
|
else if(a==='launches list --json'){
|
|
if(fault(`launches-${id}`))process.exit(1);
|
|
let l=[];try{l=JSON.parse(fs.readFileSync(path.join(sd,'launches.json'),'utf8'))}catch{}
|
|
console.log(JSON.stringify(l));
|
|
}else{console.error(`fixture mosaic: ${a}`);process.exit(4)}
|
|
}
|
|
|
|
// A copy's migrate-schema.mjs: records the backups it saw, then sets 3h, or 3h-partial and exit 1.
|
|
function migrateStub(){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),{DatabaseSync}=process.getBuiltinModule('node:sqlite');
|
|
const data=process.argv[2],sd=process.env.SD_STATE,dir=path.join(data,'backups','stack');
|
|
const backups=fs.existsSync(dir)?fs.readdirSync(dir).sort().map((name)=>({name,sums:fs.existsSync(path.join(dir,name,'SHA256SUMS'))})):[];
|
|
fs.writeFileSync(path.join(sd,'migrate.json'),JSON.stringify({backups}));
|
|
const fail=fs.existsSync(path.join(sd,'faults','migrate-fails'));
|
|
const db=new DatabaseSync(path.join(data,'bus','bus.sqlite'));
|
|
db.prepare("UPDATE meta SET value=? WHERE key='schema_version'").run(fail?'3h-partial':'3h');db.close();
|
|
if(fail)process.exit(1);
|
|
console.log('fixture migrate: 3h');
|
|
}
|
|
|
|
// --- The module fixture: one repo, its copies and the stand-ins, shared by every test. ---
|
|
|
|
const base=realpathSync(mkdtempSync(join(tmpdir(),'stack-release-')));
|
|
after(()=>{execFileSync('chmod',['-R','u+w',base]);rmSync(base,{recursive:true,force:true})});
|
|
const repo=join(base,'repo'),COPIES=join(base,'copies'),SBIN=join(base,'sbin');
|
|
mkdirSync(repo);mkdirSync(COPIES);mkdirSync(SBIN);
|
|
writeFileSync(join(base,'package.json'),'{"type":"commonjs"}\n');
|
|
const tracked=execFileSync('git',['-C',ROOT,'ls-files','-co','--exclude-standard','--','scripts','packages','adapters','roles','RELEASE','package.json','package-lock.json',':!packages/*/tests',':!scripts/foundation'],{encoding:'utf8'}).trim().split('\n');
|
|
for(const f of tracked)if(existsSync(join(ROOT,f)))cpSync(join(ROOT,f),join(repo,f));
|
|
const pkg={name:'copy-fixture',version:'1.0.0'};
|
|
writeFileSync(join(repo,'package.json'),JSON.stringify(pkg));
|
|
writeFileSync(join(repo,'package-lock.json'),JSON.stringify({name:pkg.name,version:pkg.version,lockfileVersion:3,packages:{'':pkg}}));
|
|
git(repo,'init','-q','-b','refactor');git(repo,'config','gc.auto','0');git(repo,'add','.');git(repo,'commit','-qm','fixture');
|
|
const commit=(msg)=>{git(repo,'commit','-q','--allow-empty','-m',msg);return git(repo,'rev-parse','HEAD')};
|
|
// refactor: F A B C1 C, then M merging side branch S (from A). U sits on a branch nothing reaches.
|
|
const F=git(repo,'rev-parse','HEAD'),A=commit('row 901: A'),B=commit('row 902: B'),C1=commit('row 903: C1'),C=commit('row 904: C');
|
|
git(repo,'checkout','-q','-b','side',A);const S=commit('row 905: S');
|
|
git(repo,'checkout','-q','-b','unreached',A);const U=commit('row 906: U');
|
|
git(repo,'checkout','-q','refactor');git(repo,'merge','-q','--no-ff','-m','merge side','side');
|
|
const M=git(repo,'rev-parse','HEAD');git(repo,'update-ref','refs/remotes/origin/refactor',M);
|
|
// `to` false leaves out the `to:` lines (C1); a string puts that value in each one (A). The --plan
|
|
// steps line can't be read from either (T27).
|
|
const storeText=(v,froms,to=true)=>`export const SCHEMA_VERSION = '${v}';\nexport const MIGRATIONS = [\n${froms.map((f,i)=>` {\n from: '${f}',\n${to===false?'':` to: '${to===true?froms[i+1]??v:to}',\n`} run() {},\n },\n`).join('')}];\n`;
|
|
for(const [id,store] of [[F,'// a copy from before SCHEMA_VERSION\n'],[A,storeText('3g',['3f'],'3x')],[B,storeText('3g',['3f'])],[C1,storeText('3g',['3f'],false)],[C,storeText('3h',['3f','3g'])],[S,storeText('3g',['3f'])],[U,storeText('3g',['3f'])]]){
|
|
const d=join(COPIES,id);
|
|
for(const [f,text,mode] of [['scripts/mosaic',script(mosaicStub),0o755],['packages/webui/src/cli.mjs','// fixture Console, never run\n',0o644],['packages/bus/src/store.mjs',store,0o644],['packages/bus/src/migrate-schema.mjs',`(${migrateStub})();\n`,0o644]]){
|
|
mkdirSync(dirname(join(d,f)),{recursive:true});writeFileSync(join(d,f),text,{mode});
|
|
}
|
|
execFileSync('chmod',['-R','a-w',d]);
|
|
}
|
|
for(const [name,fn] of [['systemctl',systemctl],['journalctl',journalctl],['systemd-analyze',systemdAnalyze]])writeFileSync(join(SBIN,name),script(fn),{mode:0o755});
|
|
writeFileSync(join(base,'holder.cjs'),`(${holder})();\n`);
|
|
const SR=join(repo,'scripts','stack-release.sh');
|
|
|
|
// --- Per-test roots and helpers ---
|
|
|
|
const sha=(f)=>createHash('sha256').update(readFileSync(f)).digest('hex');
|
|
const lines=(f)=>existsSync(f)?readFileSync(f,'utf8').split('\n').filter(Boolean):[];
|
|
const calls=(e)=>lines(join(e.sd,'calls.log'));
|
|
const startsStops=(e,from)=>calls(e).slice(from).filter((c)=>/^(start|stop) /.test(c));
|
|
const logLines=(e)=>lines(e.log).map((l)=>JSON.parse(l));
|
|
const pointer=(e)=>existsSync(e.pointer)?JSON.parse(readFileSync(e.pointer,'utf8')):null;
|
|
const units=(e)=>JSON.parse(readFileSync(join(e.sd,'units.json'),'utf8'));
|
|
const snaps=(e)=>lines(join(e.sd,'snap.jsonl')).map((l)=>JSON.parse(l));
|
|
const fault=(e,name,text='')=>writeFileSync(join(e.sd,'faults',name),text);
|
|
const sysctl=(e,...a)=>execFileSync(join(SBIN,'systemctl'),['--user',...a],{env:e.env,encoding:'utf8'});
|
|
const edit=(f,from,to)=>{const s=readFileSync(f,'utf8');assert.ok(s.includes(from),`${f} has no ${from}`);writeFileSync(f,s.replace(from,to))};
|
|
const posts=(e)=>readdirSync(e.data).filter((n)=>n.startsWith('bus.post-attempt-'));
|
|
const backupSum=(e,act)=>readFileSync(join(e.data,'backups','stack',act,'SHA256SUMS'),'utf8').split(' ')[0];
|
|
const commitLines=(range)=>git(repo,'log','--oneline','--no-merges','--max-count=200',range).split('\n').map((l)=>` ${l}`);
|
|
// The units, the wrapper, the pointer and the store, by sha256.
|
|
const fp=(e)=>({bus:sha(join(e.units,'[email protected]')),console:sha(join(e.units,CON)),wrapper:sha(join(e.bin,'mosaic')),pointer:existsSync(e.pointer)?sha(e.pointer):null,store:sha(e.store)});
|
|
// The schema version: readOnly while the holder has the store open, immutable when it is stopped.
|
|
function storeVersion(file,held){
|
|
const db=held?new DatabaseSync(file,{readOnly:true}):new DatabaseSync(pathToFileURL(file).href+'?immutable=1',{readOnly:true});
|
|
try{return db.prepare("SELECT value FROM meta WHERE key='schema_version'").get().value}finally{db.close()}
|
|
}
|
|
// Every file and directory under p with size and mtime: T23 and T16 compare it before and after.
|
|
function tree(p){
|
|
const out=[];
|
|
const walk=(q)=>{let st;try{st=statSync(q)}catch{return}out.push(st.isDirectory()?`${q}/`:`${q} ${st.size} ${st.mtimeMs}`);if(st.isDirectory())for(const n of readdirSync(q).sort())walk(join(q,n))};
|
|
walk(p);return out;
|
|
}
|
|
// T17: what a run may never remove.
|
|
const kept=(e)=>[...tree(join(e.data,'backups','stack')),...tree(join(e.state,'stack-rollback')),...posts(e).flatMap((n)=>tree(join(e.data,n))),...tree(COPIES)].map((l)=>l.replace(/ .*/,''));
|
|
|
|
async function mk(t){
|
|
const root=realpathSync(mkdtempSync(join(tmpdir(),'sr-')));
|
|
const e={root,data:join(root,'data'),bin:join(root,'bin'),sd:join(root,'sd'),units:join(root,'xcfg','systemd','user'),cfg:join(root,'cfg','config.json'),claude:join(root,'claude-bin','claude')};
|
|
Object.assign(e,{state:join(e.data,'state'),pointer:join(e.data,'state','stack-active.json'),log:join(e.data,'state','stack-activation-log.jsonl'),store:join(e.data,'bus','bus.sqlite'),lock:join(e.data,'state','stack-release.lock')});
|
|
for(const d of [join(root,'home'),dirname(e.cfg),join(e.data,'bus'),e.bin,dirname(e.claude),join(e.sd,'faults'),e.units,join(root,'xdata')])mkdirSync(d,{recursive:true});
|
|
chmodSync(join(e.data,'bus'),0o700);
|
|
writeFileSync(e.cfg,JSON.stringify({configVersion:1,environment:'development',dataRoot:e.data,execution:{backend:'docker',provider:'zai',model:'glm-5.3-flash',adapter:'mock'}}));
|
|
writeFileSync(join(e.bin,'mosaic'),`#!/usr/bin/env bash\n${MARKER}\n[ "\${1:-}" = __lease-capability ] && exec "${e.bin}/mosaic-old" "$@"\nexec "/old/scripts/mosaic" "$@"\n`,{mode:0o755});
|
|
writeFileSync(join(e.bin,'mosaic-old'),'#!/bin/sh\nexit 0\n',{mode:0o755});
|
|
writeFileSync(e.claude,'#!/bin/sh\necho "9.9.9 (Claude Code fixture)"\n',{mode:0o755});
|
|
e.env={PATH:`${SBIN}:${dirname(NODE)}:/usr/bin:/bin`,HOME:join(root,'home'),XDG_CONFIG_HOME:join(root,'xcfg'),XDG_DATA_HOME:join(root,'xdata'),MOSAIC_CONFIG:e.cfg,SD_STATE:e.sd,SD_DATA:e.data,TMPDIR:tmpdir()};
|
|
// The Console's /healthz: 200 while the stand-in has it active and no healthz-<copy> fault names its copy.
|
|
const server=createServer((q,r)=>{
|
|
let st={};try{st=units(e)[CON]||{}}catch{}
|
|
const ok=q.url==='/healthz'&&st.state==='active'&&!existsSync(join(e.sd,'faults',`healthz-${basename(st.wd||'')}`));
|
|
r.writeHead(ok?200:500);r.end();
|
|
});
|
|
server.listen(0,'127.0.0.1');await once(server,'listening');e.port=String(server.address().port);
|
|
t.after(()=>{
|
|
for(const pid of lines(join(e.sd,'pids.log')).map(Number)){
|
|
try{if(readFileSync(`/proc/${pid}/cmdline`,'utf8').includes(join(base,'holder.cjs')))process.kill(pid,'SIGKILL')}catch{}
|
|
}
|
|
server.close();execFileSync('chmod',['-R','u+w',root]);rmSync(root,{recursive:true,force:true});
|
|
});
|
|
return e;
|
|
}
|
|
|
|
async function run(e,...args){
|
|
const before=existsSync(e.data)?kept(e):[];
|
|
const ch=spawn('bash',[SR,...args,'--copies',COPIES,'--bin',e.bin],{env:e.env,stdio:['ignore','pipe','pipe']});
|
|
let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d});
|
|
const timer=setTimeout(()=>ch.kill('SIGKILL'),120000);
|
|
const [code]=await once(ch,'close');clearTimeout(timer);
|
|
const now=new Set(kept(e));
|
|
for(const p of before)assert.ok(now.has(p),`T17: ${args.join(' ')} removed ${p}`);
|
|
return {code,out,err};
|
|
}
|
|
|
|
// The stack on copy `copy` for business acme, both units active. pointer: adopt (a real adopt and
|
|
// its log line), hand (written here, no log) or none. Snapshots start after the seed.
|
|
async function seed(e,{copy=B,pm=false,store='3g',pointer:how='adopt'}={}){
|
|
const db=new DatabaseSync(e.store);
|
|
db.exec('PRAGMA journal_mode=WAL');
|
|
db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
|
|
db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
|
|
db.prepare("INSERT INTO meta VALUES ('schema_version', ?)").run(store);
|
|
for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`);
|
|
db.close();
|
|
const sh=(name,...a)=>execFileSync('bash',[join(repo,'scripts',name),...a],{env:e.env,encoding:'utf8',stdio:['ignore','pipe','pipe']});
|
|
sh('bus-service.sh','install','--copy',copy,'--copies',COPIES,...(pm?['--pm']:[]),'--claude',e.claude);
|
|
sh('webui-service.sh','install','--copy',copy,'--copies',COPIES,'--port',e.port,'--business','acme');
|
|
sh('stack-copy.sh','wrapper',copy,'--copies',COPIES,'--bin',e.bin);
|
|
// The seeded host started two minutes ago, so its boot status and PM run predate any run's start.
|
|
execFileSync(join(SBIN,'systemctl'),['--user','start',BUS],{env:{...e.env,SD_CLOCK_OFFSET:'120'}});
|
|
sysctl(e,'start',CON);
|
|
if(how==='adopt'){const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err)}
|
|
else if(how==='hand'){
|
|
mkdirSync(e.state,{recursive:true});
|
|
writeFileSync(e.pointer,JSON.stringify({copy,business:'acme',pm,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g',activation:'manual',at:'2026-10-18T00:00:00Z'})+'\n');
|
|
}
|
|
writeFileSync(join(e.sd,'snap.jsonl'),'');
|
|
}
|
|
|
|
const rolledBack=(code,to=B)=>`stack-release: ${code}; rolled back to ${to}, which passed the gate\n`;
|
|
const planTail=(e,to,pm=false)=>[`bus: scripts/bus-service.sh install --copy ${to} --copies ${COPIES}${pm?' --pm':''} --claude ${e.claude}`,`Console: scripts/webui-service.sh install --copy ${to} --copies ${COPIES} --port ${e.port} --business acme`,`CLI: scripts/stack-copy.sh wrapper ${to} --copies ${COPIES} --bin ${e.bin}`];
|
|
|
|
// --- The T table ---
|
|
|
|
test('T1 adopt writes the pointer from the units once; a failing check refuses check-failed',async(t)=>{
|
|
const e=await mk(t);await seed(e,{pointer:'none'});
|
|
const r=await run(e,'adopt','acme');assert.equal(r.code,0,r.err);
|
|
assert.equal(r.out,`adopted ${B} for acme (pm 0, schema 3g)\n`);
|
|
const {at,activation,...p}=pointer(e);
|
|
assert.deepEqual(p,{copy:B,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'});
|
|
assert.match(activation,new RegExp(`^\\d{8}T\\d{6}Z-${B.slice(0,12)}$`));assert.match(at,/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/);
|
|
assert.deepEqual(logLines(e).map((l)=>[l.op,l.from,l.to,l.result]),[['adopt',null,B,'ok']]);
|
|
const again=await run(e,'adopt','acme');assert.equal(again.code,3);
|
|
assert.equal(again.err,`stack-release: refusing already-adopted: ${e.pointer} exists (copy ${B})\n`);
|
|
assert.equal(logLines(e).at(-1).code,'already-adopted');
|
|
const f=await mk(t);await seed(f,{pointer:'none'});fault(f,`check-${B}`);
|
|
const bad=await run(f,'adopt','acme');assert.equal(bad.code,3);
|
|
assert.match(bad.err,/^stack-release: refusing check-failed: stack-copy.sh check did not pass:\nbus: /);
|
|
assert.equal(pointer(f),null);
|
|
});
|
|
|
|
test('T2 activate installs exactly the pointer\'s pm, claude and Console flags',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err+r.out);
|
|
assert.ok(r.out.endsWith(`stack-release: activate ok: ${C1} is live for acme\n`),r.out);
|
|
const bus=readFileSync(join(e.units,'[email protected]'),'utf8').split('\n'),con=readFileSync(join(e.units,CON),'utf8').split('\n');
|
|
assert.ok(bus.includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i`),bus.join('\n'));
|
|
assert.ok(bus.includes(`Environment=MOSAIC_CLAUDE_BIN=${e.claude}`));
|
|
const exec=con.find((l)=>l.startsWith('ExecStart='));
|
|
assert.ok(exec.endsWith(` ${COPIES}/${C1}/packages/webui/src/cli.mjs serve --checkout ${repo} --port ${e.port} --business acme`),exec);
|
|
assert.equal(readFileSync(join(e.bin,'mosaic'),'utf8').trimEnd().split('\n').at(-1),`exec "${COPIES}/${C1}/scripts/mosaic" "$@"`);
|
|
const {at,activation,...p}=pointer(e);
|
|
assert.deepEqual(p,{copy:C1,business:'acme',pm:false,claude:e.claude,console:['--port',e.port,'--business','acme'],schema:'3g'});
|
|
});
|
|
|
|
test('T3 a hand-edited unit refuses unit-flags-drift, names the difference and changes nothing',async(t)=>{
|
|
const cases=[
|
|
['pm',(e)=>edit(join(e.units,'[email protected]'),' bus start %i\n',' bus start %i --pm\n'),()=>' pm: pointer 0, unit 1'],
|
|
['claude',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'claude.conf'),'[Service]\nEnvironment=MOSAIC_CLAUDE_BIN=/opt/other/claude\n')},(e)=>` claude: pointer ${e.claude}, unit /opt/other/claude`],
|
|
['console',(e)=>edit(join(e.units,CON),'--business acme\n','--business acme --board http://127.0.0.1:9\n'),(e)=>` console: pointer --port ${e.port} --business acme, unit --port ${e.port} --business acme --board http://127.0.0.1:9`],
|
|
];
|
|
for(const [name,change,line] of cases)await t.test(name,async(t)=>{
|
|
const e=await mk(t);await seed(e);change(e);
|
|
const before=fp(e),n=calls(e).length,logs=logLines(e).length;
|
|
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,3);
|
|
assert.equal(r.err,`stack-release: refusing unit-flags-drift: the units differ from the pointer; settle it first:\n${line(e)}\n`);
|
|
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
|
|
assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'unit-flags-drift');
|
|
});
|
|
});
|
|
|
|
test('T4 each item 3 refusal changes nothing and logs one line with its code',async(t)=>{
|
|
const setUnit=(e,u,state)=>{const s=units(e);s[u]={...(s[u]||{pid:null,wd:''}),state};writeFileSync(join(e.sd,'units.json'),JSON.stringify(s))};
|
|
const cases=[
|
|
['copy-missing',()=>M,()=>`no copy ${M} in ${COPIES}`],
|
|
['copy-unreachable',()=>U,()=>`${U} is not reachable from refactor and origin/refactor`],
|
|
['copy-schema-unknown',()=>F,()=>`cannot read SCHEMA_VERSION and its chain from ${COPIES}/${F}/packages/bus/src/store.mjs`],
|
|
['already-active',()=>B,()=>`${B} is the pointer's copy`],
|
|
['instance-busy',(e)=>{setUnit(e,'[email protected]','active');return C1},()=>'[email protected] is active; the copy and wrapper switch reach it too','another business active'],
|
|
['instance-busy',(e)=>{setUnit(e,BUS,'activating');return C1},()=>`${BUS} is activating`,'this business activating'],
|
|
['drop-in-override',(e)=>{const d=join(e.units,`${BUS}.d`);mkdirSync(d);writeFileSync(join(d,'override.conf'),'[Service]\nWorkingDirectory=/elsewhere\n');return C1},(e)=>`${join(e.units,`${BUS}.d`,'override.conf')} overrides ExecStart or WorkingDirectory; the operator removes it as in TOOLS.md step 4`],
|
|
['no-pointer',()=>C1,(e)=>`no ${e.pointer}; run adopt first`,'',{pointer:'none'}],
|
|
];
|
|
for(const [code,setup,msg,label,opts] of cases)await t.test(label?`${code} (${label})`:code,async(t)=>{
|
|
const e=await mk(t);await seed(e,opts);const id=setup(e);
|
|
const before=fp(e),n=calls(e).length,logs=logLines(e).length;
|
|
const r=await run(e,'activate',id,'acme');assert.equal(r.code,3,r.err);
|
|
assert.equal(r.err,`stack-release: refusing ${code}: ${msg(e)}\n`);
|
|
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
|
|
assert.equal(logLines(e).length,logs+1);assert.deepEqual([logLines(e).at(-1).result,logLines(e).at(-1).code],['refused',code]);
|
|
});
|
|
});
|
|
|
|
test('T5 an activate with no migration stops the Console then the bus, installs, starts the bus then the Console',async(t)=>{
|
|
const e=await mk(t);await seed(e);const n=calls(e).length;
|
|
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.err);
|
|
assert.deepEqual(startsStops(e,n),[`stop ${CON}`,`stop ${BUS}`,`start ${BUS}`,`start ${CON}`]);
|
|
const c=calls(e).slice(n),reload=c.indexOf('daemon-reload');
|
|
assert.ok(reload>c.indexOf(`stop ${BUS}`)&&reload<c.indexOf(`start ${BUS}`),'daemon-reload between the stops and the starts');
|
|
assert.equal(pointer(e).copy,C1);assert.equal(units(e)[BUS].wd,join(COPIES,C1));
|
|
assert.ok(!existsSync(join(e.data,'backups','stack')));
|
|
const l=logLines(e).slice(-2);assert.deepEqual(l.map((x)=>[x.op,x.from,x.to,x.result]),[['activate',B,C1,'started'],['activate',B,C1,'ok']]);
|
|
});
|
|
|
|
test('T6 a migration without --migrate rolls back with migration-needs-flag',async(t)=>{
|
|
const e=await mk(t);await seed(e);const before=fp(e);
|
|
const r=await run(e,'activate',C,'acme','--timeout','3');assert.equal(r.code,1,r.out);
|
|
assert.equal(r.err,rolledBack('migration-needs-flag'));
|
|
assert.deepEqual(fp(e),before);
|
|
assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd,units(e)[CON].state],['active',join(COPIES,B),'active']);
|
|
const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','migration-needs-flag','ok']);
|
|
assert.equal(storeVersion(e.store,true),'3g');assert.ok(!existsSync(join(e.data,'backups','stack')));
|
|
});
|
|
|
|
test('T7 --migrate takes the backup before the migration runs',async(t)=>{
|
|
const e=await mk(t);await seed(e);const before=fp(e);
|
|
const r=await run(e,'activate',C,'acme','--migrate');assert.equal(r.code,0,r.err+r.out);
|
|
const act=pointer(e).activation;
|
|
assert.deepEqual(JSON.parse(readFileSync(join(e.sd,'migrate.json'),'utf8')),{backups:[{name:act,sums:true}]});
|
|
assert.equal(backupSum(e,act),before.store);
|
|
assert.deepEqual([pointer(e).copy,pointer(e).schema],[C,'3h']);assert.equal(storeVersion(e.store,true),'3h');
|
|
assert.equal(logLines(e).at(-1).backup,act);
|
|
});
|
|
|
|
test('T8 a failed migration restores the backup and keeps the store it replaced',async(t)=>{
|
|
const e=await mk(t);await seed(e);const before=fp(e);fault(e,'migrate-fails');
|
|
const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out);
|
|
assert.equal(r.err,rolledBack('migrate-failed'));
|
|
assert.equal(posts(e).length,1);assert.equal(storeVersion(join(e.data,posts(e)[0],'bus.sqlite'),false),'3h-partial');
|
|
const act=logLines(e).at(-1).activation;
|
|
assert.equal(sha(e.store),backupSum(e,act));assert.equal(sha(e.store),before.store);
|
|
assert.deepEqual(fp(e),before);assert.equal(units(e)[BUS].wd,join(COPIES,B));
|
|
});
|
|
|
|
test('T9 a store outside the target\'s chain rolls back with store-newer',async(t)=>{
|
|
const e=await mk(t);await seed(e,{store:'3z'});const before=fp(e);
|
|
const r=await run(e,'activate',C,'acme','--migrate','--timeout','3');assert.equal(r.code,1,r.out);
|
|
assert.equal(r.err,rolledBack('store-newer'));
|
|
assert.deepEqual(fp(e),before);assert.equal(storeVersion(e.store,true),'3z');assert.ok(!existsSync(join(e.data,'backups','stack')));
|
|
});
|
|
|
|
test('T10 each gate check failing alone rolls back and puts every file back',async(t)=>{
|
|
const cases=[['units-inactive',`start-dies-${C1}`],['bus-status',`status-${C1}`],['healthz',`healthz-${C1}`],['copy-check',`check-${C1}`],['pm-launch-unconfirmed',`pm-${C1}`,'none']];
|
|
for(const [code,name,text] of cases)await t.test(code,async(t)=>{
|
|
const pm=code==='pm-launch-unconfirmed',e=await mk(t);await seed(e,{pm});fault(e,name,text);
|
|
const before=fp(e);
|
|
const r=await run(e,'activate',C1,'acme',...(pm?['--pm']:[]),'--timeout','3');assert.equal(r.code,1,r.out+r.err);
|
|
assert.equal(r.err,rolledBack(code));assert.ok(r.out.includes(`stack-release: gate failed on ${C1}: ${code}`),r.out);
|
|
assert.deepEqual(fp(e),before);
|
|
assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,B)]);
|
|
const l=logLines(e).at(-1);assert.deepEqual([l.result,l.code,l.rollback],['gate-failed',code,'ok']);
|
|
});
|
|
});
|
|
|
|
test('T10b the PM check needs exactly one launched line for a live run of this start',async(t)=>{
|
|
const cases=[
|
|
['no launched line','none',/\(no launched line since the start\)/],
|
|
['two lines','two',/\(2 launched lines since the start\)/],
|
|
['a changed word','badword',/\(the launched line does not parse: launched pm-acme: job run-\d+, claude opus\)/],
|
|
['run not in launches list','notlisted',/\(run run-\d+ is not in launches list\)/],
|
|
['run started before the start','early',/\(run run-\d+ started before this start\)/],
|
|
['a human launch with no journal line','human',/\(no launched line since the start\)/],
|
|
['run for another business','other',/\(run run-\d+ is for other\)/],
|
|
['run not live','dead',/\(run run-\d+ is not live\)/],
|
|
];
|
|
for(const [name,mode,why] of cases)await t.test(name,async(t)=>{
|
|
const e=await mk(t);await seed(e,{pm:true});fault(e,`pm-${C1}`,mode);
|
|
const r=await run(e,'activate',C1,'acme','--pm','--timeout','3');assert.equal(r.code,1,r.out+r.err);
|
|
assert.equal(r.err,rolledBack('pm-launch-unconfirmed'));
|
|
const gate=r.out.split('\n').find((l)=>l.startsWith(`stack-release: gate failed on ${C1}: pm-launch-unconfirmed`));
|
|
assert.ok(gate,r.out);assert.match(gate,why);
|
|
});
|
|
await t.test('one well-formed line and a live run pass',async(t)=>{
|
|
const e=await mk(t);await seed(e,{pm:true});
|
|
const r=await run(e,'activate',C1,'acme','--pm');assert.equal(r.code,0,r.out+r.err);
|
|
assert.ok(readFileSync(join(e.units,'[email protected]'),'utf8').includes(`ExecStart=${COPIES}/${C1}/scripts/mosaic bus start %i --pm\n`));
|
|
assert.match(lines(join(e.sd,'journal-since.log')).at(-1),/^@\d+$/);
|
|
assert.equal(pointer(e).pm,true);
|
|
});
|
|
});
|
|
|
|
test('T11 a rollback whose gate fails too: rollback-failed, exit 2, both units stopped, no retry',async(t)=>{
|
|
const e=await mk(t);await seed(e);fault(e,`healthz-${C1}`);fault(e,`healthz-${B}`);const n=calls(e).length;
|
|
const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,2,r.out+r.err);
|
|
const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation);
|
|
assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with healthz; both units are stopped. Rollback set: ${rs}\n`);
|
|
assert.deepEqual([l.result,l.code,l.rollback,l.rollbackCode],['rollback-failed','healthz','failed','healthz']);
|
|
assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']);
|
|
const ss=startsStops(e,n);
|
|
assert.deepEqual(ss.filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`,`start ${BUS}`,`start ${CON}`]);
|
|
assert.deepEqual(ss.slice(-2),[`stop ${CON}`,`stop ${BUS}`]);
|
|
assert.equal(pointer(e).copy,B);
|
|
});
|
|
|
|
test('T12 rollback takes the previous copy from the log',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
assert.equal((await run(e,'activate',C1,'acme')).code,0);
|
|
const r=await run(e,'rollback','acme');assert.equal(r.code,0,r.out+r.err);
|
|
assert.ok(r.out.endsWith(`stack-release: rollback ok: ${B} is live for acme\n`),r.out);
|
|
assert.equal(pointer(e).copy,B);
|
|
const l=logLines(e).at(-1);assert.deepEqual([l.op,l.from,l.to,l.result],['rollback',C1,B,'ok']);
|
|
});
|
|
|
|
test('T13 a rollback across a migration needs --restore, then restores and goes live',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store);
|
|
const no=await run(e,'rollback','acme');assert.equal(no.code,3);
|
|
assert.equal(no.err,`stack-release: refusing schema-differs: the store is at 3h and ${B} expects 3g; pass --restore ACTIVATION\n`);
|
|
const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err);
|
|
const o=join(e.data,'backups','stack',act);
|
|
assert.match(r.out,new RegExp(`^restore: ${o}; the store now in place moves to ${e.data}/bus\\.post-attempt-\\d{8}T\\d{6}Z and is kept; 0 event\\(s\\) were written after the backup\\n`));
|
|
assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(storeVersion(e.store,true),'3g');
|
|
assert.equal(sha(e.store),backupSum(e,act));
|
|
assert.equal(posts(e).length,1);assert.equal(sha(join(e.data,posts(e)[0],'bus.sqlite')),storeC);
|
|
assert.equal(logLines(e).at(-1).backup,act);
|
|
});
|
|
|
|
test('T14 events written after the backup need --events N with the right N',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation;
|
|
const db=new DatabaseSync(e.store);for(let i=0;i<2;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run('2999-01-01T00:00:00.000Z');db.close();
|
|
const want='stack-release: refusing post-backup-events: 2 event(s) were written after the backup; Jason decides, then --events 2\n';
|
|
for(const extra of [[],['--events','1']]){
|
|
const before=fp(e),r=await run(e,'rollback','acme','--restore',act,...extra);
|
|
assert.equal(r.code,3,extra.join(' '));assert.equal(r.err,want);assert.deepEqual(fp(e),before);
|
|
}
|
|
const r=await run(e,'rollback','acme','--restore',act,'--events','2');assert.equal(r.code,0,r.out+r.err);
|
|
assert.match(r.out,/; 2 event\(s\) were written after the backup\n/);
|
|
assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);
|
|
});
|
|
|
|
test('T14b a rollback --restore whose gate fails stops both units and skips item 6',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation,storeC=sha(e.store),before=fp(e);
|
|
fault(e,`healthz-${B}`);const n=calls(e).length;
|
|
const r=await run(e,'rollback','acme','--restore',act,'--timeout','3');assert.equal(r.code,2,r.out+r.err);
|
|
const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation),post=join(e.data,posts(e)[0]);
|
|
assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with not-attempted; both units are stopped. Rollback set: ${rs}, backup ${join(e.data,'backups','stack',act)}, post-attempt store ${post}\n`);
|
|
assert.deepEqual([l.result,l.code,l.rollbackCode],['rollback-failed','healthz','not-attempted']);
|
|
assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']);
|
|
assert.equal(posts(e).length,1);assert.equal(sha(join(post,'bus.sqlite')),storeC);assert.equal(sha(e.store),backupSum(e,act));
|
|
assert.deepEqual(startsStops(e,n).filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`]);
|
|
assert.equal(pointer(e).copy,C);assert.equal(sha(e.pointer),before.pointer);
|
|
});
|
|
|
|
test('T14c a rollback without --restore whose gate fails runs item 6',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
assert.equal((await run(e,'activate',C1,'acme')).code,0);const before=fp(e);
|
|
fault(e,`healthz-${B}`);
|
|
const r=await run(e,'rollback','acme','--timeout','3');assert.equal(r.code,1,r.out+r.err);
|
|
assert.equal(r.err,rolledBack('healthz',C1));
|
|
const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.code,l.rollback],['rollback','gate-failed','healthz','ok']);
|
|
assert.deepEqual(fp(e),before);assert.deepEqual([units(e)[BUS].state,units(e)[BUS].wd],['active',join(COPIES,C1)]);
|
|
assert.deepEqual(readdirSync(join(e.data,'bus')).filter((n)=>n.startsWith('bus.sqlite')).sort(),['bus.sqlite','bus.sqlite-shm','bus.sqlite-wal']);
|
|
});
|
|
|
|
test('T15 --plan names the migration and the restore and changes nothing',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const quiet=async(...args)=>{
|
|
const before=fp(e),n=calls(e).length,logs=logLines(e).length,r=await run(e,...args);
|
|
assert.equal(r.code,0,r.err);assert.equal(r.err,'');
|
|
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs);
|
|
return r.out;
|
|
};
|
|
assert.ok((await quiet('activate',C,'acme','--plan')).includes('needs: --migrate (not given; the run would roll back with migration-needs-flag)\n'));
|
|
assert.equal((await run(e,'activate',C,'acme','--migrate')).code,0);const act=pointer(e).activation;
|
|
assert.equal(await quiet('rollback','acme','--plan'),[
|
|
'plan: rollback for acme',`from: ${C}`,`to: ${B}`,
|
|
'store schema: 3h (read from the store the running bus holds)','target schema: 3g',
|
|
`restore: the store must go back to 3g; needs --restore ${act}`,
|
|
'commits: the run takes away 2 (newest first, at most 200):',...commitLines(`${B}..${C}`),
|
|
...planTail(e,B),''].join('\n'));
|
|
});
|
|
|
|
test('T26 --plan lists the commits an activate adds, after the migration lines',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const head=(mig)=>['plan: activate for acme',`from: ${B}`,`to: ${C}`,'store schema: 3g (read from the store the running bus holds)','target schema: 3h',
|
|
'migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h',`needs: --migrate (${mig})`,'needs: Jason present for this run (decision 96 condition 2)'];
|
|
const adds=commitLines(`${B}..${C}`);
|
|
assert.equal(adds.length,2);assert.match(adds[0],/^ {2}[0-9a-f]{7,} row 904: C$/);assert.match(adds[1],/^ {2}[0-9a-f]{7,} row 903: C1$/);
|
|
for(const [flags,mig] of [[[],'not given; the run would roll back with migration-needs-flag'],[['--migrate'],'given']]){
|
|
const r=await run(e,'activate',C,'acme','--plan',...flags);assert.equal(r.code,0,r.err);
|
|
assert.equal(r.out,[...head(mig),'commits: the run adds 2 (newest first, at most 200):',...adds,...planTail(e,C),''].join('\n'));
|
|
}
|
|
const same=await run(e,'activate',C1,'acme','--plan');assert.equal(same.code,0,same.err);
|
|
assert.ok(same.out.includes('target schema: 3g\nmigration: none; the store is at the target\'s version\ncommits: the run adds 1 (newest first, at most 200):\n'),same.out);
|
|
});
|
|
|
|
test('T27 --plan names each migration step, or says the steps cannot be read',async(t)=>{
|
|
const e=await mk(t);await seed(e,{store:'3f',pointer:'hand'});
|
|
const two=await run(e,'activate',C,'acme','--plan');assert.equal(two.code,0,two.err);
|
|
assert.ok(two.out.includes('\nmigration: 3f to 3h, after a backup of the stopped store\nmigration steps: 3f to 3g, 3g to 3h\nneeds: --migrate'),two.out);
|
|
const none=await run(e,'activate',C1,'acme','--plan');assert.equal(none.code,0,none.err);
|
|
assert.ok(none.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${C1}/packages/bus/src/store.mjs\nneeds: --migrate`),none.out);
|
|
const short=await run(e,'activate',A,'acme','--plan');assert.equal(short.code,0,short.err);
|
|
assert.ok(short.out.includes(`\nmigration: 3f to 3g, after a backup of the stopped store\nmigration steps: cannot be read from ${COPIES}/${A}/packages/bus/src/store.mjs\nneeds: --migrate`),short.out);
|
|
});
|
|
|
|
test('T25 --plan refuses copies-unrelated when neither copy is an ancestor of the other',async(t)=>{
|
|
const e=await mk(t);await seed(e,{pointer:'hand'});
|
|
const r=await run(e,'activate',S,'acme','--plan');assert.equal(r.code,3);
|
|
assert.equal(r.err,`stack-release: refusing copies-unrelated: neither ${B} nor ${S} is an ancestor of the other, so the plan cannot list the commits\n`);
|
|
assert.equal(r.out,'');assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock));
|
|
});
|
|
|
|
test('T23 --plan reads the store without changing a file, stopped or held',async(t)=>{
|
|
const want=['target schema: 3h','migration: 3g to 3h, after a backup of the stopped store','migration steps: 3g to 3h','needs: --migrate (not given; the run would roll back with migration-needs-flag)','needs: Jason present for this run (decision 96 condition 2)'];
|
|
for(const [name,stop,src] of [['stopped store',true,'read from the stopped store'],['held store',false,'read from the store the running bus holds']])await t.test(name,async(t)=>{
|
|
const e=await mk(t);await seed(e,{pointer:'hand'});if(stop)sysctl(e,'stop',BUS);
|
|
const before=tree(e.data),units0=[sha(join(e.units,'[email protected]')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],store0=sha(e.store);
|
|
const r=await run(e,'activate',C,'acme','--plan');assert.equal(r.code,0,r.err);
|
|
const out=r.out.split('\n');
|
|
assert.ok(out.includes(`store schema: 3g (${src})`),r.out);
|
|
assert.deepEqual(out.slice(out.indexOf('target schema: 3h'),out.indexOf('target schema: 3h')+5),want);
|
|
assert.deepEqual(tree(e.data),before);assert.equal(sha(e.store),store0);
|
|
assert.deepEqual([sha(join(e.units,'[email protected]')),sha(join(e.units,CON)),sha(join(e.bin,'mosaic'))],units0);
|
|
assert.ok(!existsSync(e.log));assert.ok(!existsSync(e.lock));
|
|
if(stop)assert.deepEqual(readdirSync(join(e.data,'bus')),['bus.sqlite']);
|
|
});
|
|
});
|
|
|
|
test('T24 business-mismatch and capture-failed refuse with exit 3; a short id is a usage error',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
let n=calls(e).length,logs=logLines(e).length;
|
|
const r=await run(e,'activate',C1,'other');assert.equal(r.code,3);
|
|
assert.equal(r.err,'stack-release: refusing business-mismatch: the pointer is for acme\n');
|
|
assert.equal(logLines(e).length,logs+1);assert.equal(logLines(e).at(-1).code,'business-mismatch');assert.deepEqual(startsStops(e,n),[]);
|
|
const sets=join(e.state,'stack-rollback');mkdirSync(sets);chmodSync(sets,0o555);
|
|
n=calls(e).length;logs=logLines(e).length;const before=fp(e);
|
|
const c=await run(e,'activate',C1,'acme');assert.equal(c.code,3);
|
|
assert.match(c.err,new RegExp(`^stack-release: refusing capture-failed: could not write the rollback set ${sets}/\\d{8}T\\d{6}Z-${C1.slice(0,12)}; nothing was stopped\\n$`));
|
|
const l=logLines(e).slice(logs);
|
|
assert.deepEqual(l.map((x)=>[x.result,x.code]),[['started',undefined],['refused','capture-failed']]);assert.equal(l[0].activation,l[1].activation);
|
|
assert.deepEqual(startsStops(e,n),[]);assert.deepEqual(fp(e),before);
|
|
const s=await run(e,'status','acme');assert.equal(s.code,0,s.err);assert.ok(!s.out.includes('interrupted run'),s.out);
|
|
logs=logLines(e).length;
|
|
const u=await run(e,'activate','abc123','acme');assert.equal(u.code,4);assert.match(u.err,/^usage: stack-release.sh adopt BUSINESS\n/);
|
|
assert.equal(logLines(e).length,logs);
|
|
});
|
|
|
|
test('T16 status prints the pointer, the log, the check, bus status and drift, and writes nothing',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const before=tree(e.data),fp0=fp(e);
|
|
const r=await run(e,'status','acme');assert.equal(r.code,0,r.err);
|
|
const copy=join(COPIES,B);
|
|
assert.equal(r.out,[
|
|
`pointer: ${e.pointer}`,...readFileSync(e.pointer,'utf8').trimEnd().split('\n').map((l)=>` ${l}`),
|
|
'log (last five lines):',...lines(e.log).map((l)=>` ${l}`),
|
|
'check:',` bus: ${copy}`,` Console: ${copy}`,` CLI: ${copy}`,' result: pass',
|
|
'bus status:',` bus: fixture status for ${B}`,
|
|
'unit flags: match the pointer',
|
|
'drift: 4 commit(s) on refactor since the pointer\'s copy',''].join('\n'));
|
|
assert.deepEqual(tree(e.data),before);assert.deepEqual(fp(e),fp0);
|
|
});
|
|
|
|
test('T19 a started line with no final line: status shows it and every changing verb refuses',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const act=`20261018T000000Z-${C1.slice(0,12)}`;
|
|
appendFileSync(e.log,JSON.stringify({at:'2026-10-18T00:00:00Z',op:'activate',from:B,to:C1,backup:null,activation:act,result:'started'})+'\n');
|
|
const s=await run(e,'status','acme');assert.equal(s.code,0);
|
|
assert.ok(s.out.includes(`interrupted run: ${act} (started, no final line; Jason or Mos settles it)\n`),s.out);
|
|
for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){
|
|
const before=fp(e),n=calls(e).length,r=await run(e,...args);
|
|
assert.equal(r.code,3,args.join(' '));
|
|
assert.equal(r.err,`stack-release: refusing interrupted-run: activation ${act} started and has no final line; Jason or Mos settles it (status shows it)\n`);
|
|
assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]);
|
|
}
|
|
});
|
|
|
|
test('T20 a held run lock refuses run-locked without a log line; status and --plan still run',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const h=spawn('flock',[e.lock,'sleep','30'],{detached:true,stdio:'ignore'});
|
|
t.after(()=>{try{process.kill(-h.pid,'SIGKILL')}catch{}});
|
|
const end=Date.now()+10000;
|
|
while(spawnSync('flock',['-n',e.lock,'true']).status!==1){assert.ok(Date.now()<end,'the lock holder did not take the lock');await new Promise((r)=>setTimeout(r,50))}
|
|
for(const args of [['adopt','acme'],['activate',C1,'acme'],['rollback','acme']]){
|
|
const before=fp(e),logs=logLines(e).length,r=await run(e,...args);
|
|
assert.equal(r.code,3,args.join(' '));
|
|
assert.equal(r.err,`stack-release: refusing run-locked: another stack-release run holds ${e.lock}\n`);
|
|
assert.equal(logLines(e).length,logs);assert.deepEqual(fp(e),before);
|
|
}
|
|
assert.equal((await run(e,'status','acme')).code,0);
|
|
const p=await run(e,'activate',C1,'acme','--plan');assert.equal(p.code,0,p.err);
|
|
});
|
|
|
|
test('T21 a bus unit with no MOSAIC_CLAUDE_BIN: adopt and activate refuse no-claude-pin',async(t)=>{
|
|
const drop=(e)=>edit(join(e.units,'[email protected]'),`Environment=MOSAIC_CLAUDE_BIN=${e.claude}\n`,'');
|
|
const want=`stack-release: refusing no-claude-pin: ${BUS} has no MOSAIC_CLAUDE_BIN\n`;
|
|
const a=await mk(t);await seed(a,{pointer:'none'});drop(a);
|
|
const r=await run(a,'adopt','acme');assert.equal(r.code,3);assert.equal(r.err,want);assert.equal(pointer(a),null);
|
|
const b=await mk(t);await seed(b);drop(b);const before=fp(b),n=calls(b).length;
|
|
const s=await run(b,'activate',C1,'acme');assert.equal(s.code,3);assert.equal(s.err,want);
|
|
assert.deepEqual(fp(b),before);assert.deepEqual(startsStops(b,n),[]);
|
|
});
|
|
|
|
test('T22 the stopped store read: a -wal or -shm refuses store-unclean and is left alone; a clean read adds no file',async(t)=>{
|
|
for(const ext of ['-wal','-shm'])await t.test(`bus.sqlite${ext}`,async(t)=>{
|
|
const e=await mk(t);await seed(e);fault(e,`stop-leaves${ext}`);
|
|
const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err);
|
|
assert.equal(r.err,rolledBack('store-unclean'));
|
|
assert.ok(r.out.includes(`stack-release: ${e.store}${ext} is present; the store is left as it is for Jason or Mos\n`),r.out);
|
|
const s=snaps(e),first=s[0],start=s.find((x)=>x.ev==='start');
|
|
assert.equal(first.ev,'stop');assert.ok(`bus.sqlite${ext}` in first.files,JSON.stringify(first));
|
|
assert.deepEqual(start.files,first.files);
|
|
});
|
|
await t.test('clean',async(t)=>{
|
|
const e=await mk(t);await seed(e);
|
|
const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err);
|
|
const s=snaps(e),stop=s.find((x)=>x.ev==='stop'),start=s.find((x)=>x.ev==='start');
|
|
assert.deepEqual(Object.keys(stop.files),['bus.sqlite']);assert.deepEqual(start.files,stop.files);
|
|
});
|
|
});
|