stack/packages/cli/tests/stack-upgrade.test.mjs

// scripts/stack-upgrade.sh (row 233, I3, #1735): root's upgrade and rollback of a stack whose units are
// root's, driving stack-release.sh --units external (row 232). Design: r6 sections 4 and 7,
// agents/darkwing/work/queue-195/DESIGN.md, as the I3 brief changes them; deviations:
// agents/darkwing/work/queue-233/BUILD.md. The stand-ins and the module fixture are
// stack-release-fixture.mjs's. Root's sudo, case2-unit.sh render and row 240 hostcheck are stand-ins
// here; the stack's user's steps are the real stack-release.sh from the user's clone, except the
// rehearsal, which the fixture's stub copies can't pass (its own tests are in stack-release.test.mjs).
import test from 'node:test';
import assert from 'node:assert/strict';
import {execFileSync,spawn} from 'node:child_process';
import {mkdirSync,writeFileSync,readFileSync,existsSync,readdirSync,rmSync,chmodSync,copyFileSync,symlinkSync,statSync} from 'node:fs';
import {DatabaseSync} from 'node:sqlite';
import {join,dirname} from 'node:path';
import {once} from 'node:events';
import {B,C,C1,U,COPIES,NODE,SBIN,base,fault,git,lines,logLines,mk,pointer,repo,sha,storeVersion,sysctl,units} from './stack-release-fixture.mjs';

// --- Root's stand-ins. Each is serialized into its own file with K, its baked paths. ---

// sudo -u USER -H env PATH=P CMD...: the form the driver must use, logged to sudo.log, then CMD with
// SBIN before P. A one-shot hook arm/<key>.sh runs first and arm/<key>.id replaces CMD's ID. The
// rehearsal prints a fixed BACKUP_STORE line and exits faults/rehearse-rc unless faults/real-rehearse.
function fakeSudo(K){
 const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
 const a=process.argv.slice(2),sd=process.env.SD_STATE,f=(n)=>path.join(sd,'faults',n);
 if(a[0]!=='-u'||!a[1]||a[2]!=='-H'||a[3]!=='env'||!/^PATH=/.test(a[4]||'')||a.length<6){console.error(`fake sudo: not the driver's form: ${a.join(' ')}`);process.exit(99)}
 const P=a[4].slice(5),cmd=a.slice(5);
 let key=path.basename(cmd[0]);
 if(cmd[0].endsWith('/scripts/stack-release.sh')){
  key=cmd[1];
  if(['activate','rollback'].includes(key)&&cmd.includes('--plan'))key+='-plan';
  if(key==='confirm'&&cmd.includes('--rollback'))key='confirm-rollback';
 }else if(cmd[0]==='git')key=`git-${cmd[3]}`;
 fs.appendFileSync(path.join(sd,'sudo.log'),JSON.stringify({user:a[1],path:P,key,cmd})+'\n');
 const env={...process.env,PATH:`${K.sbin}:${P}`},arm=(n)=>path.join(sd,'arm',`${key}.${n}`);
 if(fs.existsSync(arm('sh'))){const run=arm('sh')+'.ran';fs.renameSync(arm('sh'),run);cp.spawnSync('bash',[run],{env,stdio:'inherit'})}
 if(fs.existsSync(arm('id'))){cmd[2]=fs.readFileSync(arm('id'),'utf8').trim();fs.rmSync(arm('id'))}
 if(key==='rehearse'&&!fs.existsSync(f('real-rehearse'))){
  console.log(`fixture rehearse: ${cmd.slice(1).join(' ')}`);
  let line='BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g';try{line=fs.readFileSync(f('rehearse-store-line'),'utf8').replace(/\n$/,'')}catch{}
  if(line)console.log(line);
  let rc=0;try{rc=Number(fs.readFileSync(f('rehearse-rc'),'utf8'))}catch{}
  process.exit(rc);
 }
 const r=cp.spawnSync(cmd[0],cmd.slice(1),{env,stdio:'inherit'});process.exit(r.status??1);
}

// case2-unit.sh: logs its argv, then installs the copy's units as the fixture's user units (its
// WorkingDirectory under the stack's copies) and starts the bus, then the Console, as case2-unit.sh
// does. faults/render-fails-<copy> exits 1 first; faults/render-probe-<copy> (one-shot) first runs
// probe.sh, a second driver, and records its exit and stderr.
function fakeRender(K){
 const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
 const a=process.argv.slice(2),sd=process.env.SD_STATE,f=(n)=>path.join(sd,'faults',n);
 fs.appendFileSync(path.join(sd,'render.log'),JSON.stringify(a)+'\n');
 const cut=a.indexOf('--'),pre=a.slice(0,cut),serve=a.slice(cut+1),get=(k)=>pre[pre.indexOf(k)+1];
 const copy=get('--copy'),copies=path.join(get('--home'),'.local','share','mosaic-stack','copies');
 if(fs.existsSync(f(`render-probe-${copy}`))){
  fs.rmSync(f(`render-probe-${copy}`));
  const p=cp.spawnSync('bash',[path.join(sd,'probe.sh')],{encoding:'utf8'});
  fs.writeFileSync(path.join(sd,'probe.json'),JSON.stringify({code:p.status,err:p.stderr}));
 }
 if(fs.existsSync(f(`render-fails-${copy}`))){console.error(`fixture render: ${copy} fails`);process.exit(1)}
 const sh=(n,...x)=>cp.execFileSync('bash',[path.join(K.repo,'scripts',n),...x],{stdio:['ignore','ignore','inherit']});
 sh('bus-service.sh','install','--copy',copy,'--copies',copies,...(pre.includes('--pm')?['--pm']:[]),'--claude',get('--claude'));
 sh('webui-service.sh','install','--copy',copy,'--copies',copies,...serve);
 for(const u of [`mosaic-bus@${get('--business')}.service`,'mosaic-webui.service'])cp.execFileSync(path.join(K.sbin,'systemctl'),['start',u],{stdio:'inherit'});
}

// Root's systemctl: the fixture's, with systemd's no-op start of an active unit, and a one-shot
// faults/stop-noop-<unit> that leaves the unit running, as a stop past its timeout would.
function fakeSystemctl(K){
 const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
 const a=process.argv.slice(2),sd=process.env.SD_STATE;
 fs.appendFileSync(path.join(sd,'root-systemctl.log'),a.join(' ')+'\n');
 const noop=path.join(sd,'faults',`stop-noop-${a[1]}`);
 if(a[0]==='stop'&&fs.existsSync(noop)){fs.rmSync(noop);process.exit(0)}
 if(a[0]==='start'){let s={};try{s=JSON.parse(fs.readFileSync(path.join(sd,'units.json'),'utf8'))}catch{}if((s[a[1]]||{}).state==='active')process.exit(0)}
 const r=cp.spawnSync(path.join(K.sbin,'systemctl'),a,{stdio:'inherit'});process.exit(r.status??1);
}

const FAKES=join(base,'upgrade-fakes');mkdirSync(FAKES);
const K={sbin:SBIN,repo};
for(const [n,fn] of [['sudo',fakeSudo],['render',fakeRender],['systemctl',fakeSystemctl]])writeFileSync(join(FAKES,n),`#!${NODE}\n(${fn})(${JSON.stringify(K)});\n`,{mode:0o755});
writeFileSync(join(FAKES,'hostcheck'),'#!/bin/sh\necho "$*" >> "$SD_STATE/hostcheck.log"\nexit "$(cat "$SD_STATE/faults/hostcheck-rc" 2>/dev/null || echo 0)"\n',{mode:0o755});
const BUNDLE=join(base,'fixture.bundle');git(repo,'bundle','create','-q',BUNDLE,'refactor');
// Root's checkout of a reviewed commit: a clone of the fixture repo, detached at it.
const checkouts=new Map();
const checkout=(id)=>{
 if(!checkouts.has(id)){const d=join(base,`root-${id.slice(0,12)}`);git(base,'clone','-q','--no-checkout',repo,d);git(d,'checkout','-q','--detach',id);checkouts.set(id,d)}
 return checkouts.get(id);
};
const TO=['--timeout','3'];

// --- Per-test stack ---

// The stack's user's home: its copies (the fixture's, through a link), its clone, its bin. Both units
// installed on `copy` and active, adopted under --units external, and the I2 args file naming it.
async function useed(t,{copy=B}={}){
 const e=await mk(t),home=e.env.HOME;
 e.hc=join(home,'.local','share','mosaic-stack','copies');mkdirSync(dirname(e.hc),{recursive:true});symlinkSync(COPIES,e.hc);
 const bin=join(home,'.npm-global','bin');mkdirSync(bin,{recursive:true});
 writeFileSync(join(bin,'mosaic'),readFileSync(join(e.bin,'mosaic'),'utf8').replaceAll(`${e.bin}/`,`${bin}/`),{mode:0o755});
 copyFileSync(join(e.bin,'mosaic-old'),join(bin,'mosaic-old'));chmodSync(join(bin,'mosaic-old'),0o755);
 e.bin=bin;e.clone=join(home,'src','mosaic-stack');git(base,'clone','-q',repo,e.clone);
 const db=new DatabaseSync(e.store);
 db.exec('PRAGMA journal_mode=WAL');
 db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
 db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
 db.prepare("INSERT INTO meta VALUES ('schema_version', '3g')").run();
 for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`);
 db.close();
 const sh=(n,...a)=>execFileSync('bash',[join(e.clone,'scripts',n),...a],{env:e.env,encoding:'utf8',stdio:['ignore','pipe','pipe']});
 sh('bus-service.sh','install','--copy',copy,'--copies',e.hc,'--claude',e.claude);
 sh('webui-service.sh','install','--copy',copy,'--copies',e.hc,'--port',e.port,'--business','acme');
 sh('stack-copy.sh','wrapper',copy,'--copies',e.hc,'--bin',e.bin);
 execFileSync(join(SBIN,'systemctl'),['start',e.bus],{env:{...e.env,SD_CLOCK_OFFSET:'120'}});
 sysctl(e,'start',e.con);
 sh('stack-release.sh','adopt','acme','--units','external','--copies',e.hc,'--bin',e.bin);
 e.etc=join(e.root,'etc');mkdirSync(e.etc);e.args=join(e.etc,'dev.args');
 e.argsText=`--user\nfixture\n--home\n${home}\n--business\nacme\n--copy\n${copy}\n--node\n${NODE}\n--claude\n${e.claude}\n--\n--port\n${e.port}\n--business\nacme\n`;
 writeFileSync(e.args,e.argsText);chmodSync(e.args,0o640);
 e.run=join(e.root,'run');e.slog=join(e.root,'log','upgrade-dev.log');e.lockf=join(e.run,'upgrade-dev.lock');
 mkdirSync(join(e.sd,'arm'));
 e.denv={...e.env,STACK_UPGRADE_ETC:e.etc,STACK_UPGRADE_RUN:e.run,STACK_UPGRADE_LOG_DIR:dirname(e.slog),STACK_UPGRADE_UNIT_DIR:e.units,STACK_UPGRADE_HOSTCHECK:join(FAKES,'hostcheck'),STACK_UPGRADE_RENDER:join(FAKES,'render'),STACK_UPGRADE_SUDO:join(FAKES,'sudo'),STACK_UPGRADE_SYSTEMCTL:join(FAKES,'systemctl')};
 writeFileSync(join(e.sd,'probe.sh'),`exec bash ${join(checkout(C1),'scripts','stack-upgrade.sh')} upgrade --instance dev --bundle ${BUNDLE} --sha ${C1} --timeout 3\n`);
 writeFileSync(join(e.sd,'snap.jsonl'),'');
 return e;
}

// The driver from root's checkout of `at`.
async function drive(e,at,...args){
 const ch=spawn('bash',[join(checkout(at),'scripts','stack-upgrade.sh'),...args],{env:e.denv,stdio:['ignore','pipe','pipe']});
 let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d});
 const timer=setTimeout(()=>ch.kill('SIGKILL'),180000);
 const [code]=await once(ch,'close');clearTimeout(timer);
 return {code,out,err,all:out+err};
}
const upgrade=(e,id,...a)=>drive(e,id,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',id,...TO,...a);
const arm=(e,key,text)=>writeFileSync(join(e.sd,'arm',key),text);

// The step log: every line `<time> <step> rc=N ...` or the result line, root:root 0640 (here the
// test user's), and nothing the stack's user printed. Returns the lines without the time.
const STEP=/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ (start|fetch|ff|copy|hostcheck|units|plan|rehearse|rehearse-store|rehearsal-skipped|stop-console|stop-bus|wait-down|stage|render|confirm|args|revert|render-old|confirm-rollback|gate|start-bus|start-console|stop-units) rc=\d+ |^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ result exit=[0-35] [a-z-]+ /;
function slog(e){
 const l=lines(e.slog);
 for(const x of l){assert.match(x,STEP);assert.ok(!/stack-release:|fixture rehearse|PASS |FAIL |REHEARSAL/.test(x),x)}
 assert.equal(statSync(e.slog).mode&0o777,0o640);
 return l.map((x)=>x.slice(21));
}
const names=(e)=>slog(e).map((x)=>x.split(' ')[0]);
const result=(e)=>slog(e).at(-1);
const sudoKeys=(e)=>lines(join(e.sd,'sudo.log')).map((l)=>JSON.parse(l).key);
const lastAct=(e)=>logLines(e).filter((o)=>o.result==='started').at(-1).activation;
const mine=(e,a)=>logLines(e).filter((o)=>o.activation===a);
const open=(e)=>{const m=new Set();for(const o of logLines(e)){if(!o.activation)continue;if(o.result==='started')m.add(o.activation);else m.delete(o.activation)}return [...m]};
const final=(e,a)=>{const [s,f,...more]=mine(e,a);assert.equal(s.result,'started');assert.equal(more.length,0);return f};
const live=(e)=>[units(e)[e.bus].state,units(e)[e.con].state,units(e)[e.bus].wd.split('/').at(-1),units(e)[e.con].wd.split('/').at(-1)];
// Nothing the run did reached the args file: the same bytes and mode, and no temporary file left.
const argsKept=(e)=>{assert.equal(readFileSync(e.args,'utf8'),e.argsText);assert.equal(statSync(e.args).mode&0o777,0o640);assert.deepEqual(readdirSync(e.etc),['dev.args'])};
const argsCopy=(e)=>{const l=readFileSync(e.args,'utf8').split('\n');return l[l.indexOf('--copy')+1]};
const PRE=['start','fetch','ff','copy','hostcheck','units','plan','rehearse','rehearse-store'];
const A2=['stop-console','stop-bus','wait-down','wait-down'];
const R=[...A2,'revert','render-old','confirm-rollback'];

// --- r6 sequences 1-8, through the driver ---

test('row 233: sequence 1: confirm fails healthz, the driver reverts, renders the old copy and confirm --rollback --cause A5 passes: exit 1',async(t)=>{
 const e=await useed(t);
 fault(e,`healthz-${C1}`);
 const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
 assert.equal(r.err.split('\n').at(-2),`stack-upgrade: failed: A5 failed; rolled back to ${B}, which passed the gate`);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm',...R,'result']);
 assert.equal(result(e),`result exit=1 failed A5 failed; rolled back to ${B}, which passed the gate`);
 const a=lastAct(e),l=final(e,a);assert.deepEqual([l.op,l.result,l.code,l.rollback,l.from,l.to],['activate','gate-failed','healthz','ok',B,C1]);
 assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',B,B]);
 argsKept(e);
 // The stack's user's steps went through sudo in the driver's form, as fixture, the pinned node first.
 for(const l of lines(join(e.sd,'sudo.log')).map((x)=>JSON.parse(x)))assert.deepEqual([l.user,l.path],['fixture',`${dirname(NODE)}:/usr/local/bin:/usr/bin:/bin`]);
 assert.deepEqual(sudoKeys(e),['git-fetch','git-merge','stack-copy.sh','activate-plan','rehearse','activate','confirm','revert','confirm-rollback']);
});

test('row 233: sequence 2: activate --migrate fails at the wrapper, revert restores the store, confirm --rollback --cause A3: exit 1',async(t)=>{
 const e=await useed(t);
 const s0=sha(e.store);
 arm(e,'activate.sh',`chmod 0555 ${e.bin}\n`);arm(e,'revert.sh',`chmod 0755 ${e.bin}\n`);
 const r=await upgrade(e,C,'--migrate');assert.equal(r.code,1,r.all);
 assert.deepEqual(names(e),[...PRE,...A2,'stage',...R,'result']);
 assert.equal(result(e),`result exit=1 failed A3 failed; rolled back to ${B}, which passed the gate`);
 const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['failed','wrapper','ok']);
 assert.equal(storeVersion(e.store,true),'3g');assert.deepEqual(live(e),['active','active',B,B]);
 // The bus holds the store open again; its bytes are the backup's once it stops.
 sysctl(e,'stop',e.bus);assert.equal(sha(e.store),s0);
 argsKept(e);
});

test('row 233: sequence 3: root\'s render of the new copy fails, the driver rolls back with --cause A4, and holds the lock to the end: exit 1',async(t)=>{
 const e=await useed(t);
 fault(e,`render-fails-${C1}`);fault(e,`render-probe-${B}`);
 const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
 // The probe ran inside render-old, during the rollback, and its one line is its refusal.
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render',...A2,'revert','result','render-old','confirm-rollback','result']);
 const p=JSON.parse(readFileSync(join(e.sd,'probe.json'),'utf8'));assert.equal(p.code,3,p.err);
 assert.equal(p.err,`stack-upgrade: upgrade-running: another stack-upgrade.sh run holds ${e.lockf} (nothing was stopped)\n`);
 assert.match(slog(e).at(-4),/^result exit=3 upgrade-running /);
 assert.match(slog(e).find((x)=>x.startsWith('render ')),/^render rc=1 /);
 const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['failed','units-start','ok']);
 assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: sequence 4: a capture failure is exit 3 from the stage, and A2f restarts the old units and gates them: exit 1',async(t)=>{
 const e=await useed(t);
 arm(e,'activate.sh',`: > ${join(e.sd,'faults','no-fragment')}\n`);arm(e,'gate.sh',`rm ${join(e.sd,'faults','no-fragment')}\n`);
 const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','start-bus','start-console','gate','result']);
 assert.match(slog(e).find((x)=>x.startsWith('stage ')),/^stage rc=3 /);
 assert.equal(result(e),`result exit=1 failed A3 failed; ${B} is running again and passed the gate`);
 const a=lastAct(e);assert.deepEqual(mine(e,a).map((o)=>[o.result,o.code]),[['started',undefined],['refused','capture-failed']]);
 assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: N1: a bus that outlives A2\'s timeout keeps its old start, so A2f\'s gate fails stale-start: exit 2, both units stopped',async(t)=>{
 const e=await useed(t);
 fault(e,`stop-noop-${e.bus}`);
 const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
 assert.deepEqual(names(e),[...PRE,...A2,'start-bus','start-console','gate','stop-units','result']);
 assert.ok(slog(e).includes(`wait-down rc=1 unit: ${e.bus} state: active`),slog(e).join('\n'));
 assert.ok(r.out.includes(`gate failed on ${B}: stale-start`),r.out);
 assert.equal(result(e),`result exit=2 rollback-failed A2 failed and ${B} failed its gate after the restart; both units are stopped`);
 assert.deepEqual(live(e).slice(0,2),['inactive','inactive']);
 assert.ok(!sudoKeys(e).includes('activate'));argsKept(e);
});

test('row 233: sequence 5: confirm of the wrong copy is refused, and the driver rolls back with --cause A5: exit 1',async(t)=>{
 const e=await useed(t);
 arm(e,'confirm.id',B);
 const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
 assert.ok(r.out.includes('refusing open-activation')||r.err.includes('refusing open-activation'),r.all);
 assert.match(slog(e).find((x)=>x.startsWith('confirm ')),/^confirm rc=3 /);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm',...R,'result']);
 const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','unrecorded','ok']);
 assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: sequence 6: a revert that fails stops both units at exit 2, and the next run refuses at its --plan with nothing stopped',async(t)=>{
 const e=await useed(t);
 fault(e,`render-fails-${C1}`);
 arm(e,'revert.sh',`for d in ${join(e.state,'stack-rollback')}/*/; do [ -f "$d/AWAITING-UNITS" ] && rm "$d/wrapper"; done\n`);
 const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render',...A2,'revert','stop-units','result']);
 assert.equal(result(e),`result exit=2 rollback-failed A4 failed and revert failed; both units are stopped`);
 const a=lastAct(e);assert.deepEqual(open(e),[a]);assert.deepEqual(live(e).slice(0,2),['inactive','inactive']);
 argsKept(e);
 const n=slog(e).length,sc=lines(join(e.sd,'root-systemctl.log')).length;
 rmSync(join(e.sd,'faults',`render-fails-${C1}`));
 const again=await upgrade(e,C1);assert.equal(again.code,3,again.all);
 assert.ok(again.out.includes(`refuses interrupted-run: activation ${a} started and has no final line`),again.out);
 assert.deepEqual(names(e).slice(n),PRE.slice(0,7).concat('result'));
 assert.match(result(e),/^result exit=3 plan-refused /);
 assert.equal(lines(join(e.sd,'root-systemctl.log')).length,sc);argsKept(e);
});

test('row 233: sequence 7: two upgrades and a rollback --to the previous copy with --restore, the args file following each confirm',async(t)=>{
 const e=await useed(t);
 // The args file at confirm still names the old copy; the new one moves in after it.
 arm(e,'confirm.sh',`cp ${e.args} ${join(e.sd,'args-at-confirm')}\n`);
 const one=await upgrade(e,C1);assert.equal(one.code,0,one.all);
 assert.equal(one.out.split('\n').at(-2),`stack-upgrade: upgrade ok: ${C1} is live for acme`);
 assert.equal(readFileSync(join(e.sd,'args-at-confirm'),'utf8'),e.argsText);
 assert.equal(readFileSync(e.args,'utf8'),e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
 assert.equal(statSync(e.args).mode&0o777,0o640);assert.deepEqual(readdirSync(e.etc),['dev.args']);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
 assert.ok(slog(e).includes(`args rc=0 copy: ${C1}`));
 const a1=lastAct(e);
 const two=await upgrade(e,C,'--migrate');assert.equal(two.code,0,two.all);
 const a2=lastAct(e);assert.equal(argsCopy(e),C);
 const l1=final(e,a1),l2=final(e,a2);
 assert.deepEqual([l1.op,l1.result,l1.from,l1.to],['activate','ok',B,C1]);
 assert.deepEqual([l2.op,l2.result,l2.from,l2.to,l2.backup],['activate','ok',C1,C,a2]);
 assert.equal(storeVersion(e.store,true),'3h');
 const n=slog(e).length;
 const rb=await drive(e,C,'rollback','--instance','dev','--to',C1,'--restore',a2,...TO);assert.equal(rb.code,0,rb.all);
 assert.equal(rb.out.split('\n').at(-2),`stack-upgrade: rollback ok: ${C1} is live for acme`);
 // A rollback fetches, copies and rehearses nothing.
 assert.deepEqual(names(e).slice(n),['start','hostcheck','units','plan',...A2,'stage','render','confirm','args','result']);
 const ra=lastAct(e),l3=final(e,ra);assert.deepEqual([l3.op,l3.result,l3.from,l3.to,l3.backup],['rollback','ok',C,C1,a2]);
 assert.deepEqual([pointer(e).copy,pointer(e).schema],[C1,'3g']);assert.equal(storeVersion(e.store,true),'3g');
 assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
 const m=slog(e).length,sc=lines(join(e.sd,'root-systemctl.log')).length;
 const no=await drive(e,C,'rollback','--instance','dev','--to',B,...TO);assert.equal(no.code,3,no.all);
 assert.ok(no.all.includes(`refusing to-not-previous: --to ${B} is not the previous copy; the log names ${C}`),no.all);
 assert.deepEqual(names(e).slice(m),['start','hostcheck','units','plan','result']);
 assert.equal(lines(join(e.sd,'root-systemctl.log')).length,sc);assert.equal(argsCopy(e),C1);
});

test('row 240: I3 N1: a failed args move after A5 exits 5 args-failed, the new copy live and confirmed and both args files kept',async(t)=>{
 const e=await useed(t);
 // A read-only /etc/mosaic-stack at confirm: the stage and the render are done, and the move fails.
 // The fixture's cleanup runs chmod -R u+w before it removes the root, so the 0555 dir needs no hook here.
 arm(e,'confirm.sh',`chmod 0555 ${e.etc}\n`);
 const l=await upgrade(e,C1);assert.equal(l.code,5,l.all);
 assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
 assert.ok(slog(e).includes(`args rc=1 copy: ${C1}`));
 assert.match(result(e),new RegExp(`^result exit=5 args-failed ${C1} is live and confirmed, but ${e.args} still names ${B}; put ${e.etc}/\\.dev\\.args\\.\\w{6} in its place$`));
 assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',C1,C1]);
 assert.equal(readFileSync(e.args,'utf8'),e.argsText);
 const left=readdirSync(e.etc).filter((n)=>n!=='dev.args');assert.equal(left.length,1);
 assert.equal(readFileSync(join(e.etc,left[0]),'utf8'),e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
});

test('row 233: sequence 8: a migration that writes and fails: the driver reverts to the backup byte for byte, --cause A3: exit 1',async(t)=>{
 const e=await useed(t);
 const s0=sha(e.store);
 fault(e,'migrate-fails');
 const r=await upgrade(e,C,'--migrate');assert.equal(r.code,1,r.all);
 assert.match(slog(e).find((x)=>x.startsWith('stage ')),/^stage rc=1 /);
 assert.deepEqual(names(e),[...PRE,...A2,'stage',...R,'result']);
 const a=lastAct(e),l=final(e,a);assert.deepEqual([l.result,l.code,l.rollback],['failed','migrate-failed','ok']);
 sysctl(e,'stop',e.bus);assert.equal(sha(e.store),s0);assert.equal(storeVersion(e.store,false),'3g');
 argsKept(e);
});

// --- The lock, A1's refusals and the rehearsal ---

test('row 233: a second run while the lock is held refuses upgrade-running with one log line and no call',async(t)=>{
 const e=await useed(t);
 mkdirSync(e.run,{recursive:true});
 const holder=spawn('flock',[e.lockf,'sleep','60'],{detached:true,stdio:'ignore'});
 t.after(()=>{try{process.kill(-holder.pid,'SIGKILL')}catch{}});
 for(let i=0;i<100&&execFileSync('bash',['-c','flock -n "$1" true && echo free || echo held','_',e.lockf],{encoding:'utf8'}).trim()==='free';i++)await new Promise((r)=>setTimeout(r,50));
 const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
 assert.equal(r.err,`stack-upgrade: upgrade-running: another stack-upgrade.sh run holds ${e.lockf} (nothing was stopped)\n`);
 assert.deepEqual(names(e),['result']);
 assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
 assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: A1 refuses open-activation on any AWAITING-UNITS file, through the stack\'s user\'s --plan, with nothing stopped',async(t)=>{
 const e=await useed(t);
 const fake=join(e.state,'stack-rollback',`20000101T000000Z-${C1.slice(0,12)}`);mkdirSync(fake,{recursive:true});writeFileSync(join(fake,'AWAITING-UNITS'),`${C1}\n`);
 const logs=logLines(e).length;
 const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
 assert.ok(r.out.includes('refuses open-activation'),r.out);
 assert.deepEqual(names(e),[...PRE.slice(0,7),'result']);assert.match(result(e),/^result exit=3 plan-refused /);
 assert.equal(logLines(e).length,logs);assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
 assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: a target that migrates refuses migrate-required without --migrate, with nothing stopped',async(t)=>{
 const e=await useed(t);
 const r=await upgrade(e,C);assert.equal(r.code,3,r.all);
 assert.ok(r.out.includes('refuses migrate-required'),r.out);
 assert.deepEqual(names(e),[...PRE.slice(0,7),'result']);assert.match(result(e),/^result exit=3 plan-refused /);
 assert.ok(!sudoKeys(e).includes('rehearse'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
 assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: the rehearsal runs after the plan and before A2; a failed one refuses rehearsal-failed with nothing stopped',async(t)=>{
 const e=await useed(t);
 fault(e,'rehearse-rc','1');
 const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
 assert.ok(r.out.includes('fixture rehearse: '),r.out);
 assert.deepEqual(names(e),[...PRE,'result']);
 assert.equal(result(e),'result exit=3 rehearsal-failed the rehearsal exited 1 (nothing was stopped)');
 const rh=lines(join(e.sd,'sudo.log')).map((l)=>JSON.parse(l)).find((l)=>l.key==='rehearse');
 assert.deepEqual(rh.cmd.slice(1),['rehearse',C1,'acme','--copies',e.hc,'--backup','last','--port','7339','--timeout','3']);
 assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
});

test('row 233: the rehearsal\'s BACKUP_STORE line goes to the step log after a match, a malformed one as none, the rest of its output never',async(t)=>{
 const e=await useed(t);
 fault(e,'rehearse-rc','1');
 const r=await upgrade(e,C1,'--rehearse-port','7449');assert.equal(r.code,3,r.all);
 assert.ok(slog(e).includes('rehearse-store rc=0 BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g'),slog(e).join('\n'));
 assert.ok(slog(e).find((x)=>x.startsWith('rehearse ')).includes(' --port 7449 '));
 for(const bad of ['BACKUP_STORE ../../etc schema 3g','BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g; rm -rf /','stack-release: BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g','']){
  fault(e,'rehearse-store-line',bad);
  const n=slog(e).length;
  const b=await upgrade(e,C1);assert.equal(b.code,3,b.all);
  assert.deepEqual(slog(e).slice(n).filter((x)=>x.startsWith('rehearse-store ')),['rehearse-store rc=0 BACKUP_STORE none'],bad);
 }
});

test('row 233: a real rehearsal with no backup in the log refuses no-backup, and goes on only with --no-rehearsal, which logs the skip',async(t)=>{
 const e=await useed(t);
 fault(e,'real-rehearse');
 const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
 assert.ok(r.out.includes('FAIL no-backup: the log records no backup\nREHEARSAL_FAIL\n'),r.out);
 assert.equal(result(e),'result exit=3 no-backup the log records no backup to rehearse against; only a go that names --no-rehearsal goes on (nothing was stopped)');
 assert.ok(slog(e).includes('rehearse-store rc=0 BACKUP_STORE none'));
 assert.ok(!existsSync(join(e.data,'rehearsals')));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
 const n=slog(e).length;
 const s=await upgrade(e,C1,'--no-rehearsal');assert.equal(s.code,0,s.all);
 assert.deepEqual(names(e).slice(n),[...PRE,'rehearsal-skipped',...A2,'stage','render','confirm','args','result']);
 assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
});

test('row 233: a real rehearsal after an upgrade names the log\'s newest backup; --no-rehearsal with a backup refuses rehearsal-required',async(t)=>{
 const e=await useed(t);
 const one=await upgrade(e,C1);assert.equal(one.code,0,one.all);
 const a1=lastAct(e);
 fault(e,'real-rehearse');
 const n=slog(e).length;
 // The stub copies fail the rehearsal right after it names the backup (copy-without-no-launch).
 const r=await upgrade(e,C,'--migrate');assert.equal(r.code,3,r.all);
 assert.ok(r.out.includes(`PASS backup-last: the log's newest backup is ${a1}\n`),r.out);
 assert.deepEqual(names(e).slice(n),[...PRE,'result']);assert.match(result(e),/^result exit=3 rehearsal-failed the rehearsal exited 1 /);
 rmSync(join(e.sd,'faults','real-rehearse'));
 const m=slog(e).length;
 const q=await upgrade(e,C,'--migrate','--no-rehearsal');assert.equal(q.code,3,q.all);
 assert.deepEqual(names(e).slice(m),[...PRE,'result']);
 assert.equal(result(e),'result exit=3 rehearsal-required the log records a backup and the rehearsal ran; --no-rehearsal is only for a stack with none (nothing was stopped)');
 assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
});

// --- Before A1 and the args file ---

test('row 233: the args file must be the running user\'s, with no group or other write, one --copy, and the copy the units run',async(t)=>{
 const e=await useed(t);
 const tryArgs=async(text,mode,code,why)=>{
  writeFileSync(e.args,text);chmodSync(e.args,mode);
  const n=existsSync(e.slog)?slog(e).length:0;
  const r=await upgrade(e,C1);assert.equal(r.code,3,`${why}: ${r.all}`);
  assert.match(slog(e).at(-1),new RegExp(`^result exit=3 ${code} `),why);
  assert.ok(!slog(e).slice(n).some((x)=>/^(stop|stage|render|confirm)/.test(x)),why);
 };
 await tryArgs(e.argsText,0o660,'args-unreadable','group write');
 await tryArgs(e.argsText,0o642,'args-unreadable','other write');
 await tryArgs(e.argsText.replace('--\n',`--copy\n${C1}\n--\n`),0o640,'args-unreadable','two --copy');
 await tryArgs(e.argsText.replace('--\n','--tester\n--\n'),0o640,'args-unreadable','an unknown flag');
 await tryArgs(e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`),0o640,'args-differ','the units run B');
 rmSync(e.args);symlinkSync(join(e.sd,'probe.sh'),e.args);
 const l=await upgrade(e,C1);assert.equal(l.code,3);assert.match(result(e),/^result exit=3 args-unreadable /);
 assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
});

test('row 233: hostcheck (row 240): a nonzero --check or a missing file refuses profile-drift before the plan',async(t)=>{
 const e=await useed(t);
 fault(e,'hostcheck-rc','1');
 const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
 assert.match(result(e),/^result exit=3 profile-drift /);
 assert.deepEqual(lines(join(e.sd,'hostcheck.log')),['--check']);
 assert.deepEqual(sudoKeys(e),['git-fetch','git-merge','stack-copy.sh']);
 e.denv.STACK_UPGRADE_HOSTCHECK=join(e.root,'no-hostcheck');
 const m=await upgrade(e,C1);assert.equal(m.code,3,m.all);
 assert.equal(result(e),`result exit=3 profile-drift no executable ${join(e.root,'no-hostcheck')} (row 240) (nothing was stopped)`);
 assert.ok(!sudoKeys(e).includes('activate-plan'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));argsKept(e);
});

test('row 233: the checkout must be at --sha (head-mismatch), and a commit the clone can\'t fast-forward to refuses fetch-failed',async(t)=>{
 const e=await useed(t);
 const r=await drive(e,C1,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C,'--migrate',...TO);assert.equal(r.code,3,r.all);
 assert.equal(result(e),`result exit=3 head-mismatch this checkout, ${checkout(C1)}, is at ${C1}, not ${C} (nothing was stopped)`);
 const u=await drive(e,U,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',U,...TO);assert.equal(u.code,3,u.all);
 assert.match(result(e),/^result exit=3 fetch-failed .* fast-forward /);
 assert.ok(!sudoKeys(e).includes('activate-plan'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));argsKept(e);
});

test('row 233: usage exits 4 before any log or lock',async(t)=>{
 const e=await useed(t);
 for(const a of [[],['status'],['upgrade','--instance','dev','--bundle',BUNDLE],['upgrade','--instance','other','--bundle',BUNDLE,'--sha',C1],['upgrade','--instance','dev','--bundle','rel/x','--sha',C1],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1.slice(0,12)],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1,'--to',B],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1,'--rehearse-port','80'],['rollback','--instance','dev','--to',B,'--migrate'],['rollback','--instance','dev','--to',B,'--events','3'],['rollback','--instance','dev'],['rollback','--instance','dev','--to',B,'--restore','x']]){
  const r=await drive(e,C1,...a);assert.equal(r.code,4,a.join(' '));assert.match(r.err,/^usage: stack-upgrade.sh upgrade /,a.join(' '));
 }
 assert.ok(!existsSync(dirname(e.slog)));assert.ok(!existsSync(e.run));
});