jason.woltje
9acc378555
An installed unit must equal the render but for the copy id, so a re-pin that adds --sso refused render-refused. case2-unit gains --check (every render check, nothing written) and --check-new (a first install's checks). stack-upgrade runs both checks before anything stops, and --next-args reads root's <instance>.args.next: equal before -- but for --copy, different after it. After the stage it moves the installed unit files into a new <instance>.replaced.XXXXXX directory and renders a first install. A failure moves them back unchanged and restarts the old copy, and confirm swaps the args file. The run sheet gives the commands, the rollback past the migration included. A stamped rehearsal now starts an isolated provider on 127.0.0.1 port 0 with a certificate and keys made for the run. It serves discovery and a JWKS for the live provider ids and answers 403 to authorize and token. Only the scratch Console trusts its certificate. The gate requires each provider ready (sso-ready); provider-discovery requires discovery and JWKS fetches and no code or token request. The provider stops with the run, and by itself once the run's shell is gone. The system-path test helper asserts the log's from copy again. The pinned-copy sign-in requests are bounded, and cleanup waits for each Console to exit.
952 lines
70 KiB
JavaScript
952 lines
70 KiB
JavaScript
// scripts/stack-upgrade.sh (row 233, I3, #1735): root's upgrade and rollback of a stack whose units are
|
|
// root's, driving stack-release.sh --units external (row 232). Design: r6 sections 4 and 7,
|
|
// agents/darkwing/work/queue-195/DESIGN.md, as the I3 brief changes them; deviations:
|
|
// agents/darkwing/work/queue-233/BUILD.md. The stand-ins and the module fixture are
|
|
// stack-release-fixture.mjs's. Root's sudo, case2-unit.sh render and row 240 hostcheck are stand-ins
|
|
// here; the stack's user's steps are the real stack-release.sh from the user's clone, except the
|
|
// rehearsal, which the fixture's stub copies can't pass (its own tests are in stack-release.test.mjs).
|
|
// The #385 R3-F1 cases run the real base and current case2-unit.sh instead of the render stand-in.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import {execFileSync,spawn,spawnSync} from 'node:child_process';
|
|
import {mkdirSync,writeFileSync,readFileSync,existsSync,readdirSync,rmSync,chmodSync,copyFileSync,symlinkSync,statSync,realpathSync} from 'node:fs';
|
|
import {DatabaseSync} from 'node:sqlite';
|
|
import {join,dirname} from 'node:path';
|
|
import {once} from 'node:events';
|
|
import {B,C,C1,U,COPIES,NODE,ROOT,SBIN,base,fault,git,lines,logLines,mk,pointer,repo,sha,storeVersion,sysctl,units} from './stack-release-fixture.mjs';
|
|
|
|
// --- Root's stand-ins. Each is serialized into its own file with K, its baked paths. ---
|
|
|
|
// sudo -u USER -H env PATH=P CMD...: the form the driver must use, logged to sudo.log, then CMD with
|
|
// SBIN before P. A one-shot hook arm/<key>.sh runs first and arm/<key>.id replaces CMD's ID. The
|
|
// rehearsal prints a fixed BACKUP_STORE line and exits faults/rehearse-rc unless faults/real-rehearse.
|
|
function fakeSudo(K){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
|
|
const a=process.argv.slice(2),sd=process.env.SD_STATE,f=(n)=>path.join(sd,'faults',n);
|
|
if(a[0]!=='-u'||!a[1]||a[2]!=='-H'||a[3]!=='env'||!/^PATH=/.test(a[4]||'')||a.length<6){console.error(`fake sudo: not the driver's form: ${a.join(' ')}`);process.exit(99)}
|
|
const P=a[4].slice(5),cmd=a.slice(5);
|
|
let key=path.basename(cmd[0]);
|
|
if(cmd[0].endsWith('/scripts/stack-release.sh')){
|
|
key=cmd[1];
|
|
if(['activate','rollback'].includes(key)&&cmd.includes('--plan'))key+='-plan';
|
|
if(key==='confirm'&&cmd.includes('--rollback'))key='confirm-rollback';
|
|
}else if(cmd[0]==='git')key=`git-${cmd[3]}`;
|
|
fs.appendFileSync(path.join(sd,'sudo.log'),JSON.stringify({user:a[1],path:P,key,cmd})+'\n');
|
|
const env={...process.env,PATH:`${K.sbin}:${P}`},arm=(n)=>path.join(sd,'arm',`${key}.${n}`);
|
|
if(fs.existsSync(arm('sh'))){const run=arm('sh')+'.ran';fs.renameSync(arm('sh'),run);cp.spawnSync('bash',[run],{env,stdio:'inherit'})}
|
|
if(fs.existsSync(arm('id'))){cmd[2]=fs.readFileSync(arm('id'),'utf8').trim();fs.rmSync(arm('id'))}
|
|
if(key==='rehearse'&&!fs.existsSync(f('real-rehearse'))){
|
|
console.log(`fixture rehearse: ${cmd.slice(1).join(' ')}`);
|
|
let line='BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g';try{line=fs.readFileSync(f('rehearse-store-line'),'utf8').replace(/\n$/,'')}catch{}
|
|
if(line)console.log(line);
|
|
let rc=0;try{rc=Number(fs.readFileSync(f('rehearse-rc'),'utf8'))}catch{}
|
|
process.exit(rc);
|
|
}
|
|
const r=cp.spawnSync(cmd[0],cmd.slice(1),{env,stdio:'inherit'});process.exit(r.status??1);
|
|
}
|
|
|
|
// case2-unit.sh: logs its argv, then installs the copy's units as the fixture's user units (its
|
|
// WorkingDirectory under the stack's copies) and starts the optional proxy, bus, then Console,
|
|
// as case2-unit.sh does. The business is --business's value, or --org's when that is the spelling. A first
|
|
// --check or --check-new (#1873) logs its argv and exits 1 under faults/check-fails-<copy>, else 0,
|
|
// writing nothing. faults/render-fails-<copy> exits 1 first; faults/render-probe-<copy> (one-shot) first runs
|
|
// probe.sh, a second driver, and records its exit and stderr.
|
|
function fakeRender(K){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
|
|
const a=process.argv.slice(2),sd=process.env.SD_STATE,f=(n)=>path.join(sd,'faults',n);
|
|
fs.appendFileSync(path.join(sd,'render.log'),JSON.stringify(a)+'\n');
|
|
if(a[0]==='--check'||a[0]==='--check-new'){
|
|
const c=a[a.indexOf('--copy')+1];
|
|
if(fs.existsSync(f(`check-fails-${c}`))){console.error(`fixture render: ${a[0]} ${c} fails`);process.exit(1)}
|
|
process.exit(0);
|
|
}
|
|
const cut=a.indexOf('--'),pre=a.slice(0,cut),serve=a.slice(cut+1),get=(k)=>pre[pre.indexOf(k)+1];
|
|
const copy=get('--copy'),copies=path.join(get('--home'),'.local','share','mosaic-stack','copies');
|
|
if(fs.existsSync(f(`render-probe-${copy}`))){
|
|
fs.rmSync(f(`render-probe-${copy}`));
|
|
const p=cp.spawnSync('bash',[path.join(sd,'probe.sh')],{encoding:'utf8'});
|
|
fs.writeFileSync(path.join(sd,'probe.json'),JSON.stringify({code:p.status,err:p.stderr}));
|
|
}
|
|
if(fs.existsSync(f(`render-fails-${copy}`))){console.error(`fixture render: ${copy} fails`);process.exit(1)}
|
|
const sh=(n,...x)=>cp.execFileSync('bash',[path.join(get('--home'),'src','mosaic-stack','scripts',n),...x],{stdio:['ignore','ignore','inherit']});
|
|
sh('bus-service.sh','install','--copy',copy,'--copies',copies,...(pre.includes('--pm')?['--pm']:[]),'--claude',get('--claude'));
|
|
sh('webui-service.sh','install','--copy',copy,'--copies',copies,...serve);
|
|
const names=[`mosaic-bus@${get(pre.includes('--business')?'--business':'--org')}.service`,'mosaic-webui.service'];
|
|
if(pre.includes('--proxy')){
|
|
fs.writeFileSync(path.join(process.env.XDG_CONFIG_HOME,'systemd','user','mosaic-proxy.service'),`[Service]\nWorkingDirectory=${copies}/${copy}\nExecStart=${get('--node')} ${copies}/${copy}/packages/model-proxy/src/run.mjs\n`);
|
|
names.unshift('mosaic-proxy.service');
|
|
}
|
|
for(const u of names)cp.execFileSync(path.join(K.sbin,'systemctl'),['start',u],{stdio:'inherit'});
|
|
}
|
|
|
|
// Root's systemctl: the fixture's, with systemd's no-op start of an active unit, and a one-shot
|
|
// faults/stop-noop-<unit> that leaves the unit running, as a stop past its timeout would.
|
|
function fakeSystemctl(K){
|
|
const fs=process.getBuiltinModule('node:fs'),path=process.getBuiltinModule('node:path'),cp=process.getBuiltinModule('node:child_process');
|
|
const a=process.argv.slice(2),sd=process.env.SD_STATE;
|
|
fs.appendFileSync(path.join(sd,'root-systemctl.log'),a.join(' ')+'\n');
|
|
const noop=path.join(sd,'faults',`stop-noop-${a[1]}`);
|
|
if(a[0]==='stop'&&fs.existsSync(noop)){fs.rmSync(noop);process.exit(0)}
|
|
if(a[0]==='start'&&fs.existsSync(path.join(sd,'faults',`root-start-fails-${a[1]}`)))process.exit(1);
|
|
if(a[0]==='start'){let s={};try{s=JSON.parse(fs.readFileSync(path.join(sd,'units.json'),'utf8'))}catch{}if((s[a[1]]||{}).state==='active')process.exit(0)}
|
|
const r=cp.spawnSync(path.join(K.sbin,'systemctl'),a,{stdio:'inherit'});process.exit(r.status??1);
|
|
}
|
|
|
|
const FAKES=join(base,'upgrade-fakes');mkdirSync(FAKES);
|
|
const K={sbin:SBIN,repo};
|
|
for(const [n,fn] of [['sudo',fakeSudo],['render',fakeRender],['systemctl',fakeSystemctl]])writeFileSync(join(FAKES,n),`#!${NODE}\n(${fn})(${JSON.stringify(K)});\n`,{mode:0o755});
|
|
writeFileSync(join(FAKES,'hostcheck'),'#!/bin/sh\necho "$*" >> "$SD_STATE/hostcheck.log"\nexit "$(cat "$SD_STATE/faults/hostcheck-rc" 2>/dev/null || echo 0)"\n',{mode:0o755});
|
|
const BUNDLE=join(base,'fixture.bundle');git(repo,'bundle','create','-q',BUNDLE,'refactor');
|
|
// Root's checkout of a reviewed commit: a clone of the fixture repo, detached at it.
|
|
const checkouts=new Map();
|
|
const checkout=(id)=>{
|
|
if(!checkouts.has(id)){const d=join(base,`root-${id.slice(0,12)}`);git(base,'clone','-q','--no-checkout',repo,d);git(d,'checkout','-q','--detach',id);checkouts.set(id,d)}
|
|
return checkouts.get(id);
|
|
};
|
|
const TO=['--timeout','3'];
|
|
|
|
// --- Per-test stack ---
|
|
|
|
// The stack's user's home: its copies (the fixture's, through a link), its clone, its bin. Both units
|
|
// installed on `copy` and active, adopted under --units external, and the I2 args file naming it.
|
|
async function useed(t,{copy=B}={}){
|
|
const e=await mk(t),home=e.env.HOME;
|
|
e.hc=join(home,'.local','share','mosaic-stack','copies');mkdirSync(dirname(e.hc),{recursive:true});symlinkSync(COPIES,e.hc);
|
|
const bin=join(home,'.npm-global','bin');mkdirSync(bin,{recursive:true});
|
|
writeFileSync(join(bin,'mosaic'),readFileSync(join(e.bin,'mosaic'),'utf8').replaceAll(`${e.bin}/`,`${bin}/`),{mode:0o755});
|
|
copyFileSync(join(e.bin,'mosaic-old'),join(bin,'mosaic-old'));chmodSync(join(bin,'mosaic-old'),0o755);
|
|
e.bin=bin;e.clone=join(home,'src','mosaic-stack');git(base,'clone','-q',repo,e.clone);
|
|
const db=new DatabaseSync(e.store);
|
|
db.exec('PRAGMA journal_mode=WAL');
|
|
db.exec('CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
|
|
db.exec('CREATE TABLE events (id INTEGER PRIMARY KEY, at TEXT NOT NULL)');
|
|
db.prepare("INSERT INTO meta VALUES ('schema_version', '3g')").run();
|
|
for(let i=0;i<3;i++)db.prepare('INSERT INTO events (at) VALUES (?)').run(`2000-01-01T00:00:0${i}.000Z`);
|
|
db.close();
|
|
const sh=(n,...a)=>execFileSync('bash',[join(e.clone,'scripts',n),...a],{env:e.env,encoding:'utf8',stdio:['ignore','pipe','pipe']});
|
|
sh('bus-service.sh','install','--copy',copy,'--copies',e.hc,'--claude',e.claude);
|
|
sh('webui-service.sh','install','--copy',copy,'--copies',e.hc,'--port',e.port,'--business','acme');
|
|
sh('stack-copy.sh','wrapper',copy,'--copies',e.hc,'--bin',e.bin);
|
|
execFileSync(join(SBIN,'systemctl'),['start',e.bus],{env:{...e.env,SD_CLOCK_OFFSET:'120'}});
|
|
sysctl(e,'start',e.con);
|
|
sh('stack-release.sh','adopt','acme','--units','external','--copies',e.hc,'--bin',e.bin);
|
|
e.etc=join(e.root,'etc');mkdirSync(e.etc);e.args=join(e.etc,'dev.args');
|
|
e.argsText=`--user\nfixture\n--home\n${home}\n--business\nacme\n--copy\n${copy}\n--node\n${NODE}\n--claude\n${e.claude}\n--\n--port\n${e.port}\n--business\nacme\n`;
|
|
writeFileSync(e.args,e.argsText);chmodSync(e.args,0o640);
|
|
e.run=join(e.root,'run');e.slog=join(e.root,'log','upgrade-dev.log');e.lockf=join(e.run,'upgrade-dev.lock');
|
|
mkdirSync(join(e.sd,'arm'));
|
|
e.denv={...e.env,STACK_UPGRADE_ETC:e.etc,STACK_UPGRADE_RUN:e.run,STACK_UPGRADE_LOG_DIR:dirname(e.slog),STACK_UPGRADE_UNIT_DIR:e.units,STACK_UPGRADE_HOSTCHECK:join(FAKES,'hostcheck'),STACK_UPGRADE_RENDER:join(FAKES,'render'),STACK_UPGRADE_SUDO:join(FAKES,'sudo'),STACK_UPGRADE_SYSTEMCTL:join(FAKES,'systemctl')};
|
|
writeFileSync(join(e.sd,'probe.sh'),`exec bash ${join(checkout(C1),'scripts','stack-upgrade.sh')} upgrade --instance dev --bundle ${BUNDLE} --sha ${C1} --timeout 3\n`);
|
|
writeFileSync(join(e.sd,'snap.jsonl'),'');
|
|
return e;
|
|
}
|
|
|
|
// The driver from root's checkout of `at`.
|
|
async function drive(e,at,...args){
|
|
const ch=spawn('bash',[join(checkout(at),'scripts','stack-upgrade.sh'),...args],{env:e.denv,stdio:['ignore','pipe','pipe']});
|
|
let out='',err='';ch.stdout.on('data',(d)=>{out+=d});ch.stderr.on('data',(d)=>{err+=d});
|
|
const timer=setTimeout(()=>ch.kill('SIGKILL'),180000);
|
|
const [code]=await once(ch,'close');clearTimeout(timer);
|
|
return {code,out,err,all:out+err};
|
|
}
|
|
const upgrade=(e,id,...a)=>drive(e,id,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',id,...TO,...a);
|
|
const arm=(e,key,text)=>writeFileSync(join(e.sd,'arm',key),text);
|
|
|
|
// The step log: every line `<time> <step> rc=N ...` or the result line, root:root 0640 (here the
|
|
// test user's), and nothing the stack's user printed. Returns the lines without the time.
|
|
const STEP=/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ (start|fetch|ff|copy|hostcheck|units|plan|rehearse|rehearse-store|rehearsal-skipped|render-old-check|render-check|stop-console|stop-bus|stop-proxy|wait-down|units-aside|units-back|daemon-reload|next-args|stage|render|confirm|args|revert|render-old|confirm-rollback|gate|start-proxy|start-bus|start-console|stop-units) rc=\d+ |^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ result exit=[0-35] [a-z-]+ /;
|
|
function slog(e){
|
|
const l=lines(e.slog);
|
|
for(const x of l){assert.match(x,STEP);assert.ok(!/stack-release:|fixture rehearse|PASS |FAIL |REHEARSAL/.test(x),x)}
|
|
assert.equal(statSync(e.slog).mode&0o777,0o640);
|
|
return l.map((x)=>x.slice(21));
|
|
}
|
|
const names=(e)=>slog(e).map((x)=>x.split(' ')[0]);
|
|
const result=(e)=>slog(e).at(-1);
|
|
const sudoKeys=(e)=>lines(join(e.sd,'sudo.log')).map((l)=>JSON.parse(l).key);
|
|
const lastAct=(e)=>logLines(e).filter((o)=>o.result==='started').at(-1).activation;
|
|
const mine=(e,a)=>logLines(e).filter((o)=>o.activation===a);
|
|
const open=(e)=>{const m=new Set();for(const o of logLines(e)){if(!o.activation)continue;if(o.result==='started')m.add(o.activation);else m.delete(o.activation)}return [...m]};
|
|
const final=(e,a)=>{const [s,f,...more]=mine(e,a);assert.equal(s.result,'started');assert.equal(more.length,0);return f};
|
|
const live=(e)=>[units(e)[e.bus].state,units(e)[e.con].state,units(e)[e.bus].wd.split('/').at(-1),units(e)[e.con].wd.split('/').at(-1)];
|
|
// Nothing the run did reached the args file: the same bytes and mode, and no temporary file left.
|
|
const argsKept=(e)=>{assert.equal(readFileSync(e.args,'utf8'),e.argsText);assert.equal(statSync(e.args).mode&0o777,0o640);assert.deepEqual(readdirSync(e.etc),['dev.args'])};
|
|
const argsCopy=(e)=>{const l=readFileSync(e.args,'utf8').split('\n');return l[l.indexOf('--copy')+1]};
|
|
const PRE=['start','fetch','ff','copy','hostcheck','units','plan','render-old-check','render-check','rehearse','rehearse-store'];
|
|
const A2=['stop-console','stop-bus','wait-down','wait-down'];
|
|
const R=[...A2,'revert','render-old','confirm-rollback'];
|
|
|
|
// #772: the proxy is another selected unit, including A2f and failed-upgrade recovery.
|
|
// The real renderer's installed-unit checks are exercised separately below; this fixture keeps
|
|
// the existing high-port health server and user-unit stand-ins for stack-release's real gate.
|
|
const PROXY='mosaic-proxy.service';
|
|
const P2=['stop-console','stop-bus','stop-proxy','wait-down','wait-down','wait-down'];
|
|
async function proxyHub(t){
|
|
const e=await useed(t);
|
|
e.argsText=e.argsText.replace('--\n','--profile\nfactory\n--proxy\n--\n');writeFileSync(e.args,e.argsText);
|
|
writeFileSync(join(e.units,PROXY),`[Service]\nWorkingDirectory=${e.hc}/${B}\nExecStart=${NODE} ${e.hc}/${B}/packages/model-proxy/src/run.mjs\n`);
|
|
sysctl(e,'start',PROXY);
|
|
return e;
|
|
}
|
|
const transitions=(e,from=0)=>lines(join(e.sd,'calls.log')).slice(from).filter((x)=>/^(stop|start) /.test(x));
|
|
const proxyLive=(e,copy)=>{
|
|
assert.deepEqual(live(e),['active','active',copy,copy]);
|
|
assert.deepEqual([units(e)[PROXY].state,units(e)[PROXY].wd],['active',`${e.hc}/${copy}`]);
|
|
assert.match(readFileSync(join(e.units,PROXY),'utf8'),new RegExp(`^WorkingDirectory=${e.hc}/${copy}$`,'m'));
|
|
};
|
|
const P_CYCLE=['stop mosaic-webui.service','stop [email protected]','stop mosaic-proxy.service','start mosaic-proxy.service','start [email protected]','start mosaic-webui.service'];
|
|
|
|
test('#772: a --proxy hub stops Console, bus, proxy, upgrades all three and starts proxy first',async(t)=>{
|
|
const e=await proxyHub(t),n=lines(join(e.sd,'calls.log')).length;
|
|
const r=await upgrade(e,C1);assert.equal(r.code,0,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...P2,'stage','render','confirm','args','result']);
|
|
assert.deepEqual(transitions(e,n),P_CYCLE);proxyLive(e,C1);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
|
|
assert.ok(JSON.parse(lines(join(e.sd,'render.log'))[0]).includes('--proxy'));
|
|
const m=lines(join(e.sd,'calls.log')).length;
|
|
const rb=await drive(e,C1,'rollback','--instance','dev','--to',B,...TO);assert.equal(rb.code,0,rb.all);
|
|
assert.deepEqual(transitions(e,m),P_CYCLE);proxyLive(e,B);argsKept(e);
|
|
});
|
|
|
|
test('#772: a failed upgrade restores and restarts all three units on the old copy',async(t)=>{
|
|
const e=await proxyHub(t),n=lines(join(e.sd,'calls.log')).length;
|
|
const before=Object.fromEntries([e.tmpl,e.con,PROXY].map((u)=>[u,readFileSync(join(e.units,u),'utf8')]));
|
|
fault(e,`healthz-${C1}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...P2,'stage','render','confirm',...P2,'revert','render-old','confirm-rollback','result']);
|
|
assert.deepEqual(transitions(e,n),[...P_CYCLE,...P_CYCLE]);proxyLive(e,B);argsKept(e);
|
|
for(const [u,text] of Object.entries(before))assert.equal(readFileSync(join(e.units,u),'utf8'),text,u);
|
|
assert.equal(final(e,lastAct(e)).rollback,'ok');
|
|
});
|
|
|
|
test('#772: A2f restarts the proxy before the old bus and Console after a refused stage',async(t)=>{
|
|
const e=await proxyHub(t),n=lines(join(e.sd,'calls.log')).length;
|
|
arm(e,'activate.sh',`: > ${join(e.sd,'faults','no-fragment')}\n`);arm(e,'gate.sh',`rm ${join(e.sd,'faults','no-fragment')}\n`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...P2,'stage','start-proxy','start-bus','start-console','gate','result']);
|
|
assert.deepEqual(transitions(e,n),P_CYCLE);proxyLive(e,B);argsKept(e);
|
|
});
|
|
|
|
test('#772: failed rollback leaves all three selected units stopped',async(t)=>{
|
|
const e=await proxyHub(t);
|
|
fault(e,`render-fails-${C1}`);fault(e,`render-fails-${B}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
|
|
assert.deepEqual([units(e)[e.con].state,units(e)[e.bus].state,units(e)[PROXY].state],['inactive','inactive','inactive']);
|
|
assert.deepEqual(transitions(e).slice(-3),P_CYCLE.slice(0,3));argsKept(e);
|
|
});
|
|
|
|
test('#772: a failed proxy restart in A2f stops all three and never starts its dependents',async(t)=>{
|
|
const e=await proxyHub(t),n=lines(join(e.sd,'calls.log')).length;
|
|
arm(e,'activate.sh',`: > ${join(e.sd,'faults','no-fragment')}\n`);
|
|
fault(e,`root-start-fails-${PROXY}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...P2,'stage','start-proxy','stop-units','result']);
|
|
assert.deepEqual(transitions(e,n),[...P_CYCLE.slice(0,3),...P_CYCLE.slice(0,3)]);
|
|
assert.deepEqual([units(e)[e.con].state,units(e)[e.bus].state,units(e)[PROXY].state],['inactive','inactive','inactive']);argsKept(e);
|
|
});
|
|
|
|
test('#772: a hub without --proxy keeps its two-unit upgrade and never creates or calls a proxy unit',async(t)=>{
|
|
const e=await useed(t),n=lines(join(e.sd,'calls.log')).length;
|
|
const r=await upgrade(e,C1);assert.equal(r.code,0,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
|
|
assert.deepEqual(transitions(e,n),['stop mosaic-webui.service','stop [email protected]','start [email protected]','start mosaic-webui.service']);
|
|
assert.ok(!existsSync(join(e.units,PROXY)));assert.ok(!lines(join(e.sd,'calls.log')).some((x)=>x.includes(PROXY)));
|
|
assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
});
|
|
|
|
test('#772: malformed proxy args refuse before any unit or user command, for upgrade and rollback',async(t)=>{
|
|
const e=await proxyHub(t),n=lines(join(e.sd,'calls.log')).length;
|
|
for(const bad of ['--proxy=true','--proxy true','--proxy\ntrue'])for(const op of ['upgrade','rollback']){
|
|
const text=e.argsText.replace('--proxy\n',bad+'\n');writeFileSync(e.args,text);
|
|
const r=op==='upgrade'?await upgrade(e,C1):await drive(e,B,'rollback','--instance','dev','--to',B,...TO);
|
|
assert.equal(r.code,3,r.all);assert.match(result(e),/^result exit=3 args-unreadable /);
|
|
assert.equal(readFileSync(e.args,'utf8'),text);assert.deepEqual(readdirSync(e.etc),['dev.args']);
|
|
}
|
|
assert.deepEqual(transitions(e,n),[]);assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'render.log')));proxyLive(e,B);
|
|
});
|
|
|
|
test('#772: an absent, linked, malformed or disagreeing proxy unit refuses before stopping anything',async(t)=>{
|
|
const e=await proxyHub(t),file=join(e.units,PROXY),text=readFileSync(file,'utf8'),n=lines(join(e.sd,'calls.log')).length;
|
|
for(const [why,put,code] of [
|
|
['absent',()=>{},'units-unreadable'],
|
|
['linked',()=>symlinkSync(join(e.units,e.con),file),'units-unreadable'],
|
|
['malformed',()=>writeFileSync(file,'WorkingDirectory=/foreign/copy\n'),'units-unreadable'],
|
|
['disagreeing',()=>writeFileSync(file,text.replaceAll(B,C1)),'units-disagree'],
|
|
]){
|
|
rmSync(file,{force:true});put();
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,`${why}: ${r.all}`);assert.match(result(e),new RegExp(`^result exit=3 ${code} `),why);argsKept(e);
|
|
}
|
|
writeFileSync(file,text);
|
|
e.argsText=e.argsText.replace('--proxy\n','');writeFileSync(e.args,e.argsText);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);assert.match(result(e),/^result exit=3 units-unreadable /);argsKept(e);
|
|
assert.deepEqual(transitions(e,n),[]);proxyLive(e,B);
|
|
});
|
|
|
|
// --- r6 sequences 1-8, through the driver ---
|
|
|
|
test('row 233: sequence 1: confirm fails healthz, the driver reverts, renders the old copy and confirm --rollback --cause A5 passes: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`healthz-${C1}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
assert.equal(r.err.split('\n').at(-2),`stack-upgrade: failed: A5 failed; rolled back to ${B}, which passed the gate`);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm',...R,'result']);
|
|
assert.equal(result(e),`result exit=1 failed A5 failed; rolled back to ${B}, which passed the gate`);
|
|
const a=lastAct(e),l=final(e,a);assert.deepEqual([l.op,l.result,l.code,l.rollback,l.from,l.to],['activate','gate-failed','healthz','ok',B,C1]);
|
|
assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',B,B]);
|
|
argsKept(e);
|
|
// The stack's user's steps went through sudo in the driver's form, as fixture, the pinned node first.
|
|
for(const l of lines(join(e.sd,'sudo.log')).map((x)=>JSON.parse(x)))assert.deepEqual([l.user,l.path],['fixture',`${dirname(NODE)}:/usr/local/bin:/usr/bin:/bin`]);
|
|
assert.deepEqual(sudoKeys(e),['git-fetch','git-merge','stack-copy.sh','activate-plan','rehearse','activate','confirm','revert','confirm-rollback']);
|
|
});
|
|
|
|
test('row 233: sequence 2: activate --migrate fails at the wrapper, revert restores the store, confirm --rollback --cause A3: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
const s0=sha(e.store);
|
|
arm(e,'activate.sh',`chmod 0555 ${e.bin}\n`);arm(e,'revert.sh',`chmod 0755 ${e.bin}\n`);
|
|
const r=await upgrade(e,C,'--migrate');assert.equal(r.code,1,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage',...R,'result']);
|
|
assert.equal(result(e),`result exit=1 failed A3 failed; rolled back to ${B}, which passed the gate`);
|
|
const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['failed','wrapper','ok']);
|
|
assert.equal(storeVersion(e.store,true),'3g');assert.deepEqual(live(e),['active','active',B,B]);
|
|
// The bus holds the store open again; its bytes are the backup's once it stops.
|
|
sysctl(e,'stop',e.bus);assert.equal(sha(e.store),s0);
|
|
argsKept(e);
|
|
});
|
|
|
|
test('row 233: sequence 3: root\'s render of the new copy fails, the driver rolls back with --cause A4, and holds the lock to the end: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`render-fails-${C1}`);fault(e,`render-probe-${B}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
// The probe ran inside render-old, during the rollback, and its one line is its refusal.
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render',...A2,'revert','result','render-old','confirm-rollback','result']);
|
|
const p=JSON.parse(readFileSync(join(e.sd,'probe.json'),'utf8'));assert.equal(p.code,3,p.err);
|
|
assert.equal(p.err,`stack-upgrade: upgrade-running: another stack-upgrade.sh run holds ${e.lockf} (nothing was stopped)\n`);
|
|
assert.match(slog(e).at(-4),/^result exit=3 upgrade-running /);
|
|
assert.match(slog(e).find((x)=>x.startsWith('render ')),/^render rc=1 /);
|
|
const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['failed','units-start','ok']);
|
|
assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: sequence 4: a capture failure is exit 3 from the stage, and A2f restarts the old units and gates them: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
arm(e,'activate.sh',`: > ${join(e.sd,'faults','no-fragment')}\n`);arm(e,'gate.sh',`rm ${join(e.sd,'faults','no-fragment')}\n`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','start-bus','start-console','gate','result']);
|
|
assert.match(slog(e).find((x)=>x.startsWith('stage ')),/^stage rc=3 /);
|
|
assert.equal(result(e),`result exit=1 failed A3 failed; ${B} is running again and passed the gate`);
|
|
const a=lastAct(e);assert.deepEqual(mine(e,a).map((o)=>[o.result,o.code]),[['started',undefined],['refused','capture-failed']]);
|
|
assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: N1: a bus that outlives A2\'s timeout keeps its old start, so A2f\'s gate fails stale-start: exit 2, both units stopped',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`stop-noop-${e.bus}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'start-bus','start-console','gate','stop-units','result']);
|
|
assert.ok(slog(e).includes(`wait-down rc=1 unit: ${e.bus} state: active`),slog(e).join('\n'));
|
|
assert.ok(r.out.includes(`gate failed on ${B}: stale-start`),r.out);
|
|
assert.equal(result(e),`result exit=2 rollback-failed A2 failed and ${B} failed its gate after the restart; both units are stopped`);
|
|
assert.deepEqual(live(e).slice(0,2),['inactive','inactive']);
|
|
assert.ok(!sudoKeys(e).includes('activate'));argsKept(e);
|
|
});
|
|
|
|
test('row 233: sequence 5: confirm of the wrong copy is refused, and the driver rolls back with --cause A5: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
arm(e,'confirm.id',B);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,1,r.all);
|
|
assert.ok(r.out.includes('refusing open-activation')||r.err.includes('refusing open-activation'),r.all);
|
|
assert.match(slog(e).find((x)=>x.startsWith('confirm ')),/^confirm rc=3 /);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm',...R,'result']);
|
|
const l=final(e,lastAct(e));assert.deepEqual([l.result,l.code,l.rollback],['gate-failed','unrecorded','ok']);
|
|
assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: sequence 6: a revert that fails stops both units at exit 2, and the next run refuses at its --plan with nothing stopped',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`render-fails-${C1}`);
|
|
arm(e,'revert.sh',`for d in ${join(e.state,'stack-rollback')}/*/; do [ -f "$d/AWAITING-UNITS" ] && rm "$d/wrapper"; done\n`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,2,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render',...A2,'revert','stop-units','result']);
|
|
assert.equal(result(e),`result exit=2 rollback-failed A4 failed and revert failed; both units are stopped`);
|
|
const a=lastAct(e);assert.deepEqual(open(e),[a]);assert.deepEqual(live(e).slice(0,2),['inactive','inactive']);
|
|
argsKept(e);
|
|
const n=slog(e).length,sc=lines(join(e.sd,'root-systemctl.log')).length;
|
|
rmSync(join(e.sd,'faults',`render-fails-${C1}`));
|
|
const again=await upgrade(e,C1);assert.equal(again.code,3,again.all);
|
|
assert.ok(again.out.includes(`refuses interrupted-run: activation ${a} started and has no final line`),again.out);
|
|
assert.deepEqual(names(e).slice(n),PRE.slice(0,7).concat('result'));
|
|
assert.match(result(e),/^result exit=3 plan-refused /);
|
|
assert.equal(lines(join(e.sd,'root-systemctl.log')).length,sc);argsKept(e);
|
|
});
|
|
|
|
test('row 233: sequence 7: two upgrades and a rollback --to the previous copy with --restore, the args file following each confirm',async(t)=>{
|
|
const e=await useed(t);
|
|
// The args file at confirm still names the old copy; the new one moves in after it.
|
|
arm(e,'confirm.sh',`cp ${e.args} ${join(e.sd,'args-at-confirm')}\n`);
|
|
const one=await upgrade(e,C1);assert.equal(one.code,0,one.all);
|
|
assert.equal(one.out.split('\n').at(-2),`stack-upgrade: upgrade ok: ${C1} is live for acme`);
|
|
assert.equal(readFileSync(join(e.sd,'args-at-confirm'),'utf8'),e.argsText);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
|
|
assert.equal(statSync(e.args).mode&0o777,0o640);assert.deepEqual(readdirSync(e.etc),['dev.args']);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
|
|
assert.ok(slog(e).includes(`args rc=0 copy: ${C1}`));
|
|
const a1=lastAct(e);
|
|
const two=await upgrade(e,C,'--migrate');assert.equal(two.code,0,two.all);
|
|
const a2=lastAct(e);assert.equal(argsCopy(e),C);
|
|
const l1=final(e,a1),l2=final(e,a2);
|
|
assert.deepEqual([l1.op,l1.result,l1.from,l1.to],['activate','ok',B,C1]);
|
|
assert.deepEqual([l2.op,l2.result,l2.from,l2.to,l2.backup],['activate','ok',C1,C,a2]);
|
|
assert.equal(storeVersion(e.store,true),'3h');
|
|
const n=slog(e).length;
|
|
const rb=await drive(e,C,'rollback','--instance','dev','--to',C1,'--restore',a2,...TO);assert.equal(rb.code,0,rb.all);
|
|
assert.equal(rb.out.split('\n').at(-2),`stack-upgrade: rollback ok: ${C1} is live for acme`);
|
|
// A rollback fetches, copies and rehearses nothing.
|
|
assert.deepEqual(names(e).slice(n),['start','hostcheck','units','plan','render-old-check','render-check',...A2,'stage','render','confirm','args','result']);
|
|
const ra=lastAct(e),l3=final(e,ra);assert.deepEqual([l3.op,l3.result,l3.from,l3.to,l3.backup],['rollback','ok',C,C1,a2]);
|
|
assert.deepEqual([pointer(e).copy,pointer(e).schema],[C1,'3g']);assert.equal(storeVersion(e.store,true),'3g');
|
|
assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
const m=slog(e).length,sc=lines(join(e.sd,'root-systemctl.log')).length;
|
|
const no=await drive(e,C,'rollback','--instance','dev','--to',B,...TO);assert.equal(no.code,3,no.all);
|
|
assert.ok(no.all.includes(`refusing to-not-previous: --to ${B} is not the previous copy; the log names ${C}`),no.all);
|
|
assert.deepEqual(names(e).slice(m),['start','hostcheck','units','plan','result']);
|
|
assert.equal(lines(join(e.sd,'root-systemctl.log')).length,sc);assert.equal(argsCopy(e),C1);
|
|
});
|
|
|
|
test('row 240: I3 N1: a failed args move after A5 exits 5 args-failed, the new copy live and confirmed and both args files kept',async(t)=>{
|
|
const e=await useed(t);
|
|
// A read-only /etc/mosaic-stack at confirm: the stage and the render are done, and the move fails.
|
|
// The fixture's cleanup runs chmod -R u+w before it removes the root, so the 0555 dir needs no hook here.
|
|
arm(e,'confirm.sh',`chmod 0555 ${e.etc}\n`);
|
|
const l=await upgrade(e,C1);assert.equal(l.code,5,l.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
|
|
assert.ok(slog(e).includes(`args rc=1 copy: ${C1}`));
|
|
assert.match(result(e),new RegExp(`^result exit=5 args-failed ${C1} is live and confirmed, but ${e.args} still names ${B}; put ${e.etc}/\\.dev\\.args\\.\\w{6} in its place$`));
|
|
assert.deepEqual(open(e),[]);assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText);
|
|
const left=readdirSync(e.etc).filter((n)=>n!=='dev.args');assert.equal(left.length,1);
|
|
assert.equal(readFileSync(join(e.etc,left[0]),'utf8'),e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
|
|
});
|
|
|
|
test('row 233: sequence 8: a migration that writes and fails: the driver reverts to the backup byte for byte, --cause A3: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
const s0=sha(e.store);
|
|
fault(e,'migrate-fails');
|
|
const r=await upgrade(e,C,'--migrate');assert.equal(r.code,1,r.all);
|
|
assert.match(slog(e).find((x)=>x.startsWith('stage ')),/^stage rc=1 /);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage',...R,'result']);
|
|
const a=lastAct(e),l=final(e,a);assert.deepEqual([l.result,l.code,l.rollback],['failed','migrate-failed','ok']);
|
|
sysctl(e,'stop',e.bus);assert.equal(sha(e.store),s0);assert.equal(storeVersion(e.store,false),'3g');
|
|
argsKept(e);
|
|
});
|
|
|
|
// --- The lock, A1's refusals and the rehearsal ---
|
|
|
|
test('row 233: a second run while the lock is held refuses upgrade-running with one log line and no call',async(t)=>{
|
|
const e=await useed(t);
|
|
mkdirSync(e.run,{recursive:true});
|
|
const holder=spawn('flock',[e.lockf,'sleep','60'],{detached:true,stdio:'ignore'});
|
|
t.after(()=>{try{process.kill(-holder.pid,'SIGKILL')}catch{}});
|
|
for(let i=0;i<100&&execFileSync('bash',['-c','flock -n "$1" true && echo free || echo held','_',e.lockf],{encoding:'utf8'}).trim()==='free';i++)await new Promise((r)=>setTimeout(r,50));
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.equal(r.err,`stack-upgrade: upgrade-running: another stack-upgrade.sh run holds ${e.lockf} (nothing was stopped)\n`);
|
|
assert.deepEqual(names(e),['result']);
|
|
assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
|
|
assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: A1 refuses open-activation on any AWAITING-UNITS file, through the stack\'s user\'s --plan, with nothing stopped',async(t)=>{
|
|
const e=await useed(t);
|
|
const fake=join(e.state,'stack-rollback',`20000101T000000Z-${C1.slice(0,12)}`);mkdirSync(fake,{recursive:true});writeFileSync(join(fake,'AWAITING-UNITS'),`${C1}\n`);
|
|
const logs=logLines(e).length;
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.ok(r.out.includes('refuses open-activation'),r.out);
|
|
assert.deepEqual(names(e),[...PRE.slice(0,7),'result']);assert.match(result(e),/^result exit=3 plan-refused /);
|
|
assert.equal(logLines(e).length,logs);assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
|
|
assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: a target that migrates refuses migrate-required without --migrate, with nothing stopped',async(t)=>{
|
|
const e=await useed(t);
|
|
const r=await upgrade(e,C);assert.equal(r.code,3,r.all);
|
|
assert.ok(r.out.includes('refuses migrate-required'),r.out);
|
|
assert.deepEqual(names(e),[...PRE.slice(0,7),'result']);assert.match(result(e),/^result exit=3 plan-refused /);
|
|
assert.ok(!sudoKeys(e).includes('rehearse'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
|
|
assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: the rehearsal runs after the plan and before A2; a failed one refuses rehearsal-failed with nothing stopped',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,'rehearse-rc','1');
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.ok(r.out.includes('fixture rehearse: '),r.out);
|
|
assert.deepEqual(names(e),[...PRE,'result']);
|
|
assert.equal(result(e),'result exit=3 rehearsal-failed the rehearsal exited 1 (nothing was stopped)');
|
|
const rh=lines(join(e.sd,'sudo.log')).map((l)=>JSON.parse(l)).find((l)=>l.key==='rehearse');
|
|
assert.deepEqual(rh.cmd.slice(1),['rehearse',C1,'acme','--copies',e.hc,'--backup','last','--port','7339','--timeout','3']);
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('row 233: the rehearsal\'s BACKUP_STORE line goes to the step log after a match, a malformed one as none, the rest of its output never',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,'rehearse-rc','1');
|
|
const r=await upgrade(e,C1,'--rehearse-port','7449');assert.equal(r.code,3,r.all);
|
|
assert.ok(slog(e).includes('rehearse-store rc=0 BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g'),slog(e).join('\n'));
|
|
assert.ok(slog(e).find((x)=>x.startsWith('rehearse ')).includes(' --port 7449 '));
|
|
for(const bad of ['BACKUP_STORE ../../etc schema 3g','BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g; rm -rf /','stack-release: BACKUP_STORE 20261020T000000Z-0123456789ab schema 3g','']){
|
|
fault(e,'rehearse-store-line',bad);
|
|
const n=slog(e).length;
|
|
const b=await upgrade(e,C1);assert.equal(b.code,3,b.all);
|
|
assert.deepEqual(slog(e).slice(n).filter((x)=>x.startsWith('rehearse-store ')),['rehearse-store rc=0 BACKUP_STORE none'],bad);
|
|
}
|
|
});
|
|
|
|
test('row 233: a real rehearsal with no backup in the log refuses no-backup, and goes on only with --no-rehearsal, which logs the skip',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,'real-rehearse');
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.ok(r.out.includes('FAIL no-backup: the log records no backup\nREHEARSAL_FAIL\n'),r.out);
|
|
assert.equal(result(e),'result exit=3 no-backup the log records no backup to rehearse against; only a go that names --no-rehearsal goes on (nothing was stopped)');
|
|
assert.ok(slog(e).includes('rehearse-store rc=0 BACKUP_STORE none'));
|
|
assert.ok(!existsSync(join(e.data,'rehearsals')));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));
|
|
const n=slog(e).length;
|
|
const s=await upgrade(e,C1,'--no-rehearsal');assert.equal(s.code,0,s.all);
|
|
assert.deepEqual(names(e).slice(n),[...PRE,'rehearsal-skipped',...A2,'stage','render','confirm','args','result']);
|
|
assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
});
|
|
|
|
test('row 233: a real rehearsal after an upgrade names the log\'s newest backup; --no-rehearsal with a backup refuses rehearsal-required',async(t)=>{
|
|
const e=await useed(t);
|
|
const one=await upgrade(e,C1);assert.equal(one.code,0,one.all);
|
|
const a1=lastAct(e);
|
|
fault(e,'real-rehearse');
|
|
const n=slog(e).length;
|
|
// The stub copies fail the rehearsal right after it names the backup (copy-without-no-launch).
|
|
const r=await upgrade(e,C,'--migrate');assert.equal(r.code,3,r.all);
|
|
assert.ok(r.out.includes(`PASS backup-last: the log's newest backup is ${a1}\n`),r.out);
|
|
assert.deepEqual(names(e).slice(n),[...PRE,'result']);assert.match(result(e),/^result exit=3 rehearsal-failed the rehearsal exited 1 /);
|
|
rmSync(join(e.sd,'faults','real-rehearse'));
|
|
const m=slog(e).length;
|
|
const q=await upgrade(e,C,'--migrate','--no-rehearsal');assert.equal(q.code,3,q.all);
|
|
assert.deepEqual(names(e).slice(m),[...PRE,'result']);
|
|
assert.equal(result(e),'result exit=3 rehearsal-required the log records a backup and the rehearsal ran; --no-rehearsal is only for a stack with none (nothing was stopped)');
|
|
assert.equal(argsCopy(e),C1);assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
});
|
|
|
|
// --- Before A1 and the args file ---
|
|
|
|
test('row 233: the args file must be the running user\'s, with no group or other write, one --copy, and the copy the units run',async(t)=>{
|
|
const e=await useed(t);
|
|
const tryArgs=async(text,mode,code,why)=>{
|
|
writeFileSync(e.args,text);chmodSync(e.args,mode);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.match(slog(e).at(-1),new RegExp(`^result exit=3 ${code} `),why);
|
|
assert.ok(!slog(e).slice(n).some((x)=>/^(stop|stage|render|confirm)/.test(x)),why);
|
|
};
|
|
await tryArgs(e.argsText,0o660,'args-unreadable','group write');
|
|
await tryArgs(e.argsText,0o642,'args-unreadable','other write');
|
|
await tryArgs(e.argsText.replace('--\n',`--copy\n${C1}\n--\n`),0o640,'args-unreadable','two --copy');
|
|
await tryArgs(e.argsText.replace('--\n','--tester\n--\n'),0o640,'args-unreadable','an unknown flag');
|
|
await tryArgs(e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`),0o640,'args-differ','the units run B');
|
|
rmSync(e.args);symlinkSync(join(e.sd,'probe.sh'),e.args);
|
|
const l=await upgrade(e,C1);assert.equal(l.code,3);assert.match(result(e),/^result exit=3 args-unreadable /);
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
// R2-F1 (#385, PR 1836): base case2-unit.sh required --send for the library profile, and the current one
|
|
// refuses it, so a legacy args file would stop both units at A2 and fail the A4 render and render-old.
|
|
// read_args refuses it first, for an upgrade and a rollback alike, and never rewrites the file.
|
|
const legacySend=(f)=>`${f} has --send after --, which case2-unit.sh refuses for the library profile: remove the --send line, then rerun; the library Console is read-only until #412 (#385) (nothing was stopped)`;
|
|
test('#385 R2-F1: a library args file holding --send after -- refuses args-unreadable before anything stops, for upgrade and rollback',async(t)=>{
|
|
const e=await useed(t);
|
|
const legacy=e.argsText+'--send\n';
|
|
const cases=[
|
|
['the default profile',legacy,()=>upgrade(e,C1)],
|
|
['an explicit --profile library',legacy.replace('--\n','--profile\nlibrary\n--\n'),()=>upgrade(e,C1)],
|
|
['the last --profile is library',legacy.replace('--\n','--profile\nfactory\n--profile\nlibrary\n--\n'),()=>upgrade(e,C1)],
|
|
['a rollback',legacy,()=>drive(e,B,'rollback','--instance','dev','--to',B,...TO)],
|
|
];
|
|
for(const [why,text,run] of cases){
|
|
writeFileSync(e.args,text);chmodSync(e.args,0o640);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await run();assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.equal(r.err.split('\n').at(-2),`stack-upgrade: args-unreadable: ${legacySend(e.args)}`,why);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','result'],why);
|
|
assert.equal(slog(e).at(-1),`result exit=3 args-unreadable ${legacySend(e.args)}`,why);
|
|
// The file is left as the operator wrote it.
|
|
assert.equal(readFileSync(e.args,'utf8'),text,why);assert.deepEqual(readdirSync(e.etc),['dev.args'],why);
|
|
}
|
|
assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'render.log')));
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('#385 R2-F1 controls: library args without --send and a factory args file pass read_args as before',async(t)=>{
|
|
const e=await useed(t);
|
|
// A bundle that does not exist stops the run at its first step after read_args, with nothing stopped.
|
|
const gone=join(e.root,'no.bundle');
|
|
for(const [why,text] of [
|
|
['library, no --send',e.argsText.replace('--\n','--profile\nlibrary\n--\n')],
|
|
['default profile, no --send',e.argsText],
|
|
['factory with --send, which case2-unit.sh refuses at the render as before',e.argsText.replace('--\n','--profile\nfactory\n--\n')+'--send\n'],
|
|
['factory, no --send',e.argsText.replace('--\n','--profile\nfactory\n--\n')],
|
|
]){
|
|
writeFileSync(e.args,text);chmodSync(e.args,0o640);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await drive(e,C1,'upgrade','--instance','dev','--bundle',gone,'--sha',C1,...TO);assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','fetch','result'],why);
|
|
assert.match(slog(e).at(-1),/^result exit=3 fetch-failed /,why);
|
|
}
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
// Rename r2, "Selector scopes and parser matrix": --org is the same option as --business before -- (the
|
|
// unit's flags) and after it (the serve flags). selArgs moves the business into each scope as given: the
|
|
// unit's selectors first, the serve flags' last, the other lines as useed wrote them.
|
|
const selArgs=(e,outer,serve)=>{
|
|
const l=e.argsText.trimEnd().split('\n'),cut=l.indexOf('--'),drop=(a)=>{a.splice(a.indexOf('--business'),2);return a};
|
|
return [...outer,...drop(l.slice(0,cut)),'--',...drop(l.slice(cut+1)),...serve].join('\n')+'\n';
|
|
};
|
|
test('rename r2: --org or --business names the business before and after --, read_args passes either spelling or both, matched, in either order',async(t)=>{
|
|
const e=await useed(t);
|
|
// A bundle that does not exist stops the run at its first step after read_args, with nothing stopped.
|
|
const gone=join(e.root,'no.bundle');
|
|
for(const [outer,serve] of [
|
|
[['--org','acme'],['--org','acme']],
|
|
[['--business','acme'],['--org','acme']],
|
|
[['--org','acme'],['--business','acme']],
|
|
[['--business','acme','--org','acme'],['--org','acme','--business','acme']],
|
|
[['--org','acme','--business','acme'],['--business','acme','--org','acme']],
|
|
]){
|
|
const why=`${outer.join(' ')} -- ${serve.join(' ')}`,text=selArgs(e,outer,serve);
|
|
writeFileSync(e.args,text);chmodSync(e.args,0o640);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await drive(e,C1,'upgrade','--instance','dev','--bundle',gone,'--sha',C1,...TO);assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','fetch','result'],why);
|
|
assert.match(slog(e).at(-1),/^result exit=3 fetch-failed /,why);
|
|
assert.equal(readFileSync(e.args,'utf8'),text,why);
|
|
}
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('rename r2: an upgrade from an args file that spells both selectors --org keeps every line but --copy\'s value, and the render gets them as written',async(t)=>{
|
|
const e=await useed(t);
|
|
e.argsText=selArgs(e,['--org','acme'],['--org','acme']);writeFileSync(e.args,e.argsText);
|
|
arm(e,'confirm.sh',`cp ${e.args} ${join(e.sd,'args-at-confirm')}\n`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,0,r.all);
|
|
assert.equal(r.out.split('\n').at(-2),`stack-upgrade: upgrade ok: ${C1} is live for acme`);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','render','confirm','args','result']);
|
|
assert.equal(readFileSync(join(e.sd,'args-at-confirm'),'utf8'),e.argsText);
|
|
const after=e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`);assert.notEqual(after,e.argsText);
|
|
assert.equal(readFileSync(e.args,'utf8'),after);
|
|
assert.equal(statSync(e.args).mode&0o777,0o640);assert.deepEqual(readdirSync(e.etc),['dev.args']);
|
|
// #1873: before anything stops, --check renders the installed copy, then the target, from the same lines.
|
|
const al=after.trimEnd().split('\n');
|
|
assert.deepEqual(lines(join(e.sd,'render.log')).map((x)=>JSON.parse(x)),[['--check',...e.argsText.trimEnd().split('\n')],['--check',...al],al]);
|
|
// The stack's user's steps get the business as a word, never a spelling.
|
|
for(const k of ['activate-plan','rehearse','activate','confirm'])assert.ok(JSON.parse(lines(join(e.sd,'sudo.log')).find((x)=>JSON.parse(x).key===k)).cmd.includes('acme'),k);
|
|
assert.ok(!lines(join(e.sd,'sudo.log')).some((x)=>JSON.parse(x).cmd.includes('--org')));
|
|
assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
});
|
|
|
|
test('rename r2: a bad selector in either scope, or scopes that name different businesses, refuse args-unreadable before anything stops',async(t)=>{
|
|
const e=await useed(t),f=e.args;
|
|
const l=e.argsText.trimEnd().split('\n'),pre=l.slice(0,l.indexOf('--')).filter((x,i,a)=>x!=='--business'&&a[i-1]!=='--business');
|
|
const before=(m)=>`${f} before --: ${m}`,after=(m)=>`${f} after --: ${m}`;
|
|
const cases=[
|
|
['unit: two spellings, two businesses',selArgs(e,['--business','acme','--org','beta'],['--business','acme']),before('--business acme and --org beta conflict')],
|
|
['unit: two businesses, the other order',selArgs(e,['--org','acme','--business','beta'],['--business','acme']),before('--org acme and --business beta conflict')],
|
|
['unit: --org twice',selArgs(e,['--org','acme','--org','acme'],['--business','acme']),before('--org given twice')],
|
|
['unit: --business twice',selArgs(e,['--business','acme','--business','acme'],['--business','acme']),before('--business given twice')],
|
|
['unit: --org at the end',[...pre,'--org'].join('\n')+'\n',before('--org needs one value')],
|
|
['unit: --org with an empty line',selArgs(e,['--org',''],['--business','acme']),before('--org needs one value')],
|
|
['unit: --org before a flag',selArgs(e,['--org'],['--business','acme']),before('--org needs one value, not --user')],
|
|
['unit: --business before --',[...pre,'--business','--','--port',e.port,'--business','acme'].join('\n')+'\n',before('--business needs one value, not --')],
|
|
['unit: an invalid business',selArgs(e,['--org','Acme'],['--org','Acme']),before('--org Acme is not valid')],
|
|
['unit: --org as --home\'s value is no selector',selArgs(e,['--business','acme'],['--business','acme']).replace(`--home\n${e.env.HOME}\n`,'--home\n--org\n'),`${f} lacks a plain --user, --home, --business, --node or --copy`],
|
|
['serve: two spellings, two businesses',selArgs(e,['--business','acme'],['--business','acme','--org','beta']),after('--business acme and --org beta conflict')],
|
|
['serve: two businesses, the other order',selArgs(e,['--business','acme'],['--org','beta','--business','acme']),after('--org beta and --business acme conflict')],
|
|
['serve: --org twice',selArgs(e,['--business','acme'],['--org','acme','--org','acme']),after('--org given twice')],
|
|
['serve: --business twice',selArgs(e,['--org','acme'],['--business','acme','--business','acme']),after('--business given twice')],
|
|
['serve: --org at the end',selArgs(e,['--business','acme'],['--org']),after('--org needs one value')],
|
|
['serve: --org before a flag',selArgs(e,['--business','acme'],['--org','--sso']),after('--org needs one value, not --sso')],
|
|
['serve: an invalid business',selArgs(e,['--business','acme'],['--org','acme_x']),after('--org acme_x is not valid')],
|
|
['serve: another business',selArgs(e,['--business','acme'],['--org','beta']),`${f} names business acme before -- and beta after it`],
|
|
['serve: another business, unit spelled --org',selArgs(e,['--org','acme'],['--business','beta']),`${f} names business acme before -- and beta after it`],
|
|
['serve: no selector',selArgs(e,['--business','acme'],[]),`${f} needs --business or --org after --, as case2-unit.sh does`],
|
|
['serve: --business as --allow-user\'s value is no selector',selArgs(e,['--business','acme'],['--allow-user','--business']),`${f} needs --business or --org after --, as case2-unit.sh does`],
|
|
['serve: no -- at all',[...pre,'--org','acme'].join('\n')+'\n',`${f} needs --business or --org after --, as case2-unit.sh does`],
|
|
['serve: a flag case2-unit.sh does not take',selArgs(e,['--business','acme'],['--business','acme','--tester']),`${f} has --tester after --, which case2-unit.sh does not take`],
|
|
];
|
|
for(const [why,text,msg] of cases){
|
|
writeFileSync(e.args,text);chmodSync(e.args,0o640);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.equal(r.err.split('\n').at(-2),`stack-upgrade: args-unreadable: ${msg} (nothing was stopped)`,why);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','result'],why);
|
|
assert.equal(readFileSync(e.args,'utf8'),text,why);assert.deepEqual(readdirSync(e.etc),['dev.args'],why);
|
|
}
|
|
// A rollback reads the same file the same way.
|
|
writeFileSync(e.args,cases[0][1]);
|
|
const rb=await drive(e,B,'rollback','--instance','dev','--to',B,...TO);assert.equal(rb.code,3,rb.all);
|
|
assert.equal(rb.err.split('\n').at(-2),`stack-upgrade: args-unreadable: ${cases[0][2]} (nothing was stopped)`);
|
|
assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'render.log')));
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('rename r2: a checkout without scripts/selector.sh refuses checkout-unreadable before reading the args file',async(t)=>{
|
|
const e=await useed(t),d=join(realpathSync(e.root),'lone');
|
|
mkdirSync(join(d,'scripts'),{recursive:true});copyFileSync(join(checkout(C1),'scripts','stack-upgrade.sh'),join(d,'scripts','stack-upgrade.sh'));
|
|
const r=spawnSync('bash',[join(d,'scripts','stack-upgrade.sh'),'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1,...TO],{env:e.denv,encoding:'utf8'});
|
|
assert.equal(r.status,3,r.stdout+r.stderr);
|
|
assert.equal(r.stderr.split('\n').at(-2),`stack-upgrade: checkout-unreadable: cannot read ${d}/scripts/selector.sh (nothing was stopped)`);
|
|
assert.deepEqual(slog(e).map((x)=>x.split(' ')[0]),['start','result']);
|
|
assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));argsKept(e);
|
|
});
|
|
|
|
// R3-F1 (#385, PR 1836): a library host rendered by base case2-unit.sh has --send in its installed Console
|
|
// unit's ExecStart, not only in its args file. With the args line removed, A2 would stop both units and A4
|
|
// and render-old would both refuse the installed unit (check_unit's ExecStart and the full-text match), so
|
|
// the driver refuses console-unit-legacy right after read_args. These cases run real renderers: base
|
|
// 09a847eda's case2-unit.sh, from git show into the module fixture, renders the legacy pair, and the
|
|
// checkout's own case2-unit.sh is A4's RENDER, both into an owned unit directory with fake getent and
|
|
// systemctl.
|
|
const BASE_REV='09a847eda';
|
|
// The real renderer requires a root-owned node path; the test runner's
|
|
// process.execPath may be an unprivileged toolchain installation.
|
|
const RENDER_NODE='/usr/bin/node';
|
|
let baseRenderer;
|
|
const baseCase2=()=>{
|
|
if(baseRenderer)return baseRenderer;
|
|
const d=join(base,'case2-base');
|
|
for(const p of ['scripts/case2-unit.sh','packages/cli/systemd/mosaic-bus.service.in','packages/webui/systemd/mosaic-webui.service.in']){
|
|
mkdirSync(dirname(join(d,p)),{recursive:true});writeFileSync(join(d,p),execFileSync('git',['-C',ROOT,'show',`${BASE_REV}:${p}`]),{mode:p.endsWith('.sh')?0o755:0o644});
|
|
}
|
|
return baseRenderer=join(d,'scripts','case2-unit.sh');
|
|
};
|
|
// The stack's home gets a proof file; dir is a Case 2 unit directory that A4's real renderer and the driver
|
|
// read. args(copy, extra) is a library args file's lines without --send; legacy() renders base's pair into dir.
|
|
function case2Host(e,dir){
|
|
const home=e.env.HOME,secrets=join(home,'.config','mosaic-dev','secrets');mkdirSync(secrets,{recursive:true,mode:0o700});
|
|
copyFileSync(e.cfg,join(home,'.config','mosaic-dev','config.json'));
|
|
const proof=join(secrets,'proxy-proof');writeFileSync(proof,'made-up-proof\n');chmodSync(proof,0o600);
|
|
const c2=join(e.root,'case2'),bin=join(c2,'bin'),log=join(c2,'systemctl.log');mkdirSync(bin,{recursive:true});mkdirSync(dir,{recursive:true});
|
|
const uid=process.getuid(),gid=process.getgid();
|
|
writeFileSync(join(bin,'getent'),`#!/bin/sh\ncase "$1 $2" in\n "passwd fixture"|"passwd ") echo "fixture:x:${uid}:${gid}::${home}:/usr/sbin/nologin" ;;\n "group ${gid}") echo "fixture:x:${gid}:" ;;\n *) exit 2 ;;\nesac\n`,{mode:0o755});
|
|
writeFileSync(join(bin,'systemctl'),`#!/bin/sh\necho "$*" >> '${log}'\n[ "$1" != is-active ] || exit 3\n`,{mode:0o755});
|
|
// M3: the current renderer queries the config under the selected uid for every
|
|
// profile. These copies deliberately stub the CLI; give that query its own
|
|
// fixture loader and sudo rather than invoking the machine's sudo.
|
|
for(const id of [B,C1,C]){
|
|
const d=join(COPIES,id),packages=join(d,'packages'),cli=join(packages,'cli'),src=join(cli,'src');
|
|
if(!existsSync(join(src,'config.mjs'))){
|
|
chmodSync(d,0o755);chmodSync(packages,0o755);mkdirSync(src,{recursive:true});
|
|
writeFileSync(join(src,'config.mjs'),`import fs from 'node:fs';import os from 'node:os';import path from 'node:path';export const loadSystem=()=>JSON.parse(fs.readFileSync(path.join(os.homedir(),'.config/mosaic-dev/config.json'),'utf8'));`);
|
|
chmodSync(join(src,'config.mjs'),0o444);chmodSync(src,0o555);chmodSync(cli,0o555);chmodSync(packages,0o555);chmodSync(d,0o555);
|
|
}
|
|
}
|
|
writeFileSync(join(bin,'sudo'),'#!/bin/sh\n[ "$1" = -u ] && [ "$2" = fixture ] && [ "$3" = -H ] || exit 97\nshift 3\nexec "$@"\n',{mode:0o755});
|
|
const env={PATH:`${bin}:${e.env.PATH}`,CASE2_UNIT_DIR:dir,CASE2_GETENT:join(bin,'getent'),CASE2_SYSTEMCTL:join(bin,'systemctl')};
|
|
const serve=['--port','443','--business','acme','--public-host','books.example','--auth-header','X-Auth-User','--allow-user','melanie','--proof-header','X-Proxy-Proof','--proof-file',proof];
|
|
const args=(copy=B,extra=[])=>['--user','fixture','--home',home,'--business','acme','--copy',copy,'--node',RENDER_NODE,...extra,'--',...serve];
|
|
const render=(script,a)=>{const r=spawnSync('bash',[script,...a],{env:{PATH:e.env.PATH,...env},encoding:'utf8'});return {code:r.status,out:r.stdout,err:r.stderr}};
|
|
const legacy=()=>{const a=args();a.splice(a.indexOf('melanie')+1,0,'--send');const r=render(baseCase2(),a);assert.equal(r.code,0,r.err);return r};
|
|
const bytes=()=>Object.fromEntries(readdirSync(dir).sort().map((n)=>[n,readFileSync(join(dir,n),'utf8')]));
|
|
const syslog=()=>lines(log);
|
|
// The driver reads dir, and A4's RENDER is the checkout's real case2-unit.sh with the same seams.
|
|
const use=(text)=>{writeFileSync(e.args,text);chmodSync(e.args,0o640);e.argsText=text;e.denv.STACK_UPGRADE_UNIT_DIR=dir;delete e.denv.STACK_UPGRADE_RENDER;Object.assign(e.denv,env)};
|
|
return {home,dir,args,render,legacy,bytes,syslog,use,con:join(dir,'mosaic-webui.service')};
|
|
}
|
|
|
|
test('#772: the real Case 2 renderer re-pins and rolls back all three factory units with the saved --proxy args',async(t)=>{
|
|
const e=await useed(t),h=case2Host(e,join(e.root,'case2','units')),script=join(checkout(C1),'scripts','case2-unit.sh');
|
|
const args=(copy)=>['--user','fixture','--home',h.home,'--business','acme','--copy',copy,'--node',RENDER_NODE,'--profile','factory','--proxy','--','--port','443','--business','acme'];
|
|
const first=h.render(script,args(B));assert.equal(first.code,0,first.out+first.err);
|
|
const before=h.bytes();
|
|
const next=h.render(script,args(C1));assert.equal(next.code,0,next.out+next.err);
|
|
for(const [u,text] of Object.entries(h.bytes()))assert.ok(text.includes(`WorkingDirectory=${e.hc}/${C1}\n`),u);
|
|
assert.ok(h.bytes()[PROXY].includes(`ExecStart=${RENDER_NODE} ${e.hc}/${C1}/packages/model-proxy/src/run.mjs\n`));
|
|
const changed=h.bytes(),n=h.syslog().length;
|
|
const missing=h.render(script,args(C1).filter((x)=>x!=='--proxy'));assert.equal(missing.code,1,missing.out+missing.err);
|
|
assert.deepEqual(h.bytes(),changed);assert.equal(h.syslog().length,n);
|
|
const back=h.render(script,args(B));assert.equal(back.code,0,back.out+back.err);assert.deepEqual(h.bytes(),before);
|
|
assert.deepEqual(h.syslog().filter((x)=>x.startsWith('start ')),[
|
|
'start mosaic-proxy.service','start [email protected]','start mosaic-webui.service',
|
|
'start mosaic-proxy.service','start [email protected]','start mosaic-webui.service',
|
|
'start mosaic-proxy.service','start [email protected]','start mosaic-webui.service',
|
|
]);
|
|
});
|
|
const legacyUnit=(f)=>`${f} runs serve with --send, so it was rendered before #385, and stack-upgrade.sh cannot migrate it to the read-only library Console: the units are left running; stop and report (docs/guides/host-run-sheet.md section 7) (nothing was stopped)`;
|
|
test('#385 R3-F1: an installed pre-#385 library Console unit with --send refuses console-unit-legacy before anything stops, for upgrade and rollback',async(t)=>{
|
|
const e=await useed(t),h=case2Host(e,join(e.root,'case2','units'));
|
|
h.legacy();
|
|
const c=readFileSync(h.con,'utf8');
|
|
assert.ok(c.includes(`\nExecStart=${RENDER_NODE} ${e.hc}/${B}/packages/webui/src/cli.mjs serve --port 443 --business acme --public-host books.example --auth-header X-Auth-User --allow-user melanie --send --proof-header X-Proxy-Proof --proof-file `),c);
|
|
// The operator removed the args file's --send line, as R3's section 7 said, so read_args passes.
|
|
h.use(h.args().join('\n')+'\n');
|
|
assert.ok(!e.argsText.includes('--send'));
|
|
const before=h.bytes(),sys=h.syslog().length;
|
|
// The premise: the current renderer, as A4 runs it, refuses the installed legacy unit and writes nothing.
|
|
const p=h.render(join(checkout(C1),'scripts','case2-unit.sh'),h.args(C1));
|
|
assert.equal(p.code,1,p.out+p.err);
|
|
assert.ok(p.err.startsWith(`case2-unit: installed ${h.con}: ExecStart= must be exactly once in [Service], as ExecStart=${RENDER_NODE} ${e.hc}/${B}/packages/webui/src/cli.mjs serve --port 443 `),p.err);
|
|
assert.deepEqual(h.bytes(),before);assert.equal(h.syslog().length,sys);
|
|
for(const [why,run] of [['upgrade',()=>upgrade(e,C1)],['rollback',()=>drive(e,B,'rollback','--instance','dev','--to',B,...TO)]]){
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await run();assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.equal(r.err.split('\n').at(-2),`stack-upgrade: console-unit-legacy: ${legacyUnit(h.con)}`,why);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','result'],why);
|
|
assert.equal(slog(e).at(-1),`result exit=3 console-unit-legacy ${legacyUnit(h.con)}`,why);
|
|
argsKept(e);assert.deepEqual(h.bytes(),before,why);
|
|
}
|
|
assert.ok(!existsSync(join(e.sd,'sudo.log')));assert.ok(!existsSync(join(e.sd,'render.log')));
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.equal(h.syslog().length,sys);
|
|
assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('#385 R3-F1 controls: a read-only library Console unit, a factory host\'s unit and a missing Console unit pass the check',async(t)=>{
|
|
const e=await useed(t);
|
|
const ro=case2Host(e,join(e.root,'case2','ro')),old=case2Host(e,join(e.root,'case2','legacy')),none=case2Host(e,join(e.root,'case2','none'));
|
|
const r0=ro.render(join(checkout(C1),'scripts','case2-unit.sh'),ro.args());assert.equal(r0.code,0,r0.err);
|
|
assert.ok(!readFileSync(ro.con,'utf8').includes('--send'));
|
|
old.legacy();
|
|
none.legacy();rmSync(none.con);
|
|
// A bundle that does not exist stops the run at its first step after the check, with nothing stopped.
|
|
const gone=join(e.root,'no.bundle');
|
|
for(const [why,h,extra] of [['a read-only library unit',ro,[]],['a factory args file over a unit holding --send',old,['--profile','factory']],['no installed Console unit',none,[]]]){
|
|
h.use(h.args(B,extra).join('\n')+'\n');
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await drive(e,C1,'upgrade','--instance','dev','--bundle',gone,'--sha',C1,...TO);assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.deepEqual(slog(e).slice(n).map((x)=>x.split(' ')[0]),['start','fetch','result'],why);
|
|
assert.match(slog(e).at(-1),/^result exit=3 fetch-failed /,why);
|
|
}
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('row 233: hostcheck (row 240): a nonzero --check or a missing file refuses profile-drift before the plan',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,'hostcheck-rc','1');
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.match(result(e),/^result exit=3 profile-drift /);
|
|
assert.deepEqual(lines(join(e.sd,'hostcheck.log')),['--check']);
|
|
assert.deepEqual(sudoKeys(e),['git-fetch','git-merge','stack-copy.sh']);
|
|
e.denv.STACK_UPGRADE_HOSTCHECK=join(e.root,'no-hostcheck');
|
|
const m=await upgrade(e,C1);assert.equal(m.code,3,m.all);
|
|
assert.equal(result(e),`result exit=3 profile-drift no executable ${join(e.root,'no-hostcheck')} (row 240) (nothing was stopped)`);
|
|
assert.ok(!sudoKeys(e).includes('activate-plan'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));argsKept(e);
|
|
});
|
|
|
|
test('row 233: the checkout must be at --sha (head-mismatch), and a commit the clone can\'t fast-forward to refuses fetch-failed',async(t)=>{
|
|
const e=await useed(t);
|
|
const r=await drive(e,C1,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C,'--migrate',...TO);assert.equal(r.code,3,r.all);
|
|
assert.equal(result(e),`result exit=3 head-mismatch this checkout, ${checkout(C1)}, is at ${C1}, not ${C} (nothing was stopped)`);
|
|
const u=await drive(e,U,'upgrade','--instance','dev','--bundle',BUNDLE,'--sha',U,...TO);assert.equal(u.code,3,u.all);
|
|
assert.match(result(e),/^result exit=3 fetch-failed .* fast-forward /);
|
|
assert.ok(!sudoKeys(e).includes('activate-plan'));assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));argsKept(e);
|
|
});
|
|
|
|
test('row 233: usage exits 4 before any log or lock',async(t)=>{
|
|
const e=await useed(t);
|
|
for(const a of [[],['status'],['upgrade','--instance','dev','--bundle',BUNDLE],['upgrade','--instance','other','--bundle',BUNDLE,'--sha',C1],['upgrade','--instance','dev','--bundle','rel/x','--sha',C1],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1.slice(0,12)],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1,'--to',B],['upgrade','--instance','dev','--bundle',BUNDLE,'--sha',C1,'--rehearse-port','80'],['rollback','--instance','dev','--to',B,'--migrate'],['rollback','--instance','dev','--to',B,'--events','3'],['rollback','--instance','dev'],['rollback','--instance','dev','--to',B,'--restore','x']]){
|
|
const r=await drive(e,C1,...a);assert.equal(r.code,4,a.join(' '));assert.match(r.err,/^usage: stack-upgrade.sh upgrade /,a.join(' '));
|
|
}
|
|
assert.ok(!existsSync(dirname(e.slog)));assert.ok(!existsSync(e.run));
|
|
});
|
|
|
|
// --- #1873 M3: render checks before anything stops, and --next-args ---
|
|
|
|
// The installed units' bytes, for the byte-for-byte checks across a moved-aside render.
|
|
const unitBytes=(e)=>Object.fromEntries([e.tmpl,e.con].map((u)=>[u,readFileSync(join(e.units,u),'utf8')]));
|
|
const renders=(e)=>lines(join(e.sd,'render.log')).map((l)=>JSON.parse(l));
|
|
const copyOf=(a)=>a[a.indexOf('--copy')+1];
|
|
// The next args: the args file's lines with the serve flags in another order, root's 0640.
|
|
const nextText=(e)=>e.argsText.replace(`--\n--port\n${e.port}\n--business\nacme\n`,`--\n--business\nacme\n--port\n${e.port}\n`);
|
|
const writeNext=(e,text=nextText(e),mode=0o640)=>{const f=join(e.etc,'dev.args.next');writeFileSync(f,text);chmodSync(f,mode);return f};
|
|
const replacedDirs=(e)=>readdirSync(e.etc).filter((n)=>/^dev\.replaced\.[A-Za-z0-9]{6}$/.test(n)).sort();
|
|
|
|
test('#1873: a target whose render check fails refuses render-refused, naming --next-args, before anything stops',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`check-fails-${C1}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.deepEqual(names(e),['start','fetch','ff','copy','hostcheck','units','plan','render-old-check','render-check','result']);
|
|
assert.match(result(e),new RegExp(`^result exit=3 render-refused case2-unit.sh --check refuses ${C1}'s units from .*dev\\.args; .*takes --next-args .*\\(nothing was stopped\\)$`));
|
|
assert.deepEqual(renders(e).map((a)=>[a[0],copyOf(a)]),[['--check',B],['--check',C1]]);
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('#1873: an installed copy whose render check fails refuses at render-old-check, before the target\'s check',async(t)=>{
|
|
const e=await useed(t);
|
|
fault(e,`check-fails-${B}`);
|
|
const r=await upgrade(e,C1);assert.equal(r.code,3,r.all);
|
|
assert.deepEqual(names(e),['start','fetch','ff','copy','hostcheck','units','plan','render-old-check','result']);
|
|
assert.match(result(e),new RegExp(`^result exit=3 render-refused case2-unit.sh --check refuses ${B}'s units from .*dev\\.args, so a rollback could not render them`));
|
|
assert.deepEqual(renders(e).map((a)=>[a[0],copyOf(a)]),[['--check',B]]);
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);argsKept(e);
|
|
});
|
|
|
|
test('#1873: --next-args moves the old units aside, renders the new flags as a first install, keeps the old units and args, and rolls back the same way',async(t)=>{
|
|
const e=await useed(t);
|
|
const before=unitBytes(e),next=nextText(e);writeNext(e);
|
|
const r=await upgrade(e,C1,'--next-args');assert.equal(r.code,0,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','units-aside','render','confirm','next-args','args','result']);
|
|
const [d]=replacedDirs(e);assert.ok(d,readdirSync(e.etc).join(' '));
|
|
assert.equal(result(e),`result exit=0 ok upgrade ok: ${C1} is live for acme; the replaced units and args file are in ${join(e.etc,d)}`);
|
|
assert.deepEqual(readdirSync(e.etc).sort(),['dev.args',d]);
|
|
assert.equal(readFileSync(e.args,'utf8'),next.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));assert.equal(statSync(e.args).mode&0o777,0o640);
|
|
const rd=join(e.etc,d);assert.equal(statSync(rd).mode&0o777,0o700);
|
|
assert.deepEqual(readdirSync(rd).sort(),['dev.args',e.tmpl,e.con].sort());
|
|
for(const [u,text] of Object.entries(before))assert.equal(readFileSync(join(rd,u),'utf8'),text,u);
|
|
assert.equal(readFileSync(join(rd,'dev.args'),'utf8'),e.argsText);assert.equal(statSync(join(rd,'dev.args')).mode&0o777,0o640);
|
|
const serve=(a)=>a.slice(a.indexOf('--')+1);
|
|
const rl=renders(e);assert.deepEqual(rl.map((a)=>[a[0],copyOf(a)]),[['--check',B],['--check-new',C1],['--user',C1]]);
|
|
assert.deepEqual(serve(rl[0]),['--port',String(e.port),'--business','acme']);
|
|
assert.deepEqual(serve(rl[1]),['--business','acme','--port',String(e.port)]);assert.deepEqual(serve(rl[2]),serve(rl[1]));
|
|
assert.deepEqual(live(e),['active','active',C1,C1]);
|
|
// The run sheet's rollback: the replaced args file as the next args, then rollback --next-args.
|
|
copyFileSync(join(rd,'dev.args'),join(e.etc,'dev.args.next'));chmodSync(join(e.etc,'dev.args.next'),0o640);
|
|
const after=unitBytes(e),m=slog(e).length,n=renders(e).length;
|
|
const rb=await drive(e,C1,'rollback','--instance','dev','--to',B,'--next-args',...TO);assert.equal(rb.code,0,rb.all);
|
|
assert.deepEqual(names(e).slice(m),['start','hostcheck','units','plan','render-old-check','render-check',...A2,'stage','units-aside','render','confirm','next-args','args','result']);
|
|
assert.deepEqual(renders(e).slice(n).map((a)=>[a[0],copyOf(a)]),[['--check',C1],['--check-new',B],['--user',B]]);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText);assert.deepEqual(live(e),['active','active',B,B]);
|
|
const d2=replacedDirs(e).find((x)=>x!==d);assert.ok(d2);assert.deepEqual(readdirSync(e.etc).sort(),['dev.args',d,d2].sort());
|
|
for(const [u,text] of Object.entries(after))assert.equal(readFileSync(join(e.etc,d2,u),'utf8'),text,u);
|
|
assert.equal(readFileSync(join(e.etc,d2,'dev.args'),'utf8'),next.replace(`--copy\n${B}\n`,`--copy\n${C1}\n`));
|
|
});
|
|
|
|
test('#1873: a failed render under --next-args moves the old units back unchanged and restarts them, with no render of the old copy: exit 1',async(t)=>{
|
|
const e=await useed(t);
|
|
const before=unitBytes(e),next=nextText(e);writeNext(e);
|
|
fault(e,`render-fails-${C1}`);
|
|
const r=await upgrade(e,C1,'--next-args');assert.equal(r.code,1,r.all);
|
|
assert.deepEqual(names(e),[...PRE,...A2,'stage','units-aside','render',...A2,'revert','units-back','daemon-reload','start-bus','start-console','confirm-rollback','result']);
|
|
assert.match(result(e),new RegExp(`^result exit=1 failed A4 failed; rolled back to ${B}`));
|
|
assert.deepEqual(renders(e).map((a)=>[a[0],copyOf(a)]),[['--check',B],['--check-new',C1],['--user',C1]]);
|
|
for(const [u,text] of Object.entries(before))assert.equal(readFileSync(join(e.units,u),'utf8'),text,u);
|
|
assert.deepEqual(readdirSync(e.etc).sort(),['dev.args','dev.args.next']);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText);assert.equal(readFileSync(join(e.etc,'dev.args.next'),'utf8'),next);
|
|
assert.equal(final(e,lastAct(e)).rollback,'ok');assert.deepEqual(live(e),['active','active',B,B]);
|
|
});
|
|
|
|
test('#1873: the next args must be root\'s, equal before -- but for --copy, differ after it, and render as a first install',async(t)=>{
|
|
const e=await useed(t);
|
|
const tryNext=async(text,mode,code,why,at='plan')=>{
|
|
if(text===null)rmSync(join(e.etc,'dev.args.next'),{force:true});else writeNext(e,text,mode);
|
|
const n=existsSync(e.slog)?slog(e).length:0;
|
|
const r=await upgrade(e,C1,'--next-args');assert.equal(r.code,3,`${why}: ${r.all}`);
|
|
assert.match(slog(e).at(-1),new RegExp(`^result exit=3 ${code} `),why);
|
|
const ran=names(e).slice(n);assert.ok(!ran.some((x)=>/^(stop|stage|units-aside|render$|confirm)/.test(x)),`${why}: ${ran}`);
|
|
if(at)assert.equal(ran.at(-2),at,`${why}: ${ran}`);
|
|
};
|
|
await tryNext(null,0,'args-unreadable','no next file','start');
|
|
await tryNext(nextText(e),0o660,'args-unreadable','group write','start');
|
|
await tryNext(nextText(e).replace('--home\n','--pm\n--home\n'),0o640,'next-args-differ','a flag before --','start');
|
|
await tryNext(nextText(e).replace('--user\nfixture\n','--user\nother\n'),0o640,'next-args-differ','another user','start');
|
|
await tryNext(e.argsText,0o640,'next-args-same','the same lines','start');
|
|
await tryNext(e.argsText.replace(`--copy\n${B}\n`,`--copy\n${C}\n`),0o640,'next-args-same','only --copy differs','start');
|
|
fault(e,`check-fails-${C1}`);
|
|
await tryNext(nextText(e),0o640,'render-refused','the first-install check','render-check');
|
|
assert.match(result(e),new RegExp(`^result exit=3 render-refused case2-unit.sh --check-new refuses ${C1}'s units from .*dev\\.args\\.next`));
|
|
assert.ok(!existsSync(join(e.sd,'root-systemctl.log')));assert.deepEqual(live(e),['active','active',B,B]);
|
|
assert.equal(readFileSync(e.args,'utf8'),e.argsText);assert.deepEqual(replacedDirs(e),[]);
|
|
});
|