diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 1f9a3eb28..e0447341c 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -420,22 +420,29 @@ refuses `capture-failed` before it stops anything. Adopt uses the same rule. saves the unit files, drop-ins and wrapper under `state/stack-rollback//`, stops the Console then the bus, and reads the stopped store. A `-wal` or `-shm` beside it refuses - `store-unclean` and leaves the store alone. A migration takes an - owner-only backup under `/backups/stack//` first. + `store-unclean` and leaves the store alone. Every activate then takes an + owner-only backup of the stopped store under + `/backups/stack//`, before any migration (row 185), + and the log line names it. A missing store file takes no backup. A store + outside the target's chain (`store-newer`) or a migration without + `--migrate` rolls back before the backup. It installs the bus, the Console and the wrapper from the pointer's flags, starts the bus then the Console, and runs the gate: both units active, a live bus host for this start, Console `/healthz`, `stack-copy.sh check` and, with `--pm`, exactly one `launched` journal line for a live run of this start. A pass writes the pointer. -6. A gate failure puts back the saved units and wrapper, restores the backup - when a migration ran, keeps the store it replaced as - `bus.post-attempt-`, where the stamp is the activation's second and - its suffix if it has one, starts the old copy and gates it. If that gate +6. A gate failure puts back the saved units and wrapper. The backup is + taken on every activate, but it is restored only when a migration ran. + A restore keeps the store it replaced as `bus.post-attempt-`, where + the stamp is the activation's second and its suffix if it has one. After + a same-schema activate the store stays as the failed run left it. Then it + starts the old copy and gates it. If that gate fails too, both units stay stopped, the log says `rollback-failed`, and Jason or Mos settles it from the rollback set the message names. 7. `rollback BUSINESS` returns to the copy before the pointer's, taken from the log. When the schemas differ it needs `--restore ACTIVATION`, with or - without a `-1` to `-9` suffix. Rows written after that backup need + without a `-1` to `-9` suffix. A same-schema rollback can take + `--restore` too, with the backup the activate took. Rows written after that backup need `--events N`, and Jason decides first. The count covers events and every table in `COUNTED` (row 181), the tables a bus writer fills without an event. The `post-backup-events` refusal prints one count per table with diff --git a/packages/cli/tests/stack-release.test.mjs b/packages/cli/tests/stack-release.test.mjs index 349266a56..e85968006 100644 --- a/packages/cli/tests/stack-release.test.mjs +++ b/packages/cli/tests/stack-release.test.mjs @@ -110,6 +110,12 @@ function systemctl(){ const s=load(),id=path.basename(f.wd); if(!isBus(u)){s[u]={state:'active',pid:null,wd:f.wd};save(s);process.exit(0)} if(fault(`start-dies-${id}`)){s[u]={state:'failed',pid:null,wd:f.wd};save(s);process.exit(0)} + // Row 185 (T48): the copy's start writes one event before the holder opens the store, so the + // store differs from a backup the run took before the start. The close leaves no -wal. + if(fault(`start-writes-row-${id}`)){ + const db=new (process.getBuiltinModule('node:sqlite').DatabaseSync)(path.join(data,'bus','bus.sqlite')); + db.prepare('INSERT INTO events (at) VALUES (?)').run('2999-01-01T00:00:00.000Z');db.close(); + } snap('start'); const off=process.env.SD_CLOCK_OFFSET||'0'; // The holder must not inherit stack-release.sh's lock (fd 9) or any other fd above 2. @@ -373,6 +379,7 @@ async function seed(e,{copy=B,pm=false,store='3g',pointer:how='adopt'}={}){ } const rolledBack=(code,to=B)=>`stack-release: ${code}; rolled back to ${to}, which passed the gate\n`; +const planBackup=(e)=>`backup: the run backs up the stopped store to ${e.data}/backups/stack// before any migration`; const planTail=(e,to,pm=false)=>[`bus: scripts/bus-service.sh install --copy ${to} --copies ${COPIES}${pm?' --pm':''} --claude ${e.claude}`,`Console: scripts/webui-service.sh install --copy ${to} --copies ${COPIES} --port ${e.port} --business acme`,`CLI: scripts/stack-copy.sh wrapper ${to} --copies ${COPIES} --bin ${e.bin}`]; // --- The T table --- @@ -453,7 +460,8 @@ test('T5 an activate with no migration stops the Console then the bus, installs, const c=calls(e).slice(n),reload=c.indexOf('daemon-reload'); assert.ok(reload>c.indexOf(`stop ${BUS}`)&&reload[x.op,x.from,x.to,x.result]),[['activate',B,C1,'started'],['activate',B,C1,'ok']]); }); @@ -538,8 +546,10 @@ test('T11 a rollback whose gate fails too: rollback-failed, exit 2, both units s const e=await mk(t);await seed(e);fault(e,`healthz-${C1}`);fault(e,`healthz-${B}`);const n=calls(e).length; const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,2,r.out+r.err); const l=logLines(e).at(-1),rs=join(e.state,'stack-rollback',l.activation); - assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with healthz; both units are stopped. Rollback set: ${rs}\n`); + assert.equal(r.err,`stack-release: rollback-failed: the run failed with healthz and the rollback with healthz; both units are stopped. Rollback set: ${rs}, backup ${join(e.data,'backups','stack',l.activation)}\n`); assert.deepEqual([l.result,l.code,l.rollback,l.rollbackCode],['rollback-failed','healthz','failed','healthz']); + // Row 185: the same-schema backup is named and was not restored, so no post-attempt store. + assert.equal(l.backup,l.activation);assert.equal(posts(e).length,0); assert.deepEqual([units(e)[BUS].state,units(e)[CON].state],['inactive','inactive']); const ss=startsStops(e,n); assert.deepEqual(ss.filter((c)=>c.startsWith('start ')),[`start ${BUS}`,`start ${CON}`,`start ${BUS}`,`start ${CON}`]); @@ -635,10 +645,10 @@ test('T26 --plan lists the commits an activate adds, after the migration lines', assert.equal(adds.length,2);assert.match(adds[0],/^ {2}[0-9a-f]{7,} row 904: C$/);assert.match(adds[1],/^ {2}[0-9a-f]{7,} row 903: C1$/); for(const [flags,mig] of [[[],'not given; the run would roll back with migration-needs-flag'],[['--migrate'],'given']]){ const r=await run(e,'activate',C,'acme','--plan',...flags);assert.equal(r.code,0,r.err); - assert.equal(r.out,[...head(mig),'commits: the run adds 2 (newest first, at most 200):',...adds,...planTail(e,C),''].join('\n')); + assert.equal(r.out,[...head(mig),planBackup(e),'commits: the run adds 2 (newest first, at most 200):',...adds,...planTail(e,C),''].join('\n')); } const same=await run(e,'activate',C1,'acme','--plan');assert.equal(same.code,0,same.err); - assert.ok(same.out.includes('target schema: 3g\nmigration: none; the store is at the target\'s version\ncommits: the run adds 1 (newest first, at most 200):\n'),same.out); + assert.ok(same.out.includes(`target schema: 3g\nmigration: none; the store is at the target's version\n${planBackup(e)}\ncommits: the run adds 1 (newest first, at most 200):\n`),same.out); }); test('T27 --plan names each migration step, or says the steps cannot be read',async(t)=>{ @@ -802,7 +812,7 @@ test('T29 a bus that stops during the held-store read: --plan says so, rollback const before=fp(e),logs=logLines(e).length; const r=await run(e,'activate',C,'acme','--plan');assert.equal(r.code,0,r.err);assert.equal(r.err,''); assert.equal(r.out,['plan: activate for acme',`from: ${B}`,`to: ${C}`, - 'store schema: unknown (the bus stopped during the read, so the store state changed; run --plan again)','target schema: 3h', + 'store schema: unknown (the bus stopped during the read, so the store state changed; run --plan again)','target schema: 3h',planBackup(e), 'commits: the run adds 2 (newest first, at most 200):',...commitLines(`${B}..${C}`),...planTail(e,C),''].join('\n')); assert.deepEqual(fp(e),before);assert.equal(logLines(e).length,logs); }); @@ -1287,6 +1297,82 @@ test('T46 row 181 item 4: every table in the bus schema is in exactly one of fou db.close(); }); +// Row 185 (#1683): every activate with a store backs it up under its activation id, before any +// migration. Only a backup taken before a migration (TAKEN) is restored after a gate failure. +test('T47 row 185 test 1: a same-schema activate leaves a whole backup and the log line names it',async(t)=>{ + const e=await mk(t);await seed(e);const before=fp(e); + const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); + const act=pointer(e).activation,o=join(e.data,'backups','stack',act); + assert.ok(r.out.includes(`backup: ${o}\n`),r.out); + for(const f of ['SHA256SUMS','DATA_ROOT','SCHEMA','STAMP'])assert.ok(existsSync(join(o,f)),f); + assert.equal(readFileSync(join(o,'DATA_ROOT'),'utf8'),`${e.data}\n`); + assert.equal(readFileSync(join(o,'SCHEMA'),'utf8'),`${storeVersion(e.store,true)}\n`);assert.equal(storeVersion(e.store,true),'3g'); + assert.match(readFileSync(join(o,'STAMP'),'utf8'),/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\n$/); + assert.equal(backupSum(e,act),before.store); + const l=logLines(e).at(-1);assert.deepEqual([l.op,l.result,l.activation,l.backup],['activate','ok',act,act]); +}); + +test('T48 row 185 test 2: a gate failure on a same-schema activate rolls back without a restore',async(t)=>{ + const e=await mk(t);await seed(e);const before=fp(e); + fault(e,`start-writes-row-${C1}`);fault(e,`healthz-${C1}`); + const r=await run(e,'activate',C1,'acme','--timeout','3');assert.equal(r.code,1,r.out+r.err); + assert.equal(r.err,rolledBack('healthz')); + const l=logLines(e).at(-1),act=l.activation; + assert.deepEqual([l.result,l.code,l.rollback,l.backup],['gate-failed','healthz','ok',act]); + // The store as the target's bus left it when the rollback stopped it, after the failed gate. + const failed=snaps(e).filter((x)=>x.ev==='stop').at(-1).files['bus.sqlite'][2]; + assert.notEqual(failed,backupSum(e,act),'the target start wrote a row after the backup'); + assert.equal(sha(e.store),failed);assert.equal(posts(e).length,0); + assert.deepEqual({...fp(e),store:before.store},before);assert.equal(units(e)[BUS].wd,join(COPIES,B)); +}); + +test('T49 row 185 test 3: a same-schema rollback --restore restores, or refuses rows written after STAMP',async(t)=>{ + await t.test('no rows after STAMP: restores and goes live',async(t)=>{ + const e=await mk(t);await seed(e); + assert.equal((await run(e,'activate',C1,'acme')).code,0);const act=pointer(e).activation,storeC1=sha(e.store); + const r=await run(e,'rollback','acme','--restore',act);assert.equal(r.code,0,r.out+r.err); + assert.ok(r.out.startsWith(`restore: ${join(e.data,'backups','stack',act)}; the store now in place moves to `),r.out); + assert.ok(r.out.includes('; 0 event(s) were written after the backup\n'),r.out); + assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(sha(e.store),backupSum(e,act)); + assert.equal(posts(e).length,1);assert.equal(sha(join(e.data,posts(e)[0],'bus.sqlite')),storeC1); + assert.equal(logLines(e).at(-1).backup,act); + }); + await t.test('one instance_names row after STAMP refuses post-backup-events; --events 1 restores',async(t)=>{ + const e=await mk(t);await seed(e); + assert.equal((await run(e,'activate',C1,'acme')).code,0);const act=pointer(e).activation; + addRow(e.store,'instance_names'); + const before=fp(e),n=calls(e).length,no=await run(e,'rollback','acme','--restore',act); + assert.equal(no.code,3,no.out+no.err); + assert.equal(no.err,'stack-release: refusing post-backup-events: 0 event(s), 1 instance_names row(s) were written after the backup; Jason decides, then --events 1\n'); + assert.deepEqual(fp(e),before);assert.deepEqual(startsStops(e,n),[]); + const r=await run(e,'rollback','acme','--restore',act,'--events','1');assert.equal(r.code,0,r.out+r.err); + assert.deepEqual([pointer(e).copy,pointer(e).schema],[B,'3g']);assert.equal(sha(e.store),backupSum(e,act)); + }); +}); + +test('T50 row 185 test 4: activate --plan prints the backup line and writes nothing',async(t)=>{ + const e=await mk(t);await seed(e); + const before=fp(e),data=tree(e.data),n=calls(e).length,logs=logLines(e).length; + const r=await run(e,'activate',C1,'acme','--plan');assert.equal(r.code,0,r.err);assert.equal(r.err,''); + assert.deepEqual(r.out.split('\n').filter((x)=>x.startsWith('backup: ')),[planBackup(e)]); + assert.deepEqual(fp(e),before);assert.deepEqual(tree(e.data),data);assert.deepEqual(startsStops(e,n),[]);assert.equal(logLines(e).length,logs); + assert.ok(!existsSync(join(e.data,'backups'))); + // Rollback is unchanged: it takes no backup, so its plan has no backup line. + assert.equal((await run(e,'activate',C1,'acme')).code,0); + const rb=await run(e,'rollback','acme','--plan');assert.equal(rb.code,0,rb.err); + assert.deepEqual(rb.out.split('\n').filter((x)=>x.startsWith('backup')),[]); +}); + +test('T51 row 185 test 5: an activate with no store file takes no backup',async(t)=>{ + const e=await mk(t);await seed(e); + sysctl(e,'stop',BUS);for(const f of readdirSync(join(e.data,'bus')))rmSync(join(e.data,'bus',f)); + const p=await run(e,'activate',C1,'acme','--plan');assert.equal(p.code,0,p.err); + assert.deepEqual(p.out.split('\n').filter((x)=>x.startsWith('backup')),['backup: none; there is no store file']); + const r=await run(e,'activate',C1,'acme');assert.equal(r.code,0,r.out+r.err); + assert.ok(!existsSync(join(e.data,'backups')));assert.ok(!r.out.includes('backup: '),r.out); + const l=logLines(e).at(-1);assert.deepEqual([l.result,l.backup],['ok',null]); +}); + // Row 178: proof metadata uses temp fixture files, never the installed slot. async function seedProof(e) { await seed(e); diff --git a/scripts/stack-release.sh b/scripts/stack-release.sh index a51b2bb68..f8051d81c 100755 --- a/scripts/stack-release.sh +++ b/scripts/stack-release.sh @@ -345,7 +345,8 @@ sums() { (cd "$1" && find . -type f ! -name SHA256SUMS ! -name steps.log -print0 sum_of() { awk -v f="./$2" '$2 == f { print $1 }' "$1/SHA256SUMS"; } # The backup of the stopped store (backup.sh's steps). Only bus.sqlite: a -wal or -shm was -# already refused as store-unclean. Sets TAKEN once the backup is whole. +# already refused as store-unclean. Every activate with a store takes one (row 185). Sets BACKUP +# once the backup is whole, for the log line; TAKEN is set by the caller only before a migration. backup_store() { local o="$DATA/backups/stack/$ACT" ( @@ -364,7 +365,7 @@ backup_store() { echo "$RS" > "$o/ROLLBACK_SET" ) || return 1 stopped || return 1 - TAKEN="$ACT" BACKUP="$ACT" + BACKUP="$ACT" say "backup: $o" } @@ -533,7 +534,8 @@ put_back() { } # A failure after step 2: put the pointer's stack back and gate it. $1 is the run's code. Only a -# backup this run took (TAKEN) is restored; a rollback's --restore backup never is. +# backup this run took before a migration (TAKEN) is restored; a same-schema activate's backup +# and a rollback's --restore backup never are. rollback_in_run() { trap - INT TERM HUP local code="$1" rcode="" @@ -809,6 +811,13 @@ plan() { else echo "restore: the store must go back to $T_SCHEMA; needs --restore $(from_log backup || echo ACTIVATION)" fi + if [ "$OP" != activate ]; then + : # rollback takes no backup of its own + elif [ -e "$STORE" ]; then + echo "backup: the run backs up the stopped store to $DATA/backups/stack// before any migration" + else + echo "backup: none; there is no store file" + fi plan_commits [ "$P_PENDING" = 0 ] || echo "pm: the pointer is pending (activate --pm-after-harness); the bus runs without its PM" [ "$RUN_PENDING" = 0 ] || echo "pm: the bus starts without --pm and the PM stays pending; next: Jason's plugins set of harness, then pm-on" @@ -836,7 +845,10 @@ run_steps() { if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then in_chain "$STORE_SCHEMA" || rollback_in_run store-newer [ "$MIGRATE" = 1 ] || rollback_in_run migration-needs-flag - backup_store || rollback_in_run backup + fi + backup_store || rollback_in_run backup + if [ "$STORE_SCHEMA" != "$T_SCHEMA" ]; then + TAKEN="$ACT" node "$COPIES/$TO/packages/bus/src/migrate-schema.mjs" "$DATA" >> "$STEPS" 2>&1 || rollback_in_run migrate-failed say "migrated the store to $T_SCHEMA" fi