125 lines
12 KiB
JavaScript
125 lines
12 KiB
JavaScript
import * as fsExtra from 'node:fs';
|
|
import * as childModule from 'node:child_process';
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
const api=await import('../../../scripts/testing/lane-state.mjs');
|
|
function fixture(t){const root=mkdtempSync('/mnt/storage/scratch/gs-');t.after(()=>rmSync(root,{recursive:true,force:true}));return root;}
|
|
test('unfinished dead-owner runs quarantine their lane and a recovery record clears only that run',t=>{
|
|
assert.equal(typeof api.appendRecord,'function','appendRecord is required');
|
|
const root=fixture(t),dir=join(root,'runs','one');
|
|
api.appendRecord(dir,{kind:'start',lane:'a',owner:{pid:123456789,start:'1',boot:'fixture'}});
|
|
assert.equal(api.status(root,{processes:[]}).a.state,'QUARANTINED');
|
|
api.appendRecord(dir,{kind:'recovered',clean:true});
|
|
assert.equal(api.status(root,{processes:[]}).a.state,'CLEAR');
|
|
});
|
|
test('a legacy landing gate beside a lane stays legacy even with the same parent',t=>{
|
|
assert.equal(typeof api.appendRecord,'function');
|
|
const root=fixture(t);const owner={pid:123,start:'5',boot:'b',state:'S',ppid:1,argv:['node','gate-lane.mjs']};
|
|
api.appendRecord(join(root,'runs','one'),{kind:'start',lane:'a',owner});
|
|
const procs=[owner,{pid:124,start:'6',boot:'b',state:'S',ppid:1,argv:['node','/fixture/landing.mjs','gate','/fixture/wt','/fixture/logs']}];
|
|
const result=api.status(root,{processes:procs});assert.equal(result.a.state,'BUSY');assert.equal(result.b.state,'CLEAR');assert.deepEqual(result.legacy,[{pid:124,start:'6'}]);
|
|
});
|
|
test('the shell status entry reads scratch receipts and reports legacy processes without mutating old slots',t=>{
|
|
const root=fixture(t);const {spawnSync}=requireChild();
|
|
const r=spawnSync('bash',[new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname,'status','--state-root',root],{encoding:'utf8'});
|
|
assert.equal(r.status,0,r.stderr);const state=JSON.parse(r.stdout);assert.equal(state.a.state,'CLEAR');assert.equal(state.b.state,'CLEAR');assert.ok(Array.isArray(state.legacy));
|
|
});
|
|
function requireChild(){return childModule;}
|
|
test('recovery supersedes an unclean terminal without rewriting either receipt',t=>{
|
|
const root=fixture(t),dir=join(root,'runs','one');api.appendRecord(dir,{kind:'start',lane:'a',owner:{pid:99999999,start:'1',boot:'fixture'}});api.appendRecord(dir,{kind:'terminal',clean:false});api.appendRecord(dir,{kind:'recovered',clean:true});assert.equal(api.status(root,{processes:[]}).a.state,'CLEAR');assert.equal(api.records(dir).length,3);
|
|
});
|
|
test('the wrapper refuses a held lane lock and preserves its inode',async t=>{
|
|
const root=fixture(t),file=join(root,'a.lock');const {spawn,spawnSync}=childModule;
|
|
const holder=spawn('flock',['-x',file,'bash','-c','echo ready; read -r done'],{stdio:['pipe','pipe','pipe']});const exited=new Promise(r=>holder.once('exit',r));await new Promise(r=>holder.stdout.once('data',r));
|
|
const {statSync}=await import('node:fs');const ino=statSync(file).ino;
|
|
t.after(async()=>{holder.stdin.end('\n');await exited;});
|
|
const r=spawnSync('bash',[new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname,'run','--state-root',root,'--lane','a','--grant','fixture'],{encoding:'utf8'});
|
|
assert.equal(r.status,2);assert.match(r.stderr,/lock busy/);assert.equal(statSync(file).ino,ino);
|
|
});
|
|
test('a held kernel lock without a start receipt is BUSY rather than CLEAR',async t=>{
|
|
const root=fixture(t),file=join(root,'a.lock');const holder=childModule.spawn('flock',['-x',file,'bash','-c','echo ready; read -r done'],{stdio:['pipe','pipe','pipe']});const exited=new Promise(r=>holder.once('exit',r));await new Promise(r=>holder.stdout.once('data',r));t.after(async()=>{holder.stdin.end('\n');await exited;});
|
|
assert.equal(api.status(root,{processes:[]}).a.state,'BUSY');
|
|
});
|
|
test('SIGKILL leaves a quarantined run and locked recovery publishes a new clean receipt',async t=>{
|
|
const root=fixture(t),temp=mkdtempSync('/mnt/storage/scratch/gt-');t.after(()=>rmSync(temp,{recursive:true,force:true}));
|
|
const child=childModule.spawn(process.execPath,[new URL('./fixtures/gate-lane-child.mjs',import.meta.url).pathname,'wait',join(temp,'started')],{stdio:['ignore','pipe','pipe']});
|
|
const exited=new Promise(r=>child.once('exit',r));await new Promise(r=>child.stdout.once('data',r));
|
|
const id='abcdef123456',dir=join(root,'runs',id),docker=join(root,'docker-fixture');writeFileSync(docker,'#!/bin/sh\nexit 0\n',{mode:0o700});
|
|
api.appendRecord(dir,{kind:'start',lane:'a',owner:api.identity(child.pid),tempRoot:temp,project:`mg197-a-${id}`,docker});
|
|
child.kill('SIGKILL');await exited;assert.equal(api.status(root,{processes:[]}).a.state,'QUARANTINED');
|
|
const r=childModule.spawnSync('bash',[new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname,'recover','--state-root',root,'--run-id',id,'--grant','fixture-recover'],{encoding:'utf8'});
|
|
assert.equal(r.status,0,r.stderr);assert.equal(api.status(root,{processes:[]}).a.state,'CLEAR');assert.equal(api.records(dir).filter(e=>e.kind==='start').length,1);assert.equal(api.records(dir).filter(e=>e.kind==='recovered').length,1);
|
|
});
|
|
test('a malformed receipt refuses status and an identity mismatch cannot look live',t=>{
|
|
const root=fixture(t),dir=join(root,'runs','one');api.appendRecord(dir,{kind:'start',lane:'a',owner:{pid:123,start:'old',boot:'b'}});
|
|
assert.equal(api.status(root,{processes:[{pid:123,start:'new',boot:'b',state:'S',ppid:1,argv:['node','owned'] }]}).a.state,'QUARANTINED');
|
|
writeFileSync(join(dir,'bad.json'),'{');assert.throws(()=>api.status(root,{processes:[]}),/invalid run receipt/);
|
|
});
|
|
test('a symlink lock is refused before acquiring or replacing its target',t=>{
|
|
const root=fixture(t),target=join(root,'foreign');writeFileSync(target,'untouched');const {symlinkSync}=fsExtra;symlinkSync(target,join(root,'a.lock'));
|
|
const r=childModule.spawnSync('bash',[new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname,'run','--state-root',root,'--lane','a','--grant','fixture'],{encoding:'utf8'});
|
|
assert.equal(r.status,2);assert.match(r.stderr,/ordinary file/);assert.equal(readFileSync(target,'utf8'),'untouched');
|
|
});
|
|
test('failed recovery keeps quarantine and never publishes a clean recovery receipt',async t=>{
|
|
const root=fixture(t),temp=mkdtempSync('/mnt/storage/scratch/gt-');t.after(()=>rmSync(temp,{recursive:true,force:true}));
|
|
const id='abcdef123456',dir=join(root,'runs',id),docker=join(root,'docker-fixture');writeFileSync(docker,'#!/bin/sh\nexit 1\n',{mode:0o700});
|
|
api.appendRecord(dir,{kind:'start',lane:'a',owner:{pid:99999999,start:'1',boot:'fixture'},tempRoot:temp,project:`mg197-a-${id}`,docker});
|
|
const r=childModule.spawnSync('bash',[new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname,'recover','--state-root',root,'--run-id',id,'--grant','fixture-recover'],{encoding:'utf8'});
|
|
assert.equal(r.status,2);assert.equal(api.status(root,{processes:[]}).a.state,'QUARANTINED');assert.equal(api.records(dir).some(e=>e.kind==='recovered'),false);
|
|
});
|
|
test('two shared image holders coexist, but exclusive preparation refuses their lock',async t=>{
|
|
const root=fixture(t),file=join(root,'image.lock');const holder=childModule.spawn('flock',['-s',file,'bash','-c','echo ready; read -r done'],{stdio:['pipe','pipe','pipe']});const exited=new Promise(r=>holder.once('exit',r));await new Promise(r=>holder.stdout.once('data',r));t.after(async()=>{holder.stdin.end('\n');await exited;});
|
|
const entry=new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname;
|
|
const shared=childModule.spawnSync('bash',[entry,'run','--state-root',root,'--lane','a','--grant','fixture'],{encoding:'utf8'});
|
|
assert.equal(shared.status,2);assert.doesNotMatch(shared.stderr,/lock busy|descriptor|not locked/);
|
|
const exclusive=childModule.spawnSync('bash',[entry,'prepare-image','--state-root',root,'--grant','fixture'],{encoding:'utf8'});
|
|
assert.equal(exclusive.status,2);assert.match(exclusive.stderr,/lock busy/);
|
|
});
|
|
const entry=new URL('../../../scripts/testing/gate-lane.sh',import.meta.url).pathname;
|
|
const childFixture=new URL('./fixtures/gate-lane-child.mjs',import.meta.url).pathname;
|
|
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
|
|
async function until(check){for(let n=0;n<100;n++){if(check())return;await sleep(20);}throw Error('fixture barrier did not become ready');}
|
|
test('two simultaneous shell claims have one winner; supervisor retains lock FDs and ordinary children do not',async t=>{
|
|
const root=fixture(t),bin=join(root,'bin');fsExtra.mkdirSync(bin);const shim=join(bin,'node');
|
|
writeFileSync(shim,`#!/bin/sh\nif [ "$2" = --locked ]; then exec '${process.execPath}' '${childFixture}' lock-probe '${root}'; fi\nexec '${process.execPath}' "$@"\n`,{mode:0o700});
|
|
const env={...process.env,PATH:bin+':'+process.env.PATH};
|
|
const start=()=>{const p=childModule.spawn('bash',[entry,'run','--state-root',root,'--lane','a','--grant','fixture'],{env,stdio:['ignore','pipe','pipe']});p.done=new Promise(r=>p.once('exit',code=>r(code)));return p;};
|
|
const first=start(),second=start();t.after(()=>{first.kill('SIGKILL');second.kill('SIGKILL');});
|
|
await until(()=>fsExtra.existsSync(join(root,'ready')));
|
|
const all=api.runRecords(root);assert.equal(all.length,1);assert.equal(readFileSync(join(root,'out/fds.log'),'utf8'),'');
|
|
const winner=all[0].records.find(r=>r.kind==='start').owner.pid;assert.ok([first.pid,second.pid].includes(winner));
|
|
const loser=first.pid===winner?second:first;assert.equal(await loser.done,2);
|
|
const ino=fsExtra.statSync(join(root,'a.lock')).ino;
|
|
assert.equal(childModule.spawnSync('flock',['-n',join(root,'a.lock'),'true']).status,1);
|
|
const won=first.pid===winner?first:second;won.kill('SIGTERM');await won.done;
|
|
assert.equal(fsExtra.statSync(join(root,'a.lock')).ino,ino);
|
|
});
|
|
test('same checkout and a legacy land gate each refuse a claim on an otherwise free lane',async t=>{
|
|
const {activeConflicts}=await import('../../../scripts/testing/gate-lane.mjs');
|
|
const root=fixture(t),owner={pid:321,start:'1',boot:'test',state:'S',ppid:1,argv:['node','owned']};
|
|
api.appendRecord(join(root,'runs','one'),{kind:'start',lane:'a',checkout:'/fixture/candidate',owner});
|
|
const o={stateRoot:root,lane:'b',checkout:'/fixture/candidate'};
|
|
assert.throws(()=>activeConflicts(o,{processes:[owner]}),/checkout already owned/);
|
|
assert.throws(()=>activeConflicts({...o,checkout:'/fixture/other'},{processes:[owner,{pid:322,start:'2',boot:'test',state:'S',ppid:1,argv:['node','/fixture/landing.mjs','gate']}]}),/LEGACY BUSY/);
|
|
});
|
|
test('recovery kills a proven surviving child, preserves a foreign child and keeps quarantine if interrupted',async t=>{
|
|
const root=fixture(t),temp=mkdtempSync('/mnt/storage/scratch/gt-');t.after(()=>rmSync(temp,{recursive:true,force:true}));
|
|
const owned=childModule.spawn(process.execPath,[childFixture,'wait',join(temp,'child')],{cwd:temp,stdio:'ignore'});
|
|
const foreign=childModule.spawn(process.execPath,[childFixture,'wait',join(root,'foreign')],{stdio:'ignore'});
|
|
const ownedDone=new Promise(r=>owned.once('exit',r)),foreignDone=new Promise(r=>foreign.once('exit',r));t.after(async()=>{owned.kill('SIGKILL');foreign.kill('SIGKILL');await Promise.all([ownedDone,foreignDone]);});
|
|
await until(()=>fsExtra.existsSync(join(temp,'child'))&&fsExtra.existsSync(join(root,'foreign')));
|
|
const docker=join(root,'docker'),pidFile=join(root,'docker-pid'),id='abcdef123456',dir=join(root,'runs',id);
|
|
writeFileSync(docker,`#!/bin/sh\necho $$ > '${pidFile}'\nexec sleep 60\n`,{mode:0o700});
|
|
api.appendRecord(dir,{kind:'start',id,stateRoot:root,lane:'a',owner:{pid:99999999,start:'1',boot:'test'},tempRoot:temp,project:`mg197-a-${id}`,docker});
|
|
const recovery=childModule.spawn('bash',[entry,'recover','--state-root',root,'--run-id',id,'--grant','fixture'],{stdio:'ignore'});const done=new Promise(r=>recovery.once('exit',r));
|
|
await until(()=>fsExtra.existsSync(pidFile));const dockerOwner=api.identity(Number(readFileSync(pidFile,'utf8')));recovery.kill('SIGKILL');await done;
|
|
if(api.sameProcess(dockerOwner))process.kill(dockerOwner.pid,'SIGKILL');
|
|
await ownedDone;assert.equal(api.sameProcess(api.identity(foreign.pid)),true);
|
|
assert.equal(api.records(dir).some(r=>r.kind==='recovered'),false);assert.equal(api.status(root,{processes:[]}).a.state,'QUARANTINED');
|
|
writeFileSync(docker,'#!/bin/sh\nexit 0\n',{mode:0o700});
|
|
const retried=childModule.spawnSync('bash',[entry,'recover','--state-root',root,'--run-id',id,'--grant','fixture-resume'],{encoding:'utf8'});
|
|
assert.equal(retried.status,0,retried.stderr);assert.equal(api.status(root,{processes:[]}).a.state,'CLEAR');assert.equal(api.sameProcess(api.identity(foreign.pid)),true);
|
|
});
|