104 lines
6.0 KiB
Python
104 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Row219 serial gate. Requires Sage CLEAR; all children use a process group."""
|
|
import os,sys,time,json,signal,subprocess,tempfile,hashlib,re,glob
|
|
from pathlib import Path
|
|
from datetime import datetime,timezone
|
|
ROOT=Path(__file__).resolve().parents[4]
|
|
PACKET=Path(__file__).resolve().parent
|
|
OUT=PACKET/'run-2efd-mos1439';OUT.mkdir(exist_ok=True)
|
|
def stamp():return datetime.now(timezone.utc).isoformat()
|
|
def verify():
|
|
for line in (PACKET/'candidate-manifest.sha256').read_text().splitlines():
|
|
sha,path=line.split(' ',1)
|
|
if hashlib.sha256((ROOT/path).read_bytes()).hexdigest()!=sha:raise RuntimeError('candidate drift: '+path)
|
|
def sweep():
|
|
found=[]
|
|
for proc in Path('/proc').glob('[0-9]*'):
|
|
try:
|
|
cwd=(proc/'cwd').resolve();name=(proc/'comm').read_text().strip()
|
|
if (str(cwd).startswith(str(ROOT)) or str(cwd).startswith('/mnt/storage/scratch/tmp/')) and ('node' in name or 'chrom' in name):found.append(dict(pid=int(proc.name),name=name,cwd=str(cwd)))
|
|
except (FileNotFoundError,PermissionError,ProcessLookupError):pass
|
|
return found
|
|
def execute(label,args,cwd=ROOT,deadline=300,docker=False):
|
|
if args[0]=='node' and '--test' in args and '--test-reporter=tap' not in args:args=[args[0],'--test-reporter=tap',*args[1:]]
|
|
if args[0]!='timeout':args=['timeout','-k','10',str(deadline),*args]
|
|
verify();short=subprocess.check_output(['mktemp','-d','/mnt/storage/scratch/tmp/rXXXXXX'],text=True).strip()
|
|
if not re.fullmatch(r'[A-Za-z0-9/.-]+',short):raise RuntimeError('unsafe TMPDIR charset')
|
|
env=os.environ.copy();env.update(TMPDIR=short,NO_COLOR='1',COMPOSE_PROJECT_NAME='rocko-r219-gate')
|
|
env.pop('MOSAIC_GITEA_CREDENTIAL_FILE',None);env.pop('MOSAIC_LIVE_PROVIDER',None)
|
|
if docker:env.pop('DOCKER_HOST',None)
|
|
else:env['DOCKER_HOST']='unix:///mnt/storage/scratch/rocko-r219-dead.sock'
|
|
log=OUT/(label+'.tap');begin=stamp();t=time.monotonic();timeout=False
|
|
print('START',label,begin,flush=True)
|
|
with log.open('wb') as f:
|
|
child=subprocess.Popen(args,cwd=cwd,env=env,stdout=f,stderr=subprocess.STDOUT,start_new_session=True)
|
|
try:rc=child.wait(timeout=deadline+15)
|
|
except subprocess.TimeoutExpired:
|
|
timeout=True
|
|
try:os.killpg(child.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
try:child.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
try:os.killpg(child.pid,signal.SIGKILL)
|
|
except ProcessLookupError:pass
|
|
child.wait()
|
|
rc=child.returncode
|
|
finally:
|
|
try:os.killpg(child.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
timeout = timeout or rc in (124,137,-9)
|
|
text=log.read_text(errors='replace')
|
|
counts={k:int(m[-1]) if (m:=re.findall(r'^# '+k+r' (\d+)\s*$',text,re.M)) else None for k in ['tests','pass','fail','cancelled','skipped']}
|
|
row=dict(label=label,args=args,start=begin,end=stamp(),seconds=round(time.monotonic()-t,3),exit=rc,timeout=timeout,counts=counts,skips=[l for l in text.splitlines() if re.search(r'^skip\b|# SKIP\b',l,re.I)],log=str(log.relative_to(ROOT)),tmpdir=short)
|
|
with (OUT/'results.jsonl').open('a') as f:f.write(json.dumps(row)+'\n')
|
|
print('END',json.dumps(row),flush=True)
|
|
if rc or timeout or re.search(r'^\s*not ok\b',text,re.M):raise RuntimeError('RED '+label+' '+str(log))
|
|
return row
|
|
singles=[line for line in (PACKET/'changed-paths.txt').read_text().splitlines() if '/tests/' in line]
|
|
def stages():
|
|
for path in singles:execute('single-'+Path(path).parent.parent.name+'-'+Path(path).stem,['node','--test',path],deadline=300)
|
|
for package in ['bus','cli','webui','harness']:
|
|
files=sorted(glob.glob(str(ROOT/'packages'/package/'tests'/'*.test.mjs')))
|
|
execute('suite-'+package,['node','--test',*files],cwd=ROOT/'packages'/package,deadline=1800)
|
|
for path in sorted(ROOT.glob('scripts/test-*.sh')):
|
|
# timeout itself implements TERM then KILL, inside the dedicated child group.
|
|
execute(path.stem,['timeout','-k','10','7200','bash',str(path)],deadline=7200,docker=True)
|
|
# Every mutant starts from verified candidate bytes and restores in finally.
|
|
for spec in json.loads((PACKET/'mutants.json').read_text()):
|
|
verify();edits=[spec]+spec.get('additional_edits',[]);saved={}
|
|
try:
|
|
for e in edits:
|
|
path=ROOT/e['file'];saved.setdefault(path,path.read_bytes());text=path.read_text()
|
|
if text.count(e['old'])!=1:raise RuntimeError('anchor '+spec['id'])
|
|
path.write_text(text.replace(e['old'],e['new'],1))
|
|
# Mutation intentionally changes candidate bytes; execute has its own verify.
|
|
oldverify=globals()['verify'];globals()['verify']=lambda:None
|
|
try:
|
|
execute('mutant-'+spec['id'],['node','--test','--test-name-pattern',spec['pattern'],spec['test']],deadline=180)
|
|
status='survived'
|
|
except RuntimeError:
|
|
record=json.loads((OUT/'results.jsonl').read_text().splitlines()[-1]);text=(ROOT/record['log']).read_text(errors='replace')
|
|
failing=re.findall(r'^\s*not ok \d+ - (.*)$',text,re.M)
|
|
status='killed' if record['exit'] and not record['timeout'] and any(re.search(spec['pattern'],name) for name in failing) and 'SyntaxError:' not in text else 'invalid-failure'
|
|
finally:globals()['verify']=oldverify
|
|
finally:
|
|
for path,data in saved.items():path.write_bytes(data)
|
|
verify()
|
|
with (OUT/'mutants.jsonl').open('a') as f:f.write(json.dumps(dict(id=spec['id'],status=status,test=spec['test'],pattern=spec['pattern']))+'\n')
|
|
if status!='killed':raise RuntimeError(spec['id']+' '+status)
|
|
if __name__=='__main__':
|
|
if Path.cwd()!=ROOT:raise SystemExit('wrong cwd')
|
|
head=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip()
|
|
expected=(PACKET/'run-head.txt').read_text().strip()
|
|
if head!=expected:raise SystemExit('wrong HEAD')
|
|
(OUT/'head.json').write_text(json.dumps(dict(head=head,manifest=hashlib.sha256((PACKET/'candidate-manifest.sha256').read_bytes()).hexdigest(),start=stamp())))
|
|
verify()
|
|
try:stages()
|
|
finally:
|
|
verify();(OUT/'cwd-sweep.json').write_text(json.dumps(sweep(),indent=2))
|
|
# Remove only the owned fixture compose network, if any; never daemon/volumes.
|
|
|
|
with (OUT/'network-cleanup.log').open('w') as f:
|
|
for network in ['rocko-r219-gate_default', ROOT.name+'_default']:
|
|
subprocess.run(['docker','network','rm',network],stdout=f,stderr=subprocess.STDOUT)
|