94 lines
6.2 KiB
Python
94 lines
6.2 KiB
Python
import os, sys, json, time, pathlib, subprocess, shlex, re, math
|
|
ROOT=pathlib.Path('/home/jwoltje/.t3/worktrees/mosaic-stack/gander-row196')
|
|
RUN=pathlib.Path(__file__).parent
|
|
OUT=RUN/'records'; OUT.mkdir(exist_ok=True)
|
|
FOUR=['packages/seat/tests/checkout.test.mjs','packages/seat/tests/working.test.mjs','packages/cli/tests/git-socket.test.mjs','packages/cli/tests/lifecycle.test.mjs']
|
|
PACKAGES={p:sorted(str(f.relative_to(ROOT)) for f in (ROOT/'packages'/p/'tests').glob('*.test.mjs')) for p in ['seat','cli']}
|
|
results={'head':subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(),'manifest':'cdda41f5baa6b04e6e5bb9b82beff8e00ceef440c6702565cc86944f249bb5d3','files':PACKAGES,'stages':[],'status':'PREPARED'}
|
|
def save(): (OUT/'results.json').write_text(json.dumps(results,indent=2)+'\n')
|
|
def fail(message):
|
|
results['status']='STOP'; results['stop']=message; save(); print('STOP '+message,flush=True); sys.exit(1)
|
|
def busy(label):
|
|
r=subprocess.run(['bash','/mnt/storage/scratch/sage-busy-now.sh'],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
|
(OUT/(label+'.busy.log')).write_text(r.stdout)
|
|
if r.returncode or r.stdout.strip().splitlines()[-1:]!=['CLEAR']: fail('busy refusal before '+label+': '+r.stdout)
|
|
def writers(tmp):
|
|
found=[]
|
|
for p in pathlib.Path('/proc').glob('[0-9]*'):
|
|
try:
|
|
comm=(p/'comm').read_text().strip(); cwd=os.readlink(p/'cwd').removesuffix(' (deleted)')
|
|
if (comm=='git' or comm.startswith('git-')) and (cwd==str(tmp) or cwd.startswith(str(tmp)+'/')): found.append({'pid':int(p.name),'comm':comm,'cwd':cwd})
|
|
except (OSError,ValueError): pass
|
|
return found
|
|
def idle(label,tmp):
|
|
start=time.monotonic(); history=[]
|
|
while True:
|
|
live=writers(tmp); history.append({'seconds':round(time.monotonic()-start,3),'writers':live})
|
|
if not live: break
|
|
if time.monotonic()-start>60: fail('git writer remains after '+label)
|
|
time.sleep(.25)
|
|
(OUT/(label+'.sweep.json')).write_text(json.dumps(history,indent=2)+'\n')
|
|
def trace_counts(events):
|
|
starts=[]; commits=[]; maintenance=[]; files=list(events.glob('*'))
|
|
for file in files:
|
|
for line in file.read_text(errors='strict').splitlines():
|
|
e=json.loads(line); argv=e.get('argv',[])
|
|
if e.get('event')=='start':
|
|
starts.append(e)
|
|
if 'commit' in argv: commits.append(e)
|
|
if e.get('event')=='child_start' and any(argv[i:i+3]==['maintenance','run','--auto'] for i in range(len(argv)-2)): maintenance.append(e)
|
|
return {'event_files':len(files),'git_starts':len(starts),'commits':len(commits),'flagged_commits':sum('maintenance.auto=false' in e['argv'] and 'gc.auto=0' in e['argv'] for e in commits),'maintenance_children':len(maintenance),'maintenance_events':maintenance}
|
|
def run(label,args,limit,traced=False,positive=False):
|
|
busy(label)
|
|
folder=RUN/label; folder.mkdir(exist_ok=True); tmp=pathlib.Path(subprocess.check_output(['mktemp','-d','/mnt/storage/scratch/tmp/gXXXXXX'],text=True).strip())
|
|
if '__CONTROL_REPO__' in args:
|
|
repo=tmp/'repo'; repo.mkdir(); args=[str(repo) if a=='__CONTROL_REPO__' else a for a in args]
|
|
env=os.environ.copy(); env['TMPDIR']=str(tmp)
|
|
events=folder/'events'
|
|
if traced:
|
|
events.mkdir(exist_ok=True)
|
|
shim=RUN/'bin'/'git'
|
|
shim.write_text('#!/bin/bash\nexport GIT_TRACE2='+shlex.quote(str(folder/'git.trace'))+'\nexport GIT_TRACE2_EVENT='+shlex.quote(str(events))+'\nexec /usr/bin/git "$@"\n'); shim.chmod(0o755)
|
|
env['PATH']=str(RUN/'bin')+':'+env['PATH']
|
|
command=['timeout','-k','10',str(limit),*args]
|
|
header={'label':label,'head':results['head'],'manifest':results['manifest'],'argv':command,'limit':limit,'tmpdir':str(tmp)}
|
|
(OUT/(label+'.command.json')).write_text(json.dumps(header,indent=2)+'\n')
|
|
log=OUT/(label+'.log'); print('START '+json.dumps(header),flush=True)
|
|
shell='set -o pipefail\n'+shlex.join(command)+' 2>&1 | tee '+shlex.quote(str(log))
|
|
start=time.monotonic(); proc=subprocess.Popen(['bash','-c',shell],cwd=ROOT,env=env,start_new_session=True)
|
|
try: rc=proc.wait(timeout=limit+30)
|
|
except subprocess.TimeoutExpired:
|
|
import signal
|
|
os.killpg(proc.pid,signal.SIGKILL); proc.wait(); fail('outer timeout '+label)
|
|
seconds=round(time.monotonic()-start,3); text=log.read_text()
|
|
record={**header,'rc':rc,'wall_seconds':seconds,'log':str(log),'counts':{k:int(re.findall(r'^# '+k+r' (\d+)\s*$',text,re.M)[-1]) for k in ['tests','pass','fail','cancelled','skipped','todo'] if re.findall(r'^# '+k+r' (\d+)\s*$',text,re.M)},'skip_lines':[line for line in text.splitlines() if '# SKIP' in line]}
|
|
if traced:
|
|
idle(label,tmp); record['trace']=trace_counts(events)
|
|
results['stages'].append(record); save(); print('END '+json.dumps(record),flush=True)
|
|
if rc: fail('nonzero '+label+' rc='+str(rc))
|
|
if traced:
|
|
counts=record['trace']
|
|
if counts['commits']<1 or counts['event_files']<1: fail('vacuous trace '+label)
|
|
if positive and counts['maintenance_children']<1: fail('positive control missed maintenance')
|
|
if not positive and counts['maintenance_children']!=0: fail('maintenance child '+label)
|
|
if '--test' in args and record['counts'].get('fail',0): fail('test failure '+label)
|
|
def limit(file):
|
|
if file.endswith('/stack-release.test.mjs'): return 2400
|
|
if file.endswith('/case1-accept.test.mjs'): return 1200
|
|
return 900
|
|
save()
|
|
if len(sys.argv)<2 or sys.argv[1]!='run':
|
|
print(json.dumps({'prepared':True,'packages':{k:len(v) for k,v in PACKAGES.items()}})); sys.exit(0)
|
|
if float(pathlib.Path('/proc/loadavg').read_text().split()[0])>=4: fail('load not below 4 at start')
|
|
results['status']='RUNNING'; results['started_utc']=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()); save()
|
|
print(json.dumps({'head':results['head'],'manifest':results['manifest'],'whole_file_list':PACKAGES},indent=2),flush=True)
|
|
for mode in ['positive','flagged']:
|
|
run('control-'+mode,['node',str(RUN/'control.mjs'),'__CONTROL_REPO__',mode],120,traced=True,positive=mode=='positive')
|
|
for file in FOUR:
|
|
run('proof-'+pathlib.Path(file).stem,['node','--test','--test-concurrency=1','--test-reporter=tap',file],limit(file),traced=True)
|
|
for package,files in PACKAGES.items():
|
|
for file in files:
|
|
run(package+'-'+pathlib.Path(file).stem,['node','--test','--test-concurrency=1','--test-reporter=tap',file],limit(file))
|
|
results['status']='GREEN'; results['finished_utc']=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()); save()
|
|
print('GREEN all stages',flush=True)
|