import os,sys,json,time,re,pathlib,subprocess,shlex,hashlib,signal REPO=pathlib.Path('/home/jwoltje/.t3/worktrees/mosaic-stack/gander-row196') ROOT=pathlib.Path(__file__).parent; OUT=ROOT/'records'; OUT.mkdir(exist_ok=True) CASE='packages/cli/tests/case1-accept.test.mjs' HELPER='packages/cli/tests/case-p10-timing-fixture.mjs' HEAD='ffa93b054f28852a75017928d8478bdf9046e717' DIGEST='277ace0b4274970d5bd4c05cf8fe99aa1e7ebc469a719e4cefcb334a75d3865d' FILES=sorted(str(p.relative_to(REPO)) for p in (REPO/'packages/cli/tests').glob('*.test.mjs')) PLAN=json.loads((REPO/'agents/gander/work/queue-299/MUTANT-PLAN.json').read_text())['mutants'] STATE={'grant':'sage-3268','m4_ruling':'sage-3273 / mos-1426','head':HEAD,'manifest':DIGEST,'status':'PREPARED','cli_file_list':FILES,'cli_file_count':len(FILES),'stages':[],'restores':[]} def utc():return time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()) def git(*args):return subprocess.check_output(['git',*args],cwd=REPO,text=True).strip() def save(): (OUT/'results.json').write_text(json.dumps(STATE,indent=2)+'\n') def stop(reason): STATE['status']='STOP';STATE['reason']=reason;STATE['stopped_utc']=utc();save();print('STOP '+reason,flush=True);raise SystemExit(1) def load(): return pathlib.Path('/proc/loadavg').read_text().strip() def busy(label): r=subprocess.run(['bash','/mnt/storage/scratch/sage-busy-now.sh'],capture_output=True,text=True) (OUT/(label+'.busy.log')).write_text(r.stdout+r.stderr) if r.returncode or r.stdout.strip().splitlines()[-1:]!=['CLEAR']:stop('busy refusal before '+label) def parse_metrics(text): result=[] for line in text.splitlines(): m=re.match(r'^\s*#\s*(\{.*\})\s*$',line) if m: try: item=json.loads(m[1]) if 'probeSeconds' in item:result.append(item) except json.JSONDecodeError:pass return result def execute(label,args,limit,expected_red=False,syntax=False): busy(label) tmp=subprocess.check_output(['mktemp','-d','/mnt/storage/scratch/tmp/gXXXXXX'],text=True).strip() env=os.environ.copy();env['TMPDIR']=tmp argv=['timeout','-k','10',str(limit),*args] meta={'label':label,'argv':argv,'limit_seconds':limit,'tmpdir':tmp,'started_utc':utc(),'start_load':load(),'log':str(OUT/(label+'.log')),'expected_red':expected_red} (OUT/(label+'.command.json')).write_text(json.dumps(meta,indent=2)+'\n') STATE['active']=meta;save();print('START '+json.dumps(meta),flush=True) shell='set -o pipefail\n'+shlex.join(argv)+' 2>&1 | tee '+shlex.quote(meta['log']) start=time.monotonic();samples=[];p=subprocess.Popen(['bash','-c',shell],cwd=REPO,env=env,start_new_session=True) while True: samples.append({'wall_seconds':round(time.monotonic()-start,3),'utc':utc(),'load':load()}) try:rc=p.wait(timeout=5);break except subprocess.TimeoutExpired: if time.monotonic()-start>limit+30: os.killpg(p.pid,signal.SIGKILL);p.wait();stop('outer controller timeout '+label) text=pathlib.Path(meta['log']).read_text();seconds=round(time.monotonic()-start,3) counts={} for key in ['tests','pass','fail','cancelled','skipped','todo']: matches=re.findall(r'^# '+key+r' (\d+)\s*$',text,re.M) if matches:counts[key]=int(matches[-1]) row={**meta,'rc':rc,'wall_seconds':seconds,'finished_utc':utc(),'end_load':load(),'counts':counts,'metrics':parse_metrics(text),'skip_lines':[s for s in text.splitlines() if '# SKIP' in s],'external_timeout':rc in (124,137)} (OUT/(label+'.load.json')).write_text(json.dumps(samples,indent=2)+'\n') STATE['stages'].append(row);STATE.pop('active',None);save();print('END '+json.dumps(row),flush=True) if row['external_timeout']:stop('timeout '+label) if not expected_red and rc!=0:stop('unexpected red '+label) if not expected_red and not syntax: if len(row['metrics'])!=4:stop('missing direct P10 diagnostics '+label) for m in row['metrics']: lo,hi=(1.25,2.75) if m['fault']=='inner' else (3.25,4.75) if not lo<=m['probeSeconds']