86 lines
4.2 KiB
Python
86 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Granted focused RED/GREEN or whole-file scope. No implicit execution."""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
|
|
sys.dont_write_bytecode = True
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
spec = importlib.util.spec_from_file_location('row253_driver', HERE/'driver.py')
|
|
driver = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(driver)
|
|
|
|
def totals(log):
|
|
found = {}
|
|
for line in log.read_text(errors='replace').splitlines():
|
|
for key in ['tests','pass','fail','skipped']:
|
|
prefix = '# '+key+' '
|
|
if line.startswith(prefix) and line[len(prefix):].isdigit():
|
|
found[key] = int(line[len(prefix):])
|
|
return found
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--grant',required=True)
|
|
ap.add_argument('--stage',choices=['focused','green','whole'],required=True)
|
|
ap.add_argument('--out',type=Path,required=True)
|
|
args = ap.parse_args()
|
|
if not __import__('os').environ.get('TMPDIR','').startswith('/mnt/storage/scratch/tmp/g'):
|
|
raise RuntimeError('TMPDIR must be the granted scratch directory')
|
|
expected = json.loads((HERE/'CANDIDATE-SHAS.json').read_text())
|
|
for file,sha in expected.items():
|
|
assert hashlib.sha256((driver.REPO/file).read_bytes()).hexdigest()==sha,file
|
|
args.out.mkdir(parents=True,exist_ok=True)
|
|
results = dict(grant=args.grant,stage=args.stage,steps=[])
|
|
def step(name,command,expected_red=False,seconds=1200):
|
|
result = driver.run(command,args.out/(name+'.log'),seconds=seconds)
|
|
result.update(name=name,expected_red=expected_red,counts=totals(Path(result['log'])))
|
|
results['steps'].append(result)
|
|
(args.out/'results.json').write_text(json.dumps(results,indent=2)+'\n')
|
|
if result['timeout']:
|
|
raise RuntimeError('timeout: report and end window')
|
|
if expected_red and result['code']==0:
|
|
raise RuntimeError('expected RED survived: report and end window')
|
|
if not expected_red and result['code']!=0:
|
|
raise RuntimeError('unexpected failure: report before continuing')
|
|
tests = ['packages/cli/tests/case2-unit.test.mjs','packages/cli/tests/case1-accept.test.mjs']
|
|
if args.stage in ['focused','green']:
|
|
for name,file,killer,pattern in [
|
|
('installer','scripts/case2-unit.sh',tests[0],'row253 factory socket uses actual config'),
|
|
('acceptance','scripts/case1-accept.sh',tests[1],'row253 socket observation uses configured root')]:
|
|
if args.stage=='green':
|
|
continue
|
|
target = driver.REPO/file
|
|
original = target.read_bytes()
|
|
status = subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=driver.REPO)
|
|
try:
|
|
target.write_bytes(subprocess.check_output(['git','show','c3d88840c:'+file],cwd=driver.REPO))
|
|
step(name+'-red',['node','--test','--test-reporter=tap','--test-name-pattern',pattern,killer],True)
|
|
finally:
|
|
target.write_bytes(original)
|
|
assert target.read_bytes()==original
|
|
assert subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=driver.REPO)==status
|
|
for i,test in enumerate(tests):
|
|
step('focused-'+str(i+1),['node','--test','--test-reporter=tap',test])
|
|
else:
|
|
lists = json.loads((HERE/'run3-gate/FILE-LISTS.json').read_text())
|
|
for package, files in lists.items():
|
|
for index, file in enumerate(files):
|
|
step(package+'-'+str(index+1).zfill(3),['node','--test','--test-reporter=tap',file])
|
|
scripts = sorted((driver.REPO/'scripts').glob('test-*.sh'))
|
|
assert len(scripts)==10
|
|
limits = json.loads((HERE/'SHELL-LIMITS.json').read_text())['scripts']
|
|
for i,path in enumerate(scripts):
|
|
script = str(path.relative_to(driver.REPO))
|
|
limit = limits[script]['limit_seconds']
|
|
step('shell-'+str(i+1),['timeout','-k','10',str(limit),'bash',script],seconds=limit+30)
|
|
print(json.dumps(results,indent=2))
|
|
|
|
if __name__=='__main__':
|
|
main()
|