#!/usr/bin/env python3 """Mos-1594: M1 RED then the same fifty fixed T9 runs. Requires Sage's slot CLEAR.""" import argparse import hashlib import json import os from pathlib import Path import re import signal import subprocess import threading import time ROOT = Path(__file__).resolve().parents[4] TEST = Path('packages/bus/tests/library-credential.test.mjs') ANCHOR = b' runtime = await start(2147483647);' BASE_COMMAND = ['node', '--test', '--test-reporter=tap', '--test-concurrency=1', '--test-timeout=90000'] PATTERN = '^C2-03b T9 runtime:' def digest(value): return hashlib.sha256(value).hexdigest() def sample_nice(child_pid, records, stopped): """Read-only samples of the runner and timeout/Node descendants (sage-4196).""" seen = set() started = time.monotonic() while True: pending = [os.getpid(), child_pid] visited = set() while pending: pid = pending.pop() if pid in visited: continue visited.add(pid) proc = Path('/proc') / str(pid) try: stat = (proc / 'stat').read_text() fields = stat.rsplit(')', 1)[1].split() nice = int(fields[16]) # stat field19, after pid and parenthesized comm command = stat.split('(', 1)[1].rsplit(')', 1)[0] key = (pid, command, nice) if key not in seen: records.append({'pid': pid, 'command': command, 'nice': nice, 'seconds': round(time.monotonic() - started, 3)}) seen.add(key) if pid != os.getpid(): pending.extend(map(int, (proc / 'task' / str(pid) / 'children').read_text().split())) except (OSError, ValueError): pass # Short-lived children may exit between the read-only samples. if stopped.wait(.1): return def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--clear-key', required=True, help='Sage message authorizing this test slot') parser.add_argument('--output', required=True, type=Path) parser.add_argument('--runs', type=int, default=50) args = parser.parse_args() if args.runs < 50: parser.error('Mos requires at least fifty attempts per phase') output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) path = ROOT / TEST fixed = path.read_bytes() committed = subprocess.check_output(['git', 'show', f'HEAD:{TEST}'], cwd=ROOT) assert fixed == committed and fixed.count(ANCHOR) == 1, 'Start with the committed fixed test' report = {'clear': args.clear_key, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), 'fixedSHA256': digest(fixed), 'runsPerPhase': args.runs, 'loadStart': os.getloadavg(), 'runs': [], 'status': 'running'} def save(): (output / 'summary.json').write_text(json.dumps(report, indent=2) + '\n') def run(phase, number, whole=False): temporary = subprocess.check_output(['mktemp', '-d', '/mnt/storage/scratch/tmp/gXXX'], text=True).strip() command = ['timeout', '-k', '10', '120', *BASE_COMMAND] if not whole: command += ['--test-name-pattern', PATTERN] command += [str(TEST)] env = {**os.environ, 'TMPDIR': temporary} env.pop('MOSAIC_LIVE_PROVIDER', None) env.pop('NODE_TEST_CONTEXT', None) log = output / f'{phase}-{number:03}.tap' before = time.monotonic() signals = [] with log.open('w') as stream: child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) nice_samples, nice_stop = [], threading.Event() sampler = threading.Thread(target=sample_nice, args=(child.pid, nice_samples, nice_stop), daemon=True) sampler.start() try: code = child.wait(timeout=135) except subprocess.TimeoutExpired: os.killpg(child.pid, signal.SIGKILL) signals.append('KILL') code = child.wait() finally: nice_stop.set() sampler.join() try: os.killpg(child.pid, signal.SIGTERM) signals.append('TERM') time.sleep(.2) try: os.killpg(child.pid, signal.SIGKILL) signals.append('KILL') except ProcessLookupError: pass except ProcessLookupError: pass text = log.read_text() counts = {name: int(value) for name, value in re.findall(r'^# (tests|pass|fail|cancelled|skipped) (\d+)$', text, re.M)} green = code == 0 and counts.get('pass') == (6 if whole else 1) and counts.get('fail') == 0 and counts.get('cancelled') == 0 race = (not whole and code == 1 and counts.get('fail') == 1 and counts.get('cancelled') == 0 and 'ERR_ASSERTION' in text and re.search(r'^not ok \d+ - C2-03b T9 runtime:', text, re.M) is not None and "+ code: 'library-sync-busy'" in text and "- code: 'library-fetch-no-grant'" in text) entry = {'phase': phase, 'attempt': number, 'command': command, 'TMPDIR': temporary, 'load': os.getloadavg(), 'exit': code, 'counts': counts, 'green': green, 'race': race, 'seconds': round(time.monotonic() - before, 3), 'niceSamples': nice_samples, 'cleanupSignals': signals, 'TAP': log.name, 'TAPSHA256': digest(log.read_bytes())} report['runs'].append(entry) save() print(json.dumps(entry), flush=True) if not green and not (phase.startswith('M1-') and race): raise RuntimeError(f'Unexpected red in {log}; stop for Sage, do not change assertions') return race def mutant(interval): label = f'M1-{interval}' path.write_bytes(fixed.replace(ANCHOR, f' runtime = await start({interval});'.encode(), 1)) try: races = sum(run(label, number) for number in range(1, args.runs + 1)) report[label] = {'attempts': args.runs, 'raceFailures': races} return races finally: path.write_bytes(fixed) assert digest(path.read_bytes()) == report['fixedSHA256'], 'Mutant restore changed fixed test' stat = subprocess.check_output(['git', 'diff', '--stat', 'HEAD'], cwd=ROOT, text=True) status = subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True) assert not stat and not status, 'Mutant restore leaves a dirty worktree' report[label + '-restore'] = {'SHA256': digest(path.read_bytes()), 'gitDiffStat': stat, 'gitStatusPorcelain': status} save() try: races = mutant(50) if races == 0: races = mutant(1) if races == 0: raise RuntimeError('Neither M1 interval failed; fix cannot be shown to matter. Stop for Sage.') for number in range(1, args.runs + 1): run('fixed', number) run('whole-file', 1, whole=True) report['status'] = 'green' finally: path.write_bytes(fixed) report['loadEnd'] = os.getloadavg() report['restored'] = digest(path.read_bytes()) == report['fixedSHA256'] report['gitDiffStat'] = subprocess.check_output(['git', 'diff', '--stat', 'HEAD'], cwd=ROOT, text=True) report['gitStatusPorcelain'] = subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True) save() assert report['restored'] and not report['gitDiffStat'] and not report['gitStatusPorcelain'] if __name__ == '__main__': main()