167 lines
7.7 KiB
Python
167 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Mos-1594: M1 RED then the same fifty fixed T9 runs. Requires Sage's slot CLEAR."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[4]
|
|
TEST = Path('packages/bus/tests/library-credential.test.mjs')
|
|
ANCHOR = b' runtime = await start(2147483647);'
|
|
BASE_COMMAND = ['node', '--test', '--test-reporter=tap', '--test-concurrency=1', '--test-timeout=90000']
|
|
PATTERN = '^C2-03b T9 runtime:'
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def sample_nice(child_pid, records, stopped):
|
|
"""Read-only samples of the runner and timeout/Node descendants (sage-4196)."""
|
|
seen = set()
|
|
started = time.monotonic()
|
|
while True:
|
|
pending = [os.getpid(), child_pid]
|
|
visited = set()
|
|
while pending:
|
|
pid = pending.pop()
|
|
if pid in visited:
|
|
continue
|
|
visited.add(pid)
|
|
proc = Path('/proc') / str(pid)
|
|
try:
|
|
stat = (proc / 'stat').read_text()
|
|
fields = stat.rsplit(')', 1)[1].split()
|
|
nice = int(fields[16]) # stat field19, after pid and parenthesized comm
|
|
command = stat.split('(', 1)[1].rsplit(')', 1)[0]
|
|
key = (pid, command, nice)
|
|
if key not in seen:
|
|
records.append({'pid': pid, 'command': command, 'nice': nice,
|
|
'seconds': round(time.monotonic() - started, 3)})
|
|
seen.add(key)
|
|
if pid != os.getpid():
|
|
pending.extend(map(int, (proc / 'task' / str(pid) / 'children').read_text().split()))
|
|
except (OSError, ValueError):
|
|
pass # Short-lived children may exit between the read-only samples.
|
|
if stopped.wait(.1):
|
|
return
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--clear-key', required=True, help='Sage message authorizing this test slot')
|
|
parser.add_argument('--output', required=True, type=Path)
|
|
parser.add_argument('--runs', type=int, default=50)
|
|
args = parser.parse_args()
|
|
if args.runs < 50:
|
|
parser.error('Mos requires at least fifty attempts per phase')
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=False)
|
|
path = ROOT / TEST
|
|
fixed = path.read_bytes()
|
|
committed = subprocess.check_output(['git', 'show', f'HEAD:{TEST}'], cwd=ROOT)
|
|
assert fixed == committed and fixed.count(ANCHOR) == 1, 'Start with the committed fixed test'
|
|
report = {'clear': args.clear_key, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
|
|
'fixedSHA256': digest(fixed), 'runsPerPhase': args.runs, 'loadStart': os.getloadavg(), 'runs': [], 'status': 'running'}
|
|
|
|
def save():
|
|
(output / 'summary.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
|
|
def run(phase, number, whole=False):
|
|
temporary = subprocess.check_output(['mktemp', '-d', '/mnt/storage/scratch/tmp/gXXX'], text=True).strip()
|
|
command = ['timeout', '-k', '10', '120', *BASE_COMMAND]
|
|
if not whole:
|
|
command += ['--test-name-pattern', PATTERN]
|
|
command += [str(TEST)]
|
|
env = {**os.environ, 'TMPDIR': temporary}
|
|
env.pop('MOSAIC_LIVE_PROVIDER', None)
|
|
env.pop('NODE_TEST_CONTEXT', None)
|
|
log = output / f'{phase}-{number:03}.tap'
|
|
before = time.monotonic()
|
|
signals = []
|
|
with log.open('w') as stream:
|
|
child = subprocess.Popen(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True)
|
|
nice_samples, nice_stop = [], threading.Event()
|
|
sampler = threading.Thread(target=sample_nice, args=(child.pid, nice_samples, nice_stop), daemon=True)
|
|
sampler.start()
|
|
try:
|
|
code = child.wait(timeout=135)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
signals.append('KILL')
|
|
code = child.wait()
|
|
finally:
|
|
nice_stop.set()
|
|
sampler.join()
|
|
try:
|
|
os.killpg(child.pid, signal.SIGTERM)
|
|
signals.append('TERM')
|
|
time.sleep(.2)
|
|
try:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
signals.append('KILL')
|
|
except ProcessLookupError:
|
|
pass
|
|
except ProcessLookupError:
|
|
pass
|
|
text = log.read_text()
|
|
counts = {name: int(value) for name, value in re.findall(r'^# (tests|pass|fail|cancelled|skipped) (\d+)$', text, re.M)}
|
|
green = code == 0 and counts.get('pass') == (6 if whole else 1) and counts.get('fail') == 0 and counts.get('cancelled') == 0
|
|
race = (not whole and code == 1 and counts.get('fail') == 1 and counts.get('cancelled') == 0
|
|
and 'ERR_ASSERTION' in text and re.search(r'^not ok \d+ - C2-03b T9 runtime:', text, re.M) is not None
|
|
and "+ code: 'library-sync-busy'" in text and "- code: 'library-fetch-no-grant'" in text)
|
|
entry = {'phase': phase, 'attempt': number, 'command': command, 'TMPDIR': temporary, 'load': os.getloadavg(),
|
|
'exit': code, 'counts': counts, 'green': green, 'race': race, 'seconds': round(time.monotonic() - before, 3),
|
|
'niceSamples': nice_samples, 'cleanupSignals': signals, 'TAP': log.name, 'TAPSHA256': digest(log.read_bytes())}
|
|
report['runs'].append(entry)
|
|
save()
|
|
print(json.dumps(entry), flush=True)
|
|
if not green and not (phase.startswith('M1-') and race):
|
|
raise RuntimeError(f'Unexpected red in {log}; stop for Sage, do not change assertions')
|
|
return race
|
|
|
|
def mutant(interval):
|
|
label = f'M1-{interval}'
|
|
path.write_bytes(fixed.replace(ANCHOR, f' runtime = await start({interval});'.encode(), 1))
|
|
try:
|
|
races = sum(run(label, number) for number in range(1, args.runs + 1))
|
|
report[label] = {'attempts': args.runs, 'raceFailures': races}
|
|
return races
|
|
finally:
|
|
path.write_bytes(fixed)
|
|
assert digest(path.read_bytes()) == report['fixedSHA256'], 'Mutant restore changed fixed test'
|
|
stat = subprocess.check_output(['git', 'diff', '--stat', 'HEAD'], cwd=ROOT, text=True)
|
|
status = subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True)
|
|
assert not stat and not status, 'Mutant restore leaves a dirty worktree'
|
|
report[label + '-restore'] = {'SHA256': digest(path.read_bytes()), 'gitDiffStat': stat, 'gitStatusPorcelain': status}
|
|
save()
|
|
|
|
try:
|
|
races = mutant(50)
|
|
if races == 0:
|
|
races = mutant(1)
|
|
if races == 0:
|
|
raise RuntimeError('Neither M1 interval failed; fix cannot be shown to matter. Stop for Sage.')
|
|
for number in range(1, args.runs + 1):
|
|
run('fixed', number)
|
|
run('whole-file', 1, whole=True)
|
|
report['status'] = 'green'
|
|
finally:
|
|
path.write_bytes(fixed)
|
|
report['loadEnd'] = os.getloadavg()
|
|
report['restored'] = digest(path.read_bytes()) == report['fixedSHA256']
|
|
report['gitDiffStat'] = subprocess.check_output(['git', 'diff', '--stat', 'HEAD'], cwd=ROOT, text=True)
|
|
report['gitStatusPorcelain'] = subprocess.check_output(['git', 'status', '--porcelain'], cwd=ROOT, text=True)
|
|
save()
|
|
assert report['restored'] and not report['gitDiffStat'] and not report['gitStatusPorcelain']
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|