jason.woltjeandClaude Opus 5.5
e1bf582a1a
build.patch againstd13b9300(brief blobf8d14d70, rev 630), candidate manifest, identity and mutant harnesses with results, gate summary. Co-Authored-By: Claude Opus 5.5 <[email protected]>
90 lines
4.5 KiB
Python
90 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
# Dewey row 83 mutants: single-site edits to scripts/webui-service.sh in a mutant worktree (never
|
|
# the build tree). Each mutant runs against the base tests (d13b9300's service.test.mjs, copied in
|
|
# as tests/base-service.test.mjs) and the candidate tests (tests/service.test.mjs), then
|
|
# identity.mjs. The candidate is uncommitted there, so the script is restored from the copy read
|
|
# before the edit, not from git.
|
|
# Usage: mutants.py [mutant ...]. A site that doesn't match exactly once is BAD-SITE.
|
|
import os, subprocess, sys
|
|
|
|
R = os.path.expanduser('~/dewey-scratch/r83')
|
|
T = R + '/mut'
|
|
OUT = R + '/out/mut'
|
|
S = 'scripts/webui-service.sh'
|
|
BASE_TESTS = ['node', '--test', 'packages/webui/tests/base-service.test.mjs']
|
|
CAND_TESTS = ['node', '--test', 'packages/webui/tests/service.test.mjs']
|
|
IDENTITY = ['node', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'identity.mjs'), T]
|
|
|
|
CHECK = ' [ "$SEND" = 0 ] || [ "$preview" -eq 3 ] || die "--send needs the preview mode (--public-host, --auth-header and --allow-user)"\n'
|
|
APPEND = ' [ "$SEND" = 0 ] || flags="$flags --send"\n'
|
|
CASE = ' --send) SEND=1; SERVE_SET=1; shift ;;\n'
|
|
STATUS_IF = """ if grep -q '^ExecStart=.* --allow-user [^ ][^ ]* --send$' "$target"; then\n"""
|
|
STATUS_BLOCK = STATUS_IF + """ echo "send: on (--send; human messages to roles)"
|
|
else
|
|
echo "send: off (read-only)"
|
|
fi
|
|
"""
|
|
MUTANTS = [
|
|
# name, edits [(old, new)]
|
|
('send-no-preview-check', [(CHECK, '')]),
|
|
('send-needs-any-preview', [(CHECK, CHECK.replace('-eq 3', '-ne 0'))]),
|
|
('send-message-reworded', [(CHECK, CHECK.replace('needs the preview mode', 'needs the preview set'))]),
|
|
('send-no-append', [(APPEND, '')]),
|
|
('send-append-always', [(APPEND, ' flags="$flags --send"\n')]),
|
|
('send-append-before-preview', [(APPEND, ''), (' local preview=0\n', APPEND + ' local preview=0\n')]),
|
|
('send-default-on', [('SEND=0\n', 'SEND=1\n')]),
|
|
('send-no-serve-set', [(CASE, CASE.replace(' SERVE_SET=1;', ''))]),
|
|
('send-takes-value', [(CASE, ' --send) [ $# -ge 2 ] || usage; SEND=1; SERVE_SET=1; shift 2 ;;\n')]),
|
|
('send-flag-unknown', [(CASE, '')]),
|
|
('status-send-always-off', [(STATUS_IF, ' if false; then\n')]),
|
|
('status-send-always-on', [(STATUS_IF, ' if true; then\n')]),
|
|
('status-no-send-line', [(STATUS_BLOCK, '')]),
|
|
('status-send-anywhere', [(STATUS_IF, """ if grep -q -- '--send' "$target"; then\n""")]),
|
|
('status-send-suffix-only', [(STATUS_IF, """ if grep -q '^ExecStart=.* --send$' "$target"; then\n""")]),
|
|
]
|
|
|
|
def run(cmd):
|
|
try:
|
|
p = subprocess.run(cmd, cwd=T, capture_output=True, text=True, env={**os.environ, 'TMPDIR': R + '/tmp'}, timeout=600)
|
|
except subprocess.TimeoutExpired:
|
|
return 'TIMEOUT', 'timed out after 600 s'
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
def verdict(rc):
|
|
return 'survived' if rc == 0 else ('TIMEOUT' if rc == 'TIMEOUT' else 'killed')
|
|
|
|
os.makedirs(OUT, exist_ok=True)
|
|
os.makedirs(R + '/tmp', exist_ok=True)
|
|
path = os.path.join(T, S)
|
|
pristine = open(path).read()
|
|
for label, cmd in (('base tests', BASE_TESTS), ('candidate tests', CAND_TESTS), ('identity', IDENTITY)):
|
|
rc, out = run(cmd)
|
|
open(f'{OUT}/baseline-{label.replace(" ", "-")}.txt', 'w').write(out)
|
|
print(f'baseline {label}:', 'pass' if rc == 0 else f'FAIL rc={rc}', flush=True)
|
|
want = sys.argv[1:]
|
|
results = []
|
|
for name, edits in MUTANTS:
|
|
if want and name not in want:
|
|
continue
|
|
bad = [old for old, _ in edits if pristine.count(old) != 1]
|
|
if bad:
|
|
results.append(f'{name} BAD-SITE ({[pristine.count(o) for o in bad]} matches)')
|
|
print(results[-1], flush=True)
|
|
continue
|
|
mutated = pristine
|
|
for old, new in edits:
|
|
mutated = mutated.replace(old, new)
|
|
try:
|
|
open(path, 'w').write(mutated)
|
|
b_rc, b_out = run(BASE_TESTS)
|
|
c_rc, c_out = run(CAND_TESTS)
|
|
i_rc, i_out = run(IDENTITY)
|
|
finally:
|
|
open(path, 'w').write(pristine)
|
|
open(f'{OUT}/{name}.txt', 'w').write(f'== base tests rc={b_rc}\n{b_out}\n== candidate tests rc={c_rc}\n{c_out}\n== identity rc={i_rc}\n{i_out}')
|
|
results.append(f'{name:28} base tests {verdict(b_rc):8} candidate tests {verdict(c_rc):8} identity {"same" if i_rc == 0 else "differs"}')
|
|
print(results[-1], flush=True)
|
|
assert open(path).read() == pristine, 'script not restored'
|
|
killed = sum(1 for r in results if 'candidate tests killed' in r)
|
|
print(f'{len(results)} mutants: candidate tests kill {killed}', flush=True)
|