117 lines
7.4 KiB
Python
117 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
||
# Row 111 (#1605) mutants. Each takes out or bends one new check in
|
||
# startHost; every named (test file, name pattern) run must fail on it. n00
|
||
# puts BASE's host.mjs back under the new tests. Each file is restored from
|
||
# its .orig after the run, and the candidate's diff must hash the same at the
|
||
# end as at the start.
|
||
import hashlib, os, signal, subprocess, sys, time
|
||
|
||
WT = os.path.expanduser("~/darkwing-scratch/r111/wt")
|
||
OUT = os.path.expanduser("~/darkwing-scratch/r111/mutants")
|
||
ENV = dict(os.environ, TMPDIR=os.path.expanduser("~/darkwing-scratch/r111/tmp"))
|
||
os.makedirs(OUT, exist_ok=True)
|
||
os.makedirs(ENV["TMPDIR"], exist_ok=True)
|
||
|
||
HO, T = "packages/cli/src/host.mjs", "packages/cli/tests/host.test.mjs"
|
||
NFORK, NSEND = (T, "^a notifier fork that throws"), (T, "^a notifier send that throws")
|
||
READER, WRITE = (T, "^a boot reply with no reader"), (T, "^a host.json that can't be written")
|
||
EB, EN = (T, "^an error event on the broker before it replies"), (T, "^an error event on the notifier before it replies")
|
||
KEPT = (T, "^an error event on the broker while the notifier starts")
|
||
UB, UN = (T, "^an error event on the broker once the host is up"), (T, "^an error event on the notifier once the host is up")
|
||
|
||
BASE_HO = (HO, None, subprocess.run(["git", "-C", WT, "show", "HEAD:" + HO], capture_output=True, text=True, check=True).stdout)
|
||
FORK = ' notify = fork(NOTIFIER, [], { stdio: ["ignore", "inherit", "inherit", "ipc"] });\n notify.on("error", onChildError("notifier"));\n'
|
||
SEND_WAIT = ''' notify.send({ op: "start", path: ready.path, cap: reader.cap, business, dataRoot, binding: notifier.binding, base: notifier.base, pollMs: notifier.pollMs });
|
||
const ok = await firstReply(notify, START_TIMEOUT_MS, "notifier");'''
|
||
WRITES = ''' rmSync(hostFile(dataRoot), { force: true });
|
||
writeHostState(dataRoot, state);
|
||
'''
|
||
CATCH_END = ''' if (broker.connected) broker.send({ op: "close" }, () => {});
|
||
await ended(broker, CLOSE_TIMEOUT_MS);
|
||
release();
|
||
throw e;
|
||
}
|
||
'''
|
||
|
||
# (name, [(path, old, new)], [(test, pattern)], expect) with old None for a whole-file replacement.
|
||
M = [
|
||
("n00 BASE's startHost under the new tests", [BASE_HO], [NFORK, NSEND, READER, WRITE, EB, EN, KEPT, UB, UN], "killed"),
|
||
("n01 the wait for a reply ignores error", [(HO, ' child.on("error", onError);\n', "")], [EB, EN], "killed"),
|
||
("n02 no error listener on the broker", [(HO, ' broker.on("error", onChildError("broker"));\n', "")], [KEPT, UB], "killed"),
|
||
("n03 no error listener on the notifier", [(HO, ' notify.on("error", onChildError("notifier"));\n', "")], [UN], "killed"),
|
||
("n04 a broker error once up is dropped", [(HO, ' if (name === "broker") return breakChannel(`broker error: ${e.message}`);', ' if (name === "broker") return;')], [UB], "killed"),
|
||
("n05 a notifier error once up only logs", [(HO, " log(`notifier error (${e.message}); stopping the host`);\n close(1);\n", " log(`notifier error (${e.message}); stopping the host`);\n")], [UN], "killed"),
|
||
("n06 no reader check", [(HO, " if (!reader) throw new CliError(`broker has no reader for business ${business}`, 3);\n", "")], [READER], "killed"),
|
||
("n07 the notifier fork runs before the shared try", [(HO, FORK, ""), (HO, " let notify = null;\n try {\n", " let notify = null;\n if (notifier) {\n" + FORK.replace(" ", " ") + " }\n try {\n")], [NFORK], "killed"),
|
||
("n08 the notifier's wait starts before its send", [(HO, SEND_WAIT, ''' const started = firstReply(notify, START_TIMEOUT_MS, "notifier");
|
||
notify.send({ op: "start", path: ready.path, cap: reader.cap, business, dataRoot, binding: notifier.binding, base: notifier.base, pollMs: notifier.pollMs });
|
||
const ok = await started;''')], [NSEND], "killed"),
|
||
("n09 host.json is written after the shared try", [(HO, WRITES, ""), (HO, " throw e;\n }\n\n // stopping:", " throw e;\n }\n" + WRITES.replace(" ", " ") + "\n // stopping:")], [WRITE], "killed"),
|
||
("n10 a kept child error is not thrown", [(HO, " if (childError) throw childError;\n", "")], [KEPT], "killed"),
|
||
("n11 the failure path leaves the notifier running", [(HO, ''' if (notify) {
|
||
notify.kill("SIGTERM");
|
||
await ended(notify, 5000);
|
||
}
|
||
''', "")], [WRITE, NSEND], "killed"),
|
||
("n12 the failure path does not wait for the broker", [(HO, CATCH_END, CATCH_END.replace(" await ended(broker, CLOSE_TIMEOUT_MS);\n", ""))], [READER, WRITE], "killed"),
|
||
("n13 the failure path keeps the lock", [(HO, CATCH_END, CATCH_END.replace(" release();\n", ""))], [READER, WRITE, NFORK], "killed"),
|
||
]
|
||
|
||
def diffhash():
|
||
d = subprocess.run(["git", "-C", WT, "diff", "--", ".", ":!node_modules"], capture_output=True).stdout
|
||
return hashlib.sha256(d).hexdigest()
|
||
|
||
def run(test, pattern, log):
|
||
pkg = os.path.join(WT, *test.split("/")[:2])
|
||
with open(log, "ab") as f:
|
||
f.write(f"== {test} /{pattern}/\n".encode()); f.flush()
|
||
p = subprocess.Popen(["node", "--test", f"--test-name-pattern={pattern}", os.path.relpath(os.path.join(WT, test), pkg)], cwd=pkg, env=ENV,
|
||
stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
|
||
try:
|
||
rc = p.wait(timeout=600)
|
||
except subprocess.TimeoutExpired:
|
||
os.killpg(p.pid, signal.SIGKILL); p.wait(); return "timeout"
|
||
# A pattern that matched nothing passes with no tests; count that as a fault.
|
||
# node counts a test that hits its timeout as cancelled, not failed.
|
||
tail = open(log, encoding="utf8", errors="replace").read().split(f"== {test} /{pattern}/\n")[-1]
|
||
ran = sum(int(l.split()[2]) for l in tail.splitlines() if l.startswith(("ℹ pass ", "ℹ fail ", "ℹ cancelled ")))
|
||
return "no-tests" if ran == 0 else rc
|
||
|
||
for _, edits, *_ in M:
|
||
for path, *_ in edits:
|
||
if os.path.exists(os.path.join(WT, path + ".orig")):
|
||
sys.exit(f"STOP {path}.orig exists")
|
||
start = diffhash()
|
||
print(f"start diff {start[:16]} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True)
|
||
bad = []
|
||
only = set(sys.argv[1:])
|
||
for name, edits, tests, expect in M:
|
||
if only and name.split()[0] not in only: continue
|
||
srcs = {}
|
||
ok = True
|
||
for path, old, new in edits:
|
||
full = os.path.join(WT, path)
|
||
src = srcs.get(path) or open(full).read()
|
||
if old is not None and src.count(old) != 1:
|
||
print(f"{name}: ANCHOR {path} {src.count(old)}", flush=True); ok = False; break
|
||
srcs[path] = new if old is None else src.replace(old, new)
|
||
if not ok:
|
||
bad.append(name); continue
|
||
for path in srcs: os.rename(os.path.join(WT, path), os.path.join(WT, path + ".orig"))
|
||
try:
|
||
for path, text in srcs.items(): open(os.path.join(WT, path), "w").write(text)
|
||
log = os.path.join(OUT, name.split()[0] + ".log")
|
||
open(log, "w").close()
|
||
rcs = [run(t, p, log) for t, p in tests]
|
||
finally:
|
||
for path in srcs: os.replace(os.path.join(WT, path + ".orig"), os.path.join(WT, path))
|
||
if any(rc in ("timeout", "no-tests") for rc in rcs): got = "FAULT"
|
||
elif all(rc != 0 for rc in rcs): got = "killed"
|
||
elif all(rc == 0 for rc in rcs): got = "survived"
|
||
else: got = "MIXED"
|
||
print(f"{name}: {got} (expected {expect}) {list(zip([t.split('/')[-1] for t, _ in tests], rcs))}", flush=True)
|
||
if got != expect: bad.append(name)
|
||
end = diffhash()
|
||
print(f"end diff {end[:16]} {'same' if end == start else 'CHANGED'} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True)
|
||
print("MUTANTS_OK" if not bad and end == start else f"MUTANTS_BAD {bad}")
|