#!/usr/bin/env python3 # Row 106 (#1600) mutants. Each takes out or bends one new check; every named # (test file, name pattern) run must fail on it. A mutant may edit several # files; each is restored from its .orig after the run, and the candidate's # diff must hash the same at the end as at the start. The x mutants are # controls: they run the base's `have` order test against n06 and n07, which # it passes, since it takes the clone order from the request it checks. import hashlib, os, signal, subprocess, sys, time WT = os.path.expanduser("~/darkwing-scratch/r106/wt") OUT = os.path.expanduser("~/darkwing-scratch/r106/mutants") ENV = dict(os.environ, TMPDIR=os.path.expanduser("~/darkwing-scratch/r106/tmp")) os.makedirs(OUT, exist_ok=True) os.makedirs(ENV["TMPDIR"], exist_ok=True) HO, GT = "packages/cli/src/host.mjs", "packages/harness/src/git-tools.mjs" T_HO, T_GT = "packages/cli/tests/host.test.mjs", "packages/harness/tests/git-tools.test.mjs" P_FORK, P_SEND = "^a broker fork that throws", "^a broker send that throws" P_GONE, P_HAVE = "^a host.lock that goes between", "^fetch_review's have orders" TRY = ''' try { broker = fork(BROKER, [], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); broker.send({ op: "boot", config: boot }); ready = await firstReply(broker, bootTimeoutMs, "broker");''' LOOP = ''' for (const remote of ["refs/remotes/working", "refs/remotes/origin"]) for (const project of projects) { if (have.size >= HAVE_MAX) break; const r = await git(["-C", join(workspace, project), "for-each-ref", "--sort=-committerdate", "--format=%(objectname)", remote], env); if (r.ok) for (const oid of r.out.split("\\n")) if (OID.test(oid) && have.size < HAVE_MAX) have.add(oid); }''' DATE_SORT = (GT, LOOP, ''' for (const remote of ["refs/remotes/working", "refs/remotes/origin"]) { const all = []; for (const project of projects) { const r = await git(["-C", join(workspace, project), "for-each-ref", "--format=%(committerdate:unix) %(objectname)", remote], env); if (r.ok) for (const l of r.out.split("\\n")) { const [d, oid] = l.split(" "); if (OID.test(oid ?? "")) all.push([Number(d), oid]); } } all.sort((x, y) => y[0] - x[0]); for (const [, oid] of all) if (have.size < HAVE_MAX) have.add(oid); }''') REVERSE = (GT, "const projects = projectsIn(workspace);", "const projects = projectsIn(workspace).reverse();") BASE_TEST = (T_GT, None, subprocess.run(["git", "-C", WT, "show", "HEAD:" + T_GT], capture_output=True, text=True, check=True).stdout) # (name, [(path, old, new)], [(test, pattern)], expect) with old None for a whole-file replacement. M = [ ("n01 the fork runs outside the try", [(HO, TRY, ''' broker = fork(BROKER, [], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); try { broker.send({ op: "boot", config: boot }); ready = await firstReply(broker, bootTimeoutMs, "broker");''')], [(T_HO, P_FORK)], "killed"), ("n02 the send runs outside the try", [(HO, TRY, ''' try { broker = fork(BROKER, [], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); } catch (e) { release(); throw e; } broker.send({ op: "boot", config: boot }); try { ready = await firstReply(broker, bootTimeoutMs, "broker");''')], [(T_HO, P_SEND)], "killed"), ("n03 the fork and send run outside the try, as at the base", [(HO, TRY, ''' broker = fork(BROKER, [], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); broker.send({ op: "boot", config: boot }); try { ready = await firstReply(broker, bootTimeoutMs, "broker");''')], [(T_HO, P_FORK), (T_HO, P_SEND)], "killed"), ("n04 a lock gone before the read refuses as unreadable", [(HO, ' if (e.code === "ENOENT") return null;\n', "")], [(T_HO, P_GONE)], "killed"), ("n05 any read failure counts as no lock", [(HO, 'if (e.code === "ENOENT") return null;', "return null;")], [(T_HO, P_GONE)], "killed"), ("n06 have takes the clones in reverse directory order", [REVERSE], [(T_GT, P_HAVE)], "killed"), ("n07 have sorts by date across clones", [DATE_SORT], [(T_GT, P_HAVE)], "killed"), ("x01 control: the base test against n06", [REVERSE, BASE_TEST], [(T_GT, P_HAVE)], "survived"), ("x02 control: the base test against n07", [DATE_SORT, BASE_TEST], [(T_GT, P_HAVE)], "survived"), ] def diffhash(): d = subprocess.run(["git", "-C", WT, "diff", "--", ".", ":!node_modules"], capture_output=True).stdout return hashlib.sha256(d).hexdigest() def run(test, pattern, log): pkg = os.path.join(WT, *test.split("/")[:2]) with open(log, "ab") as f: f.write(f"== {test} /{pattern}/\n".encode()); f.flush() p = subprocess.Popen(["node", "--test", f"--test-name-pattern={pattern}", os.path.relpath(os.path.join(WT, test), pkg)], cwd=pkg, env=ENV, stdout=f, stderr=subprocess.STDOUT, start_new_session=True) try: rc = p.wait(timeout=600) except subprocess.TimeoutExpired: os.killpg(p.pid, signal.SIGKILL); p.wait(); return "timeout" # A pattern that matched nothing passes with no tests; count that as a fault. tail = open(log, encoding="utf8", errors="replace").read().split(f"== {test} /{pattern}/\n")[-1] ran = sum(int(l.split()[2]) for l in tail.splitlines() if l.startswith(("ℹ pass ", "ℹ fail "))) return "no-tests" if ran == 0 else rc for _, edits, *_ in M: for path, *_ in edits: if os.path.exists(os.path.join(WT, path + ".orig")): sys.exit(f"STOP {path}.orig exists") start = diffhash() print(f"start diff {start[:16]} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True) bad = [] only = set(sys.argv[1:]) for name, edits, tests, expect in M: if only and name.split()[0] not in only: continue srcs = {} ok = True for path, old, new in edits: full = os.path.join(WT, path) src = srcs.get(path) or open(full).read() if old is not None and src.count(old) != 1: print(f"{name}: ANCHOR {path} {src.count(old)}", flush=True); ok = False; break srcs[path] = new if old is None else src.replace(old, new) if not ok: bad.append(name); continue for path in srcs: os.rename(os.path.join(WT, path), os.path.join(WT, path + ".orig")) try: for path, text in srcs.items(): open(os.path.join(WT, path), "w").write(text) log = os.path.join(OUT, name.split()[0] + ".log") open(log, "w").close() rcs = [run(t, p, log) for t, p in tests] finally: for path in srcs: os.replace(os.path.join(WT, path + ".orig"), os.path.join(WT, path)) if any(rc in ("timeout", "no-tests") for rc in rcs): got = "FAULT" elif all(rc != 0 for rc in rcs): got = "killed" elif all(rc == 0 for rc in rcs): got = "survived" else: got = "MIXED" print(f"{name}: {got} (expected {expect}) {list(zip([t.split('/')[-1] for t, _ in tests], rcs))}", flush=True) if got != expect: bad.append(name) end = diffhash() print(f"end diff {end[:16]} {'same' if end == start else 'CHANGED'} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True) print("MUTANTS_OK" if not bad and end == start else f"MUTANTS_BAD {bad}")