jason.woltje
4ff4f44c98
Unclaimed files left in the canonical checkout. Inventory: canonical-cleanup-inventory.md (commit-as-evidence groups). detect-secrets scan clean on 2026-11-02: hits are sha256 pins, git object ids, test sentinels already in tracked tests and unit-file lines.
126 lines
4.5 KiB
Python
126 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Row 228 mutant: remove "ZAI_API_KEY" from ENV_ALLOW and confirm the new test kills it.
|
|
|
|
Usage: python3 -I mutants.py <worktree-root> <out-dir>
|
|
|
|
Runs packages/seat/tests/session.test.mjs on the unmutated source (baseline), then
|
|
once on the mutant. The mutant counts as killed only when the run exits nonzero,
|
|
did not time out, and the new test is reported as failing. A timeout, a missing
|
|
match, or a passing run counts as NOT killed.
|
|
|
|
The source is restored in a finally block. Its sha256 after the restore must equal
|
|
the sha256 taken before the mutant was written, which is the value in
|
|
candidate-manifest.sha256 next to this script. The script refuses to start if the
|
|
source already differs from that manifest line.
|
|
|
|
Each node run starts in its own process group. Only that group is signalled.
|
|
The log goes to <out-dir>/mutants.log.
|
|
"""
|
|
import hashlib
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = pathlib.Path(sys.argv[1]).resolve()
|
|
OUT = pathlib.Path(sys.argv[2]).resolve()
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
SRC = ROOT / "packages/seat/src/session.mjs"
|
|
TEST = "packages/seat/tests/session.test.mjs"
|
|
TMP = "/mnt/storage/scratch/researcher/tm"
|
|
TIMEOUT_S = 120
|
|
NEW_TEST = "ZAI_API_KEY passes the session allowlist, and ZAI_TOKEN does not"
|
|
OLD = '"OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",'
|
|
NEW = '"OPENAI_API_KEY", "OPENROUTER_API_KEY",'
|
|
ENV = dict(os.environ, TMPDIR=TMP)
|
|
|
|
|
|
def sha256(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def manifest_sha_for_source() -> str:
|
|
for line in (HERE / "candidate-manifest.sha256").read_text().splitlines():
|
|
digest, _, path = line.partition(" ")
|
|
if path == "packages/seat/src/session.mjs":
|
|
return digest
|
|
sys.exit("refuse: candidate-manifest.sha256 has no session.mjs line")
|
|
|
|
|
|
def run_test(label, log):
|
|
proc = subprocess.Popen(
|
|
["node", "--test", TEST],
|
|
cwd=ROOT,
|
|
env=ENV,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
start_new_session=True,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
out, _ = proc.communicate(timeout=TIMEOUT_S)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
out, _ = proc.communicate()
|
|
finally:
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
text = out.decode("utf-8", "replace")
|
|
log.write(f"===== {label} exit={proc.returncode} timed_out={timed_out}\n{text}\n")
|
|
return proc.returncode, timed_out, text
|
|
|
|
|
|
def names_failing(text: str, name: str) -> bool:
|
|
for line in text.splitlines():
|
|
if ("not ok" in line or line.lstrip().startswith("✖")) and name in line:
|
|
return True
|
|
return False
|
|
|
|
|
|
def main() -> int:
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
pathlib.Path(TMP).mkdir(parents=True, exist_ok=True)
|
|
original = SRC.read_bytes()
|
|
before = sha256(original)
|
|
if before != manifest_sha_for_source():
|
|
sys.exit("refuse: packages/seat/src/session.mjs differs from candidate-manifest.sha256")
|
|
text = original.decode("utf-8")
|
|
if text.count(OLD) != 1:
|
|
sys.exit(f"refuse: the ENV_ALLOW line matches {text.count(OLD)} times, expected 1")
|
|
mutant = text.replace(OLD, NEW, 1).encode("utf-8")
|
|
|
|
verdict = "NOT KILLED"
|
|
restored = False
|
|
with (OUT / "mutants.log").open("w") as log:
|
|
try:
|
|
rc0, to0, _ = run_test("baseline (unmutated)", log)
|
|
log.write(f"baseline: exit={rc0} timed_out={to0}\n")
|
|
if rc0 != 0 or to0:
|
|
verdict = "BASELINE FAILED, mutant not run"
|
|
else:
|
|
SRC.write_bytes(mutant)
|
|
rc1, to1, t1 = run_test("M1-remove-ZAI_API_KEY", log)
|
|
named = names_failing(t1, NEW_TEST)
|
|
killed = rc1 != 0 and not to1 and named
|
|
verdict = "KILLED" if killed else "NOT KILLED"
|
|
log.write(
|
|
f"M1-remove-ZAI_API_KEY: exit={rc1} timed_out={to1} "
|
|
f"new_test_reported_failing={named} verdict={verdict}\n"
|
|
)
|
|
finally:
|
|
SRC.write_bytes(original)
|
|
after = sha256(SRC.read_bytes())
|
|
restored = after == before
|
|
log.write(f"restore: sha256 {'matches' if restored else 'MISMATCH'} before={before} after={after}\n")
|
|
print(f"verdict: {verdict}")
|
|
print(f"restore ok: {restored}")
|
|
return 0 if verdict == "KILLED" and restored else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|