jason.woltjeandClaude Opus 5.5
0bfc166542
Build notes, patch and candidate manifest on BASE badfa52b9, the gate
logs (GATE_PASSED; the real bwrap tester case in seat sandbox.test.mjs
ran and passed) and the mutant runs (94 killed, b44 and b45 equivalent).
Co-Authored-By: Claude Opus 5.5 <[email protected]>
197 lines
19 KiB
Python
197 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
# Row 110 (#1604) mutants. Each takes out or bends one new check; every named
|
||
# (test file, name pattern) run must fail on it. b00 and s00 put BASE's
|
||
# broker.mjs and server.mjs back under the new tests. Two mutants are expected
|
||
# to survive (b44, b45); the comment at them says why. Each file is restored from its .orig after the run,
|
||
# and the candidate (tracked diff plus the new files) must hash the same at
|
||
# the end as at the start.
|
||
import hashlib, os, signal, subprocess, sys, time
|
||
|
||
WT = os.path.expanduser("~/darkwing-scratch/r110/wt")
|
||
OUT = os.path.expanduser("~/darkwing-scratch/r110/mutants")
|
||
ENV = dict(os.environ, TMPDIR=os.path.expanduser("~/darkwing-scratch/r110/tmp"))
|
||
for k in ("MOSAIC_BUS_TESTER_NONCE", "MOSAIC_BUS_CLI_NONCE"): ENV.pop(k, None)
|
||
os.makedirs(OUT, exist_ok=True)
|
||
os.makedirs(ENV["TMPDIR"], exist_ok=True)
|
||
|
||
BR, SV, CL, RT = (f"packages/bus/src/{f}.mjs" for f in ("broker", "server", "client", "runtime"))
|
||
TS, TC = "packages/bus/src/tester.mjs", "packages/bus/src/tester-cli.mjs"
|
||
CLI, TR = "packages/cli/src/cli.mjs", "packages/cli/src/transport.mjs"
|
||
VARS, RUN = "packages/business/src/vars.mjs", "packages/harness/src/runner.mjs"
|
||
BT, CT = "packages/bus/tests/tester.test.mjs", "packages/cli/tests/tester.test.mjs"
|
||
BIND, ROLE, REFUSE = (BT, "^the tester binds only"), (BT, "^a business may not name"), (BT, "^every verb outside")
|
||
SEND, REPLY, VIEWS = (BT, "^a tester message is from tester"), (BT, "^a role answers a tester"), (BT, "^the tester reads the human views")
|
||
PROOF, CROSS, NONCE = (BT, "^tester proof binds"), (BT, "^neither CLI"), (BT, "^testerNonce reads")
|
||
ENVL, SHIM = (BT, "^an envelope carries"), (BT, "^tester-cli.mjs binds through")
|
||
TALK, PREFIX, READS = (CT, "^talk --tester sends"), (CT, "^talk --tester refuses"), (CT, "^the read verbs take")
|
||
NOFLAG, TRANS, DEFAULT = (CT, "^decide, stop and launches"), (CT, "^the tester transport runs"), (CT, "^with no stand-in transport")
|
||
TYPES, MERGE = ("packages/business/tests/vars.test.mjs", "^types"), ("packages/business/tests/vars.test.mjs", "^merge: defaults")
|
||
TURN = ("packages/harness/tests/runner.test.mjs", "^turnRequest says a tester")
|
||
|
||
def base(path):
|
||
return (path, None, subprocess.run(["git", "-C", WT, "show", "HEAD:" + path], capture_output=True, text=True, check=True).stdout)
|
||
|
||
ALLOWED = ['message.send', 'message.receive', 'message.read', 'inbox', 'agents', 'tasks', 'trail', 'decisions', 'messages', 'replies']
|
||
TV = "const TESTER_VERBS = [" + ", ".join(f"'{v}'" for v in ALLOWED) + "];"
|
||
def verbs(vs): return "const TESTER_VERBS = [" + ", ".join(f"'{v}'" for v in vs) + "];"
|
||
REFUSED = ['role.claim', 'role.release', 'role.revoke', 'decision.raise', 'decision.resolve', 'decision.seen', 'decision.withdraw',
|
||
'review.request', 'review.verdict', 'launch.revoke', 'launch.restore', 'event.emit', 'state', 'credentials', 'requirements']
|
||
NEEDS = {'message.send': [SEND], 'message.receive': [REPLY], 'message.read': [REPLY]}
|
||
REPLY_BLOCK = """ // A role answers a tester only in reply to a tester's message.
|
||
if (
|
||
a.to === 'tester' &&
|
||
!(id(a.in_reply_to) && this.#store.get("SELECT 1 FROM messages WHERE id=? AND business=? AND from_role='tester'", a.in_reply_to, s.business))
|
||
)
|
||
fail('tester-reply-required');
|
||
"""
|
||
DEF_TT = "testerTransport({ socket: socketPath(system.dataRoot), business, env: io.env, spawn: deps.testerSpawn })"
|
||
|
||
# (name, [(path, old, new)], [(test, pattern)], expect) with old None for a whole-file replacement.
|
||
M = [
|
||
("b00 BASE's broker.mjs under the new tests", [base(BR)], [BIND, REFUSE, SEND, REPLY, VIEWS, ENVL, SHIM], "killed"),
|
||
("b01 no tester verb gate", [(BR, " if (s.tester && !TESTER_VERBS.includes(request.verb)) fail('tester-refused');\n", "")], [REFUSE], "killed"),
|
||
]
|
||
# These four call #agent first, and #agent refuses a tester with the same code. Each checks its
|
||
# arguments before #agent, though, so REFUSE's malformed-argument calls kill them (the first run had
|
||
# them survive). b47-b50 add each one with #agent's tester line removed too.
|
||
AGENT_FIRST = ['role.release', 'decision.raise', 'event.emit', 'requirements']
|
||
AGENT_TESTER = (BR, " if (s.tester) fail('tester-refused');\n if (s.reader) fail('read-only');\n", " if (s.reader) fail('read-only');\n")
|
||
for i, v in enumerate(REFUSED):
|
||
M.append((f"b{2 + i:02d} the tester list lets {v} through", [(BR, TV, verbs(ALLOWED + [v]))], [REFUSE] + ([ROLE] if v == "role.claim" else []), "killed"))
|
||
for i, v in enumerate(ALLOWED):
|
||
M.append((f"b{17 + i:02d} the tester list drops {v}", [(BR, TV, verbs([x for x in ALLOWED if x != v]))], NEEDS.get(v, [VIEWS]), "killed"))
|
||
M += [
|
||
("b27 a tester's reads use its own session, not the reader view", [(BR, "s.tester && !request.verb.startsWith('message.') ? { ...s, reader: true } : s", "s")], [VIEWS], "killed"),
|
||
("b28 task verbs don't refuse a tester", [(BR, " if (s.tester) fail('tester-refused');\n if (s.reader) fail('read-only');\n", " if (s.reader) fail('read-only');\n")], [REFUSE], "killed"),
|
||
("b29 bindTester ignores the switch", [(BR, " if (this.#business(record.business).vars?.['bus.tester'] !== true) fail('tester-off');\n", "")], [BIND], "killed"),
|
||
("b30 bindTester takes any truthy switch", [(BR, "this.#business(record.business).vars?.['bus.tester'] !== true", "!this.#business(record.business).vars?.['bus.tester']")], [BIND], "killed"),
|
||
("b31 bindTester takes extra keys", [(BR, " keys(record, ['business'], ['business']);\n if (this.#business(record.business).vars", " if (this.#business(record.business).vars")], [BIND], "killed"),
|
||
("b32 a business may name a role tester", [(BR, "role === 'human' || role === 'tester'", "role === 'human'")], [ROLE], "killed"),
|
||
("b33 a tester message takes corrects and decision", [(BR, " keys(a, ['to', 'body', 'class', 'in_reply_to'], ['to', 'body']);\n", "")], [SEND], "killed"),
|
||
("b34 a tester may send to human or tester", [(BR, " if (!Object.hasOwn(b.roles, a.to)) fail('unknown-role');\n if (typeof a.body", " if (typeof a.body")], [SEND], "killed"),
|
||
("b35 no [mos test] prefix check", [(BR, " if (typeof a.body !== 'string' || !a.body.startsWith(TESTER_PREFIX)) fail('tester-prefix');\n", "")], [SEND], "killed"),
|
||
("b36 the prefix may sit anywhere", [(BR, "!a.body.startsWith(TESTER_PREFIX)", "!a.body.includes(TESTER_PREFIX)")], [SEND], "killed"),
|
||
("b37 a role may send to tester unasked", [(BR, REPLY_BLOCK, "")], [REPLY], "killed"),
|
||
("b38 a reply to any message reaches tester", [(BR, " AND from_role='tester'\", a.in_reply_to", "\", a.in_reply_to")], [REPLY], "killed"),
|
||
("b39 the human may send to tester", [(BR, " this.#human(s);\n if (a.to === 'tester') fail('unknown-role');\n", " this.#human(s);\n")], [REPLY], "killed"),
|
||
("b40 a tester message's from_run is null", [(BR, "s.human ? 'human-cli' : s.tester ? 'tester-cli' : s.run", "s.human ? 'human-cli' : s.run")], [SEND, TALK], "killed"),
|
||
("b41 receive refuses a tester", [(BR, " else if (!s.tester) this.#agent(s);\n const role", " else this.#agent(s);\n const role")], [REPLY], "killed"),
|
||
("b42 read refuses a tester", [(BR, " else if (!s.tester) this.#agent(s);\n const m =", " else this.#agent(s);\n const m =")], [REPLY], "killed"),
|
||
("b43 a tester's delivery says broker", [(BR, "s.human || s.tester ? 'cli' : 'broker'", "s.human ? 'cli' : 'broker'")], [REPLY], "killed"),
|
||
# Equivalent: role_claims rows come only from role.claim (role = the session's role) and #claimEnd
|
||
# (role = an existing holder's). The constructor refuses a business role named tester (b32), bindLaunch
|
||
# refuses a role outside the business (b46), and the tester's own role.claim refuses (b02). So
|
||
# #holder(business, 'tester') is always undefined and #delivery writes a null holder either way.
|
||
("b44 a tester's receive holder looks up role tester", [(BR, " s.human || s.tester ? null : this.#holder(s.business, s.role),\n", " s.human ? null : this.#holder(s.business, s.role),\n")], [REPLY], "survived"),
|
||
("b45 a tester's read holder looks up role tester", [(BR, "'read', s.human || s.tester ? null :", "'read', s.human ? null :")], [REPLY], "survived"),
|
||
("b46 a launch may name role tester", [(BR, " if (!Object.hasOwn(b.roles, record.role)) fail('unknown-role');\n identifier(record.run);", " if (!Object.hasOwn(b.roles, record.role) && record.role !== 'tester') fail('unknown-role');\n identifier(record.run);")], [ROLE], "killed"),
|
||
*[(f"b{47 + i} the tester list lets {v} through and #agent lets a tester in", [(BR, TV, verbs(ALLOWED + [v])), AGENT_TESTER], [REFUSE], "killed") for i, v in enumerate(AGENT_FIRST)],
|
||
("s00 BASE's server.mjs under the new tests", [base(SV)], [ENVL, SHIM], "killed"),
|
||
("s01 the envelope counts only cap and human", [(SV, "[r.cap, r.human, r.tester].filter(Boolean).length !== 1", "Boolean(r.cap) === Boolean(r.human)")], [ENVL], "killed"),
|
||
("s02 the envelope has no tester key", [(SV, "['cap', 'human', 'tester', 'verb', 'args']", "['cap', 'human', 'verb', 'args']")], [ENVL], "killed"),
|
||
("s03 no tester-required without an authenticator", [(SV, " if (!authenticateTester) throw new BusError('tester-required');\n", "")], [ENVL], "killed"),
|
||
("s04 the tester's capability outlives its request", [(SV, " transient = cap = authenticateTester(r.tester);", " cap = authenticateTester(r.tester);")], [ENVL], "killed"),
|
||
("c01 the client never sends tester", [(CL, "this.tester ? { tester: this.tester } : { cap: this.cap }", "{ cap: this.cap }")], [ENVL, SHIM], "killed"),
|
||
("r01 the runtime serves no tester", [(RT, " authenticateTester: (proof) => {", " _authenticateTester: (proof) => {")], [SHIM], "killed"),
|
||
("r02 the runtime checks the human CLI's path", [(RT, "new URL('./tester-cli.mjs', import.meta.url)", "new URL('./human-cli.mjs', import.meta.url)")], [SHIM], "killed"),
|
||
("r03 the runtime gives the tester no launch records", [(RT, "cliPath: fileURLToPath(new URL('./tester-cli.mjs', import.meta.url)),\n launches: launchRecords,", "cliPath: fileURLToPath(new URL('./tester-cli.mjs', import.meta.url)),\n launches: [],")], [SHIM], "killed"),
|
||
("t01 no uid check", [(TS, " p.uid !== process.getuid() ||\n", "")], [PROOF], "killed"),
|
||
("t02 no start time check", [(TS, " p.startTime !== proof.startTime ||\n", "")], [PROOF], "killed"),
|
||
("t03 no nonce check", [(TS, " nonce !== proof.nonce ||\n", "")], [PROOF, CROSS], "killed"),
|
||
("t04 no CLI path check", [(TS, " p.argv[1] !== resolve(cliPath)\n", " false\n")], [PROOF, CROSS], "killed"),
|
||
("t05 MOSAIC_BUS_CAP is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']")], [PROOF, SHIM], "killed"),
|
||
("t06 MOSAIC_RUN_ID is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_BUS_CAP', 'MOSAIC_AGENT_RUN']")], [PROOF, SHIM], "killed"),
|
||
("t07 MOSAIC_AGENT_RUN is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID']")], [PROOF, SHIM], "killed"),
|
||
("t08 no launch record check", [(TS, " ||\n launches.some((r) => r.pid === current.pid && r.startTime === current.startTime)\n", "\n")], [PROOF, SHIM], "killed"),
|
||
("t09 no loop check", [(TS, " if (seen.has(current.pid)) refuse();\n", "")], [PROOF], "killed"),
|
||
("t10 no ppid check", [(TS, " if (current.ppid < 2) refuse();\n", "")], [PROOF], "killed"),
|
||
("t11 no re-read of the CLI", [(TS, " if (again.startTime !== proof.startTime || again.ppid !== p.ppid) refuse();\n", "")], [PROOF], "killed"),
|
||
("t12 no proof keys check", [(TS, " Object.keys(proof).some((k) => !['pid', 'startTime', 'nonce', 'business'].includes(k)) ||\n", "")], [PROOF], "killed"),
|
||
("t13 a pid of 1 passes the shape check", [(TS, " proof.pid < 2 ||\n", "")], [PROOF], "killed"),
|
||
("t14 any nonce string passes the shape check", [(TS, " !/^[a-f0-9]{64}$/.test(proof.nonce)\n", " false\n")], [PROOF], "killed"),
|
||
("t15 testerNonce reads the human's variable", [(TS, "const NONCE = 'MOSAIC_BUS_TESTER_NONCE=';", "const NONCE = 'MOSAIC_BUS_CLI_NONCE=';")], [NONCE, SHIM], "killed"),
|
||
("k01 the shim proves with the human's variable", [(TC, " nonce: process.env.MOSAIC_BUS_TESTER_NONCE,", " nonce: process.env.MOSAIC_BUS_CLI_NONCE,")], [SHIM], "killed"),
|
||
("k02 the shim takes extra request keys", [(TC, " Object.keys(request).some((k) => !['business', 'verb', 'args'].includes(k))", " false")], [SHIM], "killed"),
|
||
("l01 --tester still refuses inside an agent", [(CLI, " if (!tester) refuseInsideAgent(io.env);\n", " refuseInsideAgent(io.env);\n")], [TALK, READS], "killed"),
|
||
("l02 no agent check without --tester", [(CLI, " if (!tester) refuseInsideAgent(io.env);\n", "")], [READS], "killed"),
|
||
# Not TALK: on the human stand-in, talk waits its full 900 s for a reply that goes to the tester.
|
||
("l03 --tester uses the human transport", [(CLI, " if (tester) return deps.testerTransport ? deps.testerTransport(business) : " + DEF_TT + ";\n", "")], [READS, DEFAULT], "killed"),
|
||
("l04 the default --tester transport is the human's", [(CLI, ": " + DEF_TT, ": human" + DEF_TT[len("tester"):].replace("Transport", "Transport", 1))], [DEFAULT], "killed"),
|
||
("l05 talk --tester has no prefix check", [(CLI, " if (!receiveOnly && parsed.flags.has(\"--tester\") && !body.startsWith(TESTER_PREFIX)) throw new CliError(`a tester message starts with \"${TESTER_PREFIX}\"`, 2);\n", "")], [PREFIX], "killed"),
|
||
("l06 talk takes no --tester", [(CLI, 'flags: ["--follow", "--tester"]', 'flags: ["--follow"]')], [TALK, PREFIX], "killed"),
|
||
("l07 the read verbs take no --tester", [(CLI, '{ flags: ["--json", "--tester"], values: ["--business"] }', '{ flags: ["--json"], values: ["--business"] }')], [READS], "killed"),
|
||
("l08 decide takes --tester", [(CLI, '{ flags: ["--yes"], values: ["--business", "--note"] }', '{ flags: ["--yes", "--tester"], values: ["--business", "--note"] }')], [NOFLAG], "killed"),
|
||
# The default-transport test compares argv with the imported TESTER_CLI, so it follows this mutant; TRANS pins the path.
|
||
("x01 TESTER_CLI is the human CLI", [(TR, 'new URL("../../bus/src/tester-cli.mjs", import.meta.url)', 'new URL("../../bus/src/human-cli.mjs", import.meta.url)')], [TRANS], "killed"),
|
||
("x02 tester-required has no exit code", [(TR, ' "tester-required": 3,\n', "")], [TRANS], "killed"),
|
||
("x03 tester-off has no exit code", [(TR, ' "tester-off": 3,\n', "")], [TRANS], "killed"),
|
||
("x04 tester-refused has no exit code", [(TR, ' "tester-refused": 3,\n', "")], [TRANS], "killed"),
|
||
("x05 testerTransport runs the default CLI", [(TR, "humanTransport({ cli: TESTER_CLI, ...options })", "humanTransport({ ...options })")], [TRANS, DEFAULT], "killed"),
|
||
("v01 bus.tester at any layer", [(VARS, '"bus.tester": def(["business"], boolean,', '"bus.tester": def(["system", "business", "project", "agent"], boolean,')], [TYPES], "killed"),
|
||
("v02 bus.tester is not a var", [(VARS, ' "bus.tester": def(["business"], boolean, { default: false }),\n', "")], [TYPES, MERGE], "killed"),
|
||
("v03 bus.tester takes any string", [(VARS, '"bus.tester": def(["business"], boolean,', '"bus.tester": def(["business"], string(),')], [TYPES], "killed"),
|
||
("v04 bus.tester defaults on", [(VARS, '"bus.tester": def(["business"], boolean, { default: false })', '"bus.tester": def(["business"], boolean, { default: true })')], [MERGE], "killed"),
|
||
("h01 no tester line", [(RUN, 'm.from_role === "tester"', "false")], [TURN], "killed"),
|
||
("h02 the tester line on every message", [(RUN, 'm.from_role === "tester"', "true")], [TURN], "killed"),
|
||
]
|
||
|
||
def candidate_hash():
|
||
h = hashlib.sha256(subprocess.run(["git", "-C", WT, "diff", "--", ".", ":!node_modules"], capture_output=True).stdout)
|
||
new = subprocess.run(["git", "-C", WT, "ls-files", "--others", "--exclude-standard"], capture_output=True, text=True).stdout.split()
|
||
for f in sorted(x for x in new if x != "node_modules"):
|
||
h.update(f.encode()); h.update(open(os.path.join(WT, f), "rb").read())
|
||
return h.hexdigest()
|
||
|
||
def run(test, pattern, log):
|
||
pkg = os.path.join(WT, *test.split("/")[:2])
|
||
with open(log, "ab") as f:
|
||
f.write(f"== {test} /{pattern}/\n".encode()); f.flush()
|
||
p = subprocess.Popen(["node", "--test", f"--test-name-pattern={pattern}", os.path.relpath(os.path.join(WT, test), pkg)], cwd=pkg, env=ENV,
|
||
stdout=f, stderr=subprocess.STDOUT, start_new_session=True)
|
||
try:
|
||
rc = p.wait(timeout=300)
|
||
except subprocess.TimeoutExpired:
|
||
os.killpg(p.pid, signal.SIGKILL); p.wait(); return "timeout"
|
||
# A pattern that matched nothing passes with no tests; count that as a fault.
|
||
tail = open(log, encoding="utf8", errors="replace").read().split(f"== {test} /{pattern}/\n")[-1]
|
||
ran = sum(int(l.split()[2]) for l in tail.splitlines() if l.startswith(("ℹ pass ", "ℹ fail ", "ℹ cancelled ")))
|
||
return "no-tests" if ran == 0 else rc
|
||
|
||
for _, edits, *_ in M:
|
||
for path, *_ in edits:
|
||
if os.path.exists(os.path.join(WT, path + ".orig")):
|
||
sys.exit(f"STOP {path}.orig exists")
|
||
start = candidate_hash()
|
||
print(f"start candidate {start[:16]} {time.strftime('%H:%M:%S', time.gmtime())} mutants {len(M)}", flush=True)
|
||
bad = []
|
||
only = set(sys.argv[1:])
|
||
for name, edits, tests, expect in M:
|
||
if only and name.split()[0] not in only: continue
|
||
srcs = {}
|
||
ok = True
|
||
for path, old, new in edits:
|
||
full = os.path.join(WT, path)
|
||
src = srcs.get(path) or open(full).read()
|
||
if old is not None and src.count(old) != 1:
|
||
print(f"{name}: ANCHOR {path} {src.count(old)}", flush=True); ok = False; break
|
||
srcs[path] = new if old is None else src.replace(old, new)
|
||
if not ok:
|
||
bad.append(name); continue
|
||
for path in srcs: os.rename(os.path.join(WT, path), os.path.join(WT, path + ".orig"))
|
||
try:
|
||
for path, text in srcs.items(): open(os.path.join(WT, path), "w").write(text)
|
||
log = os.path.join(OUT, name.split()[0] + ".log")
|
||
open(log, "w").close()
|
||
rcs = [run(t, p, log) for t, p in tests]
|
||
finally:
|
||
for path in srcs: os.replace(os.path.join(WT, path + ".orig"), os.path.join(WT, path))
|
||
if any(rc in ("timeout", "no-tests") for rc in rcs): got = "FAULT"
|
||
elif all(rc != 0 for rc in rcs): got = "killed"
|
||
elif all(rc == 0 for rc in rcs): got = "survived"
|
||
else: got = "MIXED"
|
||
print(f"{name}: {got} (expected {expect}) {list(zip([t.split('/')[-1] + ' ' + p for t, p in tests], rcs))}", flush=True)
|
||
if got != expect: bad.append(name)
|
||
end = candidate_hash()
|
||
print(f"end candidate {end[:16]} {'same' if end == start else 'CHANGED'} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True)
|
||
print("MUTANTS_OK" if not bad and end == start else f"MUTANTS_BAD {bad}")
|