#!/usr/bin/env python3 # Row 110 (#1604) mutants. Each takes out or bends one new check; every named # (test file, name pattern) run must fail on it. b00 and s00 put BASE's # broker.mjs and server.mjs back under the new tests. Two mutants are expected # to survive (b44, b45); the comment at them says why. Each file is restored from its .orig after the run, # and the candidate (tracked diff plus the new files) must hash the same at # the end as at the start. import hashlib, os, signal, subprocess, sys, time WT = os.path.expanduser("~/darkwing-scratch/r110/wt") OUT = os.path.expanduser("~/darkwing-scratch/r110/mutants") ENV = dict(os.environ, TMPDIR=os.path.expanduser("~/darkwing-scratch/r110/tmp")) for k in ("MOSAIC_BUS_TESTER_NONCE", "MOSAIC_BUS_CLI_NONCE"): ENV.pop(k, None) os.makedirs(OUT, exist_ok=True) os.makedirs(ENV["TMPDIR"], exist_ok=True) BR, SV, CL, RT = (f"packages/bus/src/{f}.mjs" for f in ("broker", "server", "client", "runtime")) TS, TC = "packages/bus/src/tester.mjs", "packages/bus/src/tester-cli.mjs" CLI, TR = "packages/cli/src/cli.mjs", "packages/cli/src/transport.mjs" VARS, RUN = "packages/business/src/vars.mjs", "packages/harness/src/runner.mjs" BT, CT = "packages/bus/tests/tester.test.mjs", "packages/cli/tests/tester.test.mjs" BIND, ROLE, REFUSE = (BT, "^the tester binds only"), (BT, "^a business may not name"), (BT, "^every verb outside") SEND, REPLY, VIEWS = (BT, "^a tester message is from tester"), (BT, "^a role answers a tester"), (BT, "^the tester reads the human views") PROOF, CROSS, NONCE = (BT, "^tester proof binds"), (BT, "^neither CLI"), (BT, "^testerNonce reads") ENVL, SHIM = (BT, "^an envelope carries"), (BT, "^tester-cli.mjs binds through") TALK, PREFIX, READS = (CT, "^talk --tester sends"), (CT, "^talk --tester refuses"), (CT, "^the read verbs take") NOFLAG, TRANS, DEFAULT = (CT, "^decide, stop and launches"), (CT, "^the tester transport runs"), (CT, "^with no stand-in transport") TYPES, MERGE = ("packages/business/tests/vars.test.mjs", "^types"), ("packages/business/tests/vars.test.mjs", "^merge: defaults") TURN = ("packages/harness/tests/runner.test.mjs", "^turnRequest says a tester") def base(path): return (path, None, subprocess.run(["git", "-C", WT, "show", "HEAD:" + path], capture_output=True, text=True, check=True).stdout) ALLOWED = ['message.send', 'message.receive', 'message.read', 'inbox', 'agents', 'tasks', 'trail', 'decisions', 'messages', 'replies'] TV = "const TESTER_VERBS = [" + ", ".join(f"'{v}'" for v in ALLOWED) + "];" def verbs(vs): return "const TESTER_VERBS = [" + ", ".join(f"'{v}'" for v in vs) + "];" REFUSED = ['role.claim', 'role.release', 'role.revoke', 'decision.raise', 'decision.resolve', 'decision.seen', 'decision.withdraw', 'review.request', 'review.verdict', 'launch.revoke', 'launch.restore', 'event.emit', 'state', 'credentials', 'requirements'] NEEDS = {'message.send': [SEND], 'message.receive': [REPLY], 'message.read': [REPLY]} REPLY_BLOCK = """ // A role answers a tester only in reply to a tester's message. if ( a.to === 'tester' && !(id(a.in_reply_to) && this.#store.get("SELECT 1 FROM messages WHERE id=? AND business=? AND from_role='tester'", a.in_reply_to, s.business)) ) fail('tester-reply-required'); """ DEF_TT = "testerTransport({ socket: socketPath(system.dataRoot), business, env: io.env, spawn: deps.testerSpawn })" # (name, [(path, old, new)], [(test, pattern)], expect) with old None for a whole-file replacement. M = [ ("b00 BASE's broker.mjs under the new tests", [base(BR)], [BIND, REFUSE, SEND, REPLY, VIEWS, ENVL, SHIM], "killed"), ("b01 no tester verb gate", [(BR, " if (s.tester && !TESTER_VERBS.includes(request.verb)) fail('tester-refused');\n", "")], [REFUSE], "killed"), ] # These four call #agent first, and #agent refuses a tester with the same code. Each checks its # arguments before #agent, though, so REFUSE's malformed-argument calls kill them (the first run had # them survive). b47-b50 add each one with #agent's tester line removed too. AGENT_FIRST = ['role.release', 'decision.raise', 'event.emit', 'requirements'] AGENT_TESTER = (BR, " if (s.tester) fail('tester-refused');\n if (s.reader) fail('read-only');\n", " if (s.reader) fail('read-only');\n") for i, v in enumerate(REFUSED): M.append((f"b{2 + i:02d} the tester list lets {v} through", [(BR, TV, verbs(ALLOWED + [v]))], [REFUSE] + ([ROLE] if v == "role.claim" else []), "killed")) for i, v in enumerate(ALLOWED): M.append((f"b{17 + i:02d} the tester list drops {v}", [(BR, TV, verbs([x for x in ALLOWED if x != v]))], NEEDS.get(v, [VIEWS]), "killed")) M += [ ("b27 a tester's reads use its own session, not the reader view", [(BR, "s.tester && !request.verb.startsWith('message.') ? { ...s, reader: true } : s", "s")], [VIEWS], "killed"), ("b28 task verbs don't refuse a tester", [(BR, " if (s.tester) fail('tester-refused');\n if (s.reader) fail('read-only');\n", " if (s.reader) fail('read-only');\n")], [REFUSE], "killed"), ("b29 bindTester ignores the switch", [(BR, " if (this.#business(record.business).vars?.['bus.tester'] !== true) fail('tester-off');\n", "")], [BIND], "killed"), ("b30 bindTester takes any truthy switch", [(BR, "this.#business(record.business).vars?.['bus.tester'] !== true", "!this.#business(record.business).vars?.['bus.tester']")], [BIND], "killed"), ("b31 bindTester takes extra keys", [(BR, " keys(record, ['business'], ['business']);\n if (this.#business(record.business).vars", " if (this.#business(record.business).vars")], [BIND], "killed"), ("b32 a business may name a role tester", [(BR, "role === 'human' || role === 'tester'", "role === 'human'")], [ROLE], "killed"), ("b33 a tester message takes corrects and decision", [(BR, " keys(a, ['to', 'body', 'class', 'in_reply_to'], ['to', 'body']);\n", "")], [SEND], "killed"), ("b34 a tester may send to human or tester", [(BR, " if (!Object.hasOwn(b.roles, a.to)) fail('unknown-role');\n if (typeof a.body", " if (typeof a.body")], [SEND], "killed"), ("b35 no [mos test] prefix check", [(BR, " if (typeof a.body !== 'string' || !a.body.startsWith(TESTER_PREFIX)) fail('tester-prefix');\n", "")], [SEND], "killed"), ("b36 the prefix may sit anywhere", [(BR, "!a.body.startsWith(TESTER_PREFIX)", "!a.body.includes(TESTER_PREFIX)")], [SEND], "killed"), ("b37 a role may send to tester unasked", [(BR, REPLY_BLOCK, "")], [REPLY], "killed"), ("b38 a reply to any message reaches tester", [(BR, " AND from_role='tester'\", a.in_reply_to", "\", a.in_reply_to")], [REPLY], "killed"), ("b39 the human may send to tester", [(BR, " this.#human(s);\n if (a.to === 'tester') fail('unknown-role');\n", " this.#human(s);\n")], [REPLY], "killed"), ("b40 a tester message's from_run is null", [(BR, "s.human ? 'human-cli' : s.tester ? 'tester-cli' : s.run", "s.human ? 'human-cli' : s.run")], [SEND, TALK], "killed"), ("b41 receive refuses a tester", [(BR, " else if (!s.tester) this.#agent(s);\n const role", " else this.#agent(s);\n const role")], [REPLY], "killed"), ("b42 read refuses a tester", [(BR, " else if (!s.tester) this.#agent(s);\n const m =", " else this.#agent(s);\n const m =")], [REPLY], "killed"), ("b43 a tester's delivery says broker", [(BR, "s.human || s.tester ? 'cli' : 'broker'", "s.human ? 'cli' : 'broker'")], [REPLY], "killed"), # Equivalent: role_claims rows come only from role.claim (role = the session's role) and #claimEnd # (role = an existing holder's). The constructor refuses a business role named tester (b32), bindLaunch # refuses a role outside the business (b46), and the tester's own role.claim refuses (b02). So # #holder(business, 'tester') is always undefined and #delivery writes a null holder either way. ("b44 a tester's receive holder looks up role tester", [(BR, " s.human || s.tester ? null : this.#holder(s.business, s.role),\n", " s.human ? null : this.#holder(s.business, s.role),\n")], [REPLY], "survived"), ("b45 a tester's read holder looks up role tester", [(BR, "'read', s.human || s.tester ? null :", "'read', s.human ? null :")], [REPLY], "survived"), ("b46 a launch may name role tester", [(BR, " if (!Object.hasOwn(b.roles, record.role)) fail('unknown-role');\n identifier(record.run);", " if (!Object.hasOwn(b.roles, record.role) && record.role !== 'tester') fail('unknown-role');\n identifier(record.run);")], [ROLE], "killed"), *[(f"b{47 + i} the tester list lets {v} through and #agent lets a tester in", [(BR, TV, verbs(ALLOWED + [v])), AGENT_TESTER], [REFUSE], "killed") for i, v in enumerate(AGENT_FIRST)], ("s00 BASE's server.mjs under the new tests", [base(SV)], [ENVL, SHIM], "killed"), ("s01 the envelope counts only cap and human", [(SV, "[r.cap, r.human, r.tester].filter(Boolean).length !== 1", "Boolean(r.cap) === Boolean(r.human)")], [ENVL], "killed"), ("s02 the envelope has no tester key", [(SV, "['cap', 'human', 'tester', 'verb', 'args']", "['cap', 'human', 'verb', 'args']")], [ENVL], "killed"), ("s03 no tester-required without an authenticator", [(SV, " if (!authenticateTester) throw new BusError('tester-required');\n", "")], [ENVL], "killed"), ("s04 the tester's capability outlives its request", [(SV, " transient = cap = authenticateTester(r.tester);", " cap = authenticateTester(r.tester);")], [ENVL], "killed"), ("c01 the client never sends tester", [(CL, "this.tester ? { tester: this.tester } : { cap: this.cap }", "{ cap: this.cap }")], [ENVL, SHIM], "killed"), ("r01 the runtime serves no tester", [(RT, " authenticateTester: (proof) => {", " _authenticateTester: (proof) => {")], [SHIM], "killed"), ("r02 the runtime checks the human CLI's path", [(RT, "new URL('./tester-cli.mjs', import.meta.url)", "new URL('./human-cli.mjs', import.meta.url)")], [SHIM], "killed"), ("r03 the runtime gives the tester no launch records", [(RT, "cliPath: fileURLToPath(new URL('./tester-cli.mjs', import.meta.url)),\n launches: launchRecords,", "cliPath: fileURLToPath(new URL('./tester-cli.mjs', import.meta.url)),\n launches: [],")], [SHIM], "killed"), ("t01 no uid check", [(TS, " p.uid !== process.getuid() ||\n", "")], [PROOF], "killed"), ("t02 no start time check", [(TS, " p.startTime !== proof.startTime ||\n", "")], [PROOF], "killed"), ("t03 no nonce check", [(TS, " nonce !== proof.nonce ||\n", "")], [PROOF, CROSS], "killed"), ("t04 no CLI path check", [(TS, " p.argv[1] !== resolve(cliPath)\n", " false\n")], [PROOF, CROSS], "killed"), ("t05 MOSAIC_BUS_CAP is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']")], [PROOF, SHIM], "killed"), ("t06 MOSAIC_RUN_ID is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_BUS_CAP', 'MOSAIC_AGENT_RUN']")], [PROOF, SHIM], "killed"), ("t07 MOSAIC_AGENT_RUN is no run marker", [(TS, "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID', 'MOSAIC_AGENT_RUN']", "['MOSAIC_BUS_CAP', 'MOSAIC_RUN_ID']")], [PROOF, SHIM], "killed"), ("t08 no launch record check", [(TS, " ||\n launches.some((r) => r.pid === current.pid && r.startTime === current.startTime)\n", "\n")], [PROOF, SHIM], "killed"), ("t09 no loop check", [(TS, " if (seen.has(current.pid)) refuse();\n", "")], [PROOF], "killed"), ("t10 no ppid check", [(TS, " if (current.ppid < 2) refuse();\n", "")], [PROOF], "killed"), ("t11 no re-read of the CLI", [(TS, " if (again.startTime !== proof.startTime || again.ppid !== p.ppid) refuse();\n", "")], [PROOF], "killed"), ("t12 no proof keys check", [(TS, " Object.keys(proof).some((k) => !['pid', 'startTime', 'nonce', 'business'].includes(k)) ||\n", "")], [PROOF], "killed"), ("t13 a pid of 1 passes the shape check", [(TS, " proof.pid < 2 ||\n", "")], [PROOF], "killed"), ("t14 any nonce string passes the shape check", [(TS, " !/^[a-f0-9]{64}$/.test(proof.nonce)\n", " false\n")], [PROOF], "killed"), ("t15 testerNonce reads the human's variable", [(TS, "const NONCE = 'MOSAIC_BUS_TESTER_NONCE=';", "const NONCE = 'MOSAIC_BUS_CLI_NONCE=';")], [NONCE, SHIM], "killed"), ("k01 the shim proves with the human's variable", [(TC, " nonce: process.env.MOSAIC_BUS_TESTER_NONCE,", " nonce: process.env.MOSAIC_BUS_CLI_NONCE,")], [SHIM], "killed"), ("k02 the shim takes extra request keys", [(TC, " Object.keys(request).some((k) => !['business', 'verb', 'args'].includes(k))", " false")], [SHIM], "killed"), ("l01 --tester still refuses inside an agent", [(CLI, " if (!tester) refuseInsideAgent(io.env);\n", " refuseInsideAgent(io.env);\n")], [TALK, READS], "killed"), ("l02 no agent check without --tester", [(CLI, " if (!tester) refuseInsideAgent(io.env);\n", "")], [READS], "killed"), # Not TALK: on the human stand-in, talk waits its full 900 s for a reply that goes to the tester. ("l03 --tester uses the human transport", [(CLI, " if (tester) return deps.testerTransport ? deps.testerTransport(business) : " + DEF_TT + ";\n", "")], [READS, DEFAULT], "killed"), ("l04 the default --tester transport is the human's", [(CLI, ": " + DEF_TT, ": human" + DEF_TT[len("tester"):].replace("Transport", "Transport", 1))], [DEFAULT], "killed"), ("l05 talk --tester has no prefix check", [(CLI, " if (!receiveOnly && parsed.flags.has(\"--tester\") && !body.startsWith(TESTER_PREFIX)) throw new CliError(`a tester message starts with \"${TESTER_PREFIX}\"`, 2);\n", "")], [PREFIX], "killed"), ("l06 talk takes no --tester", [(CLI, 'flags: ["--follow", "--tester"]', 'flags: ["--follow"]')], [TALK, PREFIX], "killed"), ("l07 the read verbs take no --tester", [(CLI, '{ flags: ["--json", "--tester"], values: ["--business"] }', '{ flags: ["--json"], values: ["--business"] }')], [READS], "killed"), ("l08 decide takes --tester", [(CLI, '{ flags: ["--yes"], values: ["--business", "--note"] }', '{ flags: ["--yes", "--tester"], values: ["--business", "--note"] }')], [NOFLAG], "killed"), # The default-transport test compares argv with the imported TESTER_CLI, so it follows this mutant; TRANS pins the path. ("x01 TESTER_CLI is the human CLI", [(TR, 'new URL("../../bus/src/tester-cli.mjs", import.meta.url)', 'new URL("../../bus/src/human-cli.mjs", import.meta.url)')], [TRANS], "killed"), ("x02 tester-required has no exit code", [(TR, ' "tester-required": 3,\n', "")], [TRANS], "killed"), ("x03 tester-off has no exit code", [(TR, ' "tester-off": 3,\n', "")], [TRANS], "killed"), ("x04 tester-refused has no exit code", [(TR, ' "tester-refused": 3,\n', "")], [TRANS], "killed"), ("x05 testerTransport runs the default CLI", [(TR, "humanTransport({ cli: TESTER_CLI, ...options })", "humanTransport({ ...options })")], [TRANS, DEFAULT], "killed"), ("v01 bus.tester at any layer", [(VARS, '"bus.tester": def(["business"], boolean,', '"bus.tester": def(["system", "business", "project", "agent"], boolean,')], [TYPES], "killed"), ("v02 bus.tester is not a var", [(VARS, ' "bus.tester": def(["business"], boolean, { default: false }),\n', "")], [TYPES, MERGE], "killed"), ("v03 bus.tester takes any string", [(VARS, '"bus.tester": def(["business"], boolean,', '"bus.tester": def(["business"], string(),')], [TYPES], "killed"), ("v04 bus.tester defaults on", [(VARS, '"bus.tester": def(["business"], boolean, { default: false })', '"bus.tester": def(["business"], boolean, { default: true })')], [MERGE], "killed"), ("h01 no tester line", [(RUN, 'm.from_role === "tester"', "false")], [TURN], "killed"), ("h02 the tester line on every message", [(RUN, 'm.from_role === "tester"', "true")], [TURN], "killed"), ] def candidate_hash(): h = hashlib.sha256(subprocess.run(["git", "-C", WT, "diff", "--", ".", ":!node_modules"], capture_output=True).stdout) new = subprocess.run(["git", "-C", WT, "ls-files", "--others", "--exclude-standard"], capture_output=True, text=True).stdout.split() for f in sorted(x for x in new if x != "node_modules"): h.update(f.encode()); h.update(open(os.path.join(WT, f), "rb").read()) return h.hexdigest() def run(test, pattern, log): pkg = os.path.join(WT, *test.split("/")[:2]) with open(log, "ab") as f: f.write(f"== {test} /{pattern}/\n".encode()); f.flush() p = subprocess.Popen(["node", "--test", f"--test-name-pattern={pattern}", os.path.relpath(os.path.join(WT, test), pkg)], cwd=pkg, env=ENV, stdout=f, stderr=subprocess.STDOUT, start_new_session=True) try: rc = p.wait(timeout=300) except subprocess.TimeoutExpired: os.killpg(p.pid, signal.SIGKILL); p.wait(); return "timeout" # A pattern that matched nothing passes with no tests; count that as a fault. tail = open(log, encoding="utf8", errors="replace").read().split(f"== {test} /{pattern}/\n")[-1] ran = sum(int(l.split()[2]) for l in tail.splitlines() if l.startswith(("ℹ pass ", "ℹ fail ", "ℹ cancelled "))) return "no-tests" if ran == 0 else rc for _, edits, *_ in M: for path, *_ in edits: if os.path.exists(os.path.join(WT, path + ".orig")): sys.exit(f"STOP {path}.orig exists") start = candidate_hash() print(f"start candidate {start[:16]} {time.strftime('%H:%M:%S', time.gmtime())} mutants {len(M)}", flush=True) bad = [] only = set(sys.argv[1:]) for name, edits, tests, expect in M: if only and name.split()[0] not in only: continue srcs = {} ok = True for path, old, new in edits: full = os.path.join(WT, path) src = srcs.get(path) or open(full).read() if old is not None and src.count(old) != 1: print(f"{name}: ANCHOR {path} {src.count(old)}", flush=True); ok = False; break srcs[path] = new if old is None else src.replace(old, new) if not ok: bad.append(name); continue for path in srcs: os.rename(os.path.join(WT, path), os.path.join(WT, path + ".orig")) try: for path, text in srcs.items(): open(os.path.join(WT, path), "w").write(text) log = os.path.join(OUT, name.split()[0] + ".log") open(log, "w").close() rcs = [run(t, p, log) for t, p in tests] finally: for path in srcs: os.replace(os.path.join(WT, path + ".orig"), os.path.join(WT, path)) if any(rc in ("timeout", "no-tests") for rc in rcs): got = "FAULT" elif all(rc != 0 for rc in rcs): got = "killed" elif all(rc == 0 for rc in rcs): got = "survived" else: got = "MIXED" print(f"{name}: {got} (expected {expect}) {list(zip([t.split('/')[-1] + ' ' + p for t, p in tests], rcs))}", flush=True) if got != expect: bad.append(name) end = candidate_hash() print(f"end candidate {end[:16]} {'same' if end == start else 'CHANGED'} {time.strftime('%H:%M:%S', time.gmtime())}", flush=True) print("MUTANTS_OK" if not bad and end == start else f"MUTANTS_BAD {bad}")