89 lines
7.8 KiB
Python
89 lines
7.8 KiB
Python
# Row 90 mutants (Dewey): each swaps one exact string in the scratch copy of the candidate, runs
|
|
# the tasks, bus or harness node tests, restores the file, and reports killed (tests fail) or SURVIVED.
|
|
import subprocess, sys, os, signal
|
|
M = os.path.expanduser('~/dewey-scratch/r90mut')
|
|
env = dict(os.environ, TMPDIR=os.path.expanduser('~/dewey-scratch/tmp'))
|
|
R = 'packages/tasks/src/requirements.mjs'
|
|
J = 'packages/tasks/requirements.json'
|
|
V = 'packages/tasks/src/verbs.mjs'
|
|
B = 'packages/bus/src/broker.mjs'
|
|
T = 'packages/harness/src/tools.mjs'
|
|
RU = 'packages/harness/src/runner.mjs'
|
|
BU = 'packages/harness/src/bundle.mjs'
|
|
TASKS = ['packages/tasks/tests/*.test.mjs']
|
|
BUS = ['packages/bus/tests/*.test.mjs']
|
|
HAR = ['packages/harness/tests/*.test.mjs']
|
|
muts = [
|
|
# The list against the PRD.
|
|
('list order swapped', J, '"id": "REQ-VAR-1", "title": "Variables', '"id": "REQ-VAR-9", "title": "Variables', TASKS),
|
|
('list two ids swapped', J, '{ "id": "REQ-VAR-1", "title": "Variables in four layers, declared in code; limits intersect; unknown keys refused" },\n { "id": "REQ-VAR-2", "title": "One system config, never auto-written; secrets only as references the broker reads" },',
|
|
'{ "id": "REQ-VAR-2", "title": "One system config, never auto-written; secrets only as references the broker reads" },\n { "id": "REQ-VAR-1", "title": "Variables in four layers, declared in code; limits intersect; unknown keys refused" },', TASKS),
|
|
('list drops the last id', J, ',\n { "id": "REQ-EVT-1", "title": "Every slice 1 step emits an event; measures come from events" }', '', TASKS),
|
|
('list adds REQ-QA-1', J, ' { "id": "REQ-EVT-1",', ' { "id": "REQ-QA-1", "title": "QA" },\n { "id": "REQ-EVT-1",', TASKS),
|
|
('list version 1.1', J, '"version": "1.0"', '"version": "1.1"', TASKS),
|
|
# The schema.
|
|
('top keys open', R, "exact(list, 'list', ['prd', 'version', 'requirements']);", "if (!plain(list)) throw new RequirementsError('list: not an object');", TASKS),
|
|
('prd pattern dropped', R, " || !/^docs\\/prd\\/[a-z0-9-]+\\.md$/.test(list.prd)", "", TASKS),
|
|
('prd takes a parent step', R, "/^docs\\/prd\\/[a-z0-9-]+\\.md$/", "/^docs\\/prd\\/[a-z0-9.\\/-]+\\.md$/", TASKS),
|
|
('version any type', R, "typeof list.version !== 'string' || ", "", TASKS),
|
|
('version 0 major', R, "/^[1-9][0-9]*\\.[0-9]+$/", "/^[0-9]+\\.[0-9]+$/", TASKS),
|
|
('version minor optional', R, "/^[1-9][0-9]*\\.[0-9]+$/", "/^[1-9][0-9]*(\\.[0-9]+)?$/", TASKS),
|
|
('empty list allowed', R, " || !list.requirements.length", "", TASKS),
|
|
('entry keys open', R, "exact(r, at, ['id', 'title']);", "if (!plain(r)) throw new RequirementsError(`${at}: not an object`);", TASKS),
|
|
('id numbered 0', R, "/^REQ-[A-Z]+-[1-9][0-9]*$/;", "/^REQ-[A-Z]+-[0-9]+$/;", TASKS),
|
|
('id suffix allowed', R, "/^REQ-[A-Z]+-[1-9][0-9]*$/;", "/^REQ-[A-Z]+-[1-9][0-9]*/;", TASKS),
|
|
('id twice allowed', R, " if (seen.has(r.id)) throw new RequirementsError(`${at}.id: ${r.id} twice`);\n", "", TASKS),
|
|
('title 121', R, "!str(r.title, 120)", "!str(r.title, 121)", TASKS),
|
|
('title 119', R, "!str(r.title, 120)", "!str(r.title, 119)", TASKS),
|
|
('title newline allowed', R, " || r.title.includes('\\n')", "", TASKS),
|
|
('blank title allowed', R, "typeof x === 'string' && x.trim() !== '' &&", "typeof x === 'string' &&", TASKS),
|
|
('not frozen', R, " Object.freeze(x);\n", "", TASKS),
|
|
('frozen shallow', R, " for (const v of Object.values(x)) freeze(v);\n", "", TASKS),
|
|
('unchecked load', R, "return freeze(checkRequirements(list));", "return freeze(list);", TASKS),
|
|
('not JSON reported as unreadable', R, "e instanceof SyntaxError ? 'requirements.json: not JSON' :", "e instanceof SyntaxError ? 'requirements.json: unreadable' :", TASKS),
|
|
('ENOENT reported as unreadable', R, "`requirements.json: ${e.code ?? 'unreadable'}`", "'requirements.json: unreadable'", TASKS),
|
|
# task.create.
|
|
('unknown-requirement dropped', V, " if (!requirements.has(plan.requirement)) fail('unknown-requirement');\n", "", TASKS),
|
|
('unknown-requirement as invalid-request', V, "fail('unknown-requirement')", "fail('invalid-request')", TASKS),
|
|
('only the first id', V, "loadRequirements().requirements.map((r) => r.id)", "loadRequirements().requirements.slice(0, 1).map((r) => r.id)", TASKS),
|
|
('all but the last id', V, "loadRequirements().requirements.map((r) => r.id)", "loadRequirements().requirements.slice(0, -1).map((r) => r.id)", TASKS),
|
|
# The bus read.
|
|
('requirements agent guard dropped', B, " keys(a, []);\n this.#agent(s);\n return this.#requirements;", " keys(a, []);\n return this.#requirements;", BUS),
|
|
('requirements keys dropped', B, " keys(a, []);\n this.#agent(s);\n return this.#requirements;", " this.#agent(s);\n return this.#requirements;", BUS),
|
|
('requirements returns the ids only', B, "return this.#requirements;", "return this.#requirements.requirements;", BUS),
|
|
('constructor loads nothing', B, "this.#requirements = loadRequirements();", "this.#requirements = null;", BUS),
|
|
# The harness.
|
|
('list_requirements ungated', T, " if (held.has(\"task.create\"))\n add(\"list_requirements\"", " add(\"list_requirements\"", HAR),
|
|
('list_requirements gated on task.assign', T, "if (held.has(\"task.create\"))", "if (held.has(\"task.assign\"))", HAR),
|
|
('list_requirements without tracker', T, " // Row 90 (#1584): the ids task_create takes, for the sessions that hold it.\n if (held.has", " }\n {\n if (held.has", HAR),
|
|
('list_requirements wrong verb', T, "add(\"list_requirements\", \"requirements\",", "add(\"list_requirements\", \"tasks\",", HAR),
|
|
('list_requirements no prd.approve', T, " When none fits a request, raise a prd.approve decision instead of creating the task.", "", HAR),
|
|
('request description old', T, "str(\"The request id of the human message this task answers, shown as \\\"request <id>\\\" in that message's header; not the message's own id\")", "str(\"Id of the message or human input this task answers\")", HAR),
|
|
('requirement description old', T, "Requirement id from list_requirements, such as REQ-CRED-2", "Requirement id, such as REQ-CRED-2", HAR),
|
|
('blocking description old', T, "\"True only for a decision that holds a task: it needs task_ref, and the task waits for the answer. A request with no task is false\"", "\"True when you can't continue without the answer\"", HAR),
|
|
('turn request always', RU, "if (m.request) head.push(`request ${m.request}`);", "head.push(`request ${m.request}`);", HAR),
|
|
('turn request dropped', RU, " if (m.request) head.push(`request ${m.request}`);\n", "", HAR),
|
|
('stop line dropped', BU, " \"- Only the human stops a run, with `mosaic stop`. If you think a run should stop, raise a role.revoke decision with blocking false and the role instance as target, naming the run and saying why.\",\n", "", HAR),
|
|
]
|
|
only = sys.argv[1:]
|
|
for name, f, old, new, tests in muts:
|
|
if only and not any(o in name for o in only): continue
|
|
p = os.path.join(M, f); src = open(p).read()
|
|
n = src.count(old)
|
|
if n != 1: print(f'BAD-ANCHOR ({n}) {name}', flush=True); continue
|
|
open(p, 'w').write(src.replace(old, new))
|
|
# Its own process group, so a hung test's children die with it (a mutant that loops forever
|
|
# never fails on its own; a hang is a kill).
|
|
proc = subprocess.Popen(['node', '--test', *tests], cwd=M, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, start_new_session=True)
|
|
try:
|
|
out, _ = proc.communicate(timeout=300)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(proc.pid, signal.SIGKILL); proc.communicate()
|
|
out = None
|
|
finally:
|
|
open(p, 'w').write(src)
|
|
if out is None:
|
|
print(f'killed {name} (hung: killed after 300 s)', flush=True); continue
|
|
fails = [l for l in out.splitlines() if l.startswith('not ok')]
|
|
print(('killed ' if proc.returncode else 'SURVIVED') + f' {name}' + (f' ({len(fails)} failing)' if fails else ''), flush=True)
|