jason.woltjeandClaude Sonnet 5.5
8a73f713cf
Round 1 packet in agents/researcher/work/queue-132/: row132.patch (8 files, +162 -19) on b6a588cd2,
candidate manifest, build.md, 28 mutants killed, full gate green. No candidate change applied here.
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
103 lines
6.5 KiB
Bash
Executable File
103 lines
6.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Row 132 (#1628): harness tools for ending a run and changing the organization file.
|
||
# Each check must fail on a mutant. For each mutant this runs one named test file against the
|
||
# real source (control) and against the source with one exact-once text edit, in the given worktree,
|
||
# one test file at a time, and restores every source file on exit.
|
||
# Usage: mutants.sh <worktree> <outdir> (MUTANT_DRY=1 only proves every edit matches exactly once)
|
||
set -u
|
||
WT=$(cd "$1" && pwd); OUT=$(mkdir -p "$2" && cd "$2" && pwd)
|
||
cd "$WT" || exit 2
|
||
BAK=$(mktemp -d "$OUT/.bak.XXXXXX")
|
||
declare -A SAVED
|
||
save() { [ -n "${SAVED[$1]:-}" ] || { mkdir -p "$BAK/$(dirname "$1")"; cp "$1" "$BAK/$1"; SAVED[$1]=1; }; }
|
||
restore() { for f in "${!SAVED[@]}"; do cp "$BAK/$f" "$f"; done; rm -rf "$BAK"; }
|
||
trap restore EXIT
|
||
edit() { # file old new: exact-once replace
|
||
save "$1"
|
||
node -e '
|
||
const fs = require("fs"); const [f, a, b] = process.argv.slice(1);
|
||
const s = fs.readFileSync(f, "utf8"); const n = s.split(a).length - 1;
|
||
if (n !== 1) { console.error("expected one match, found " + n); process.exit(3); }
|
||
fs.writeFileSync(f, s.replace(a, () => b));
|
||
' "$1" "$2" "$3"
|
||
}
|
||
run() { # name testfile
|
||
if [ -n "${MUTANT_DRY:-}" ]; then echo "$1: edit ok (dry)"; return; fi
|
||
setsid -w timeout -k 10 400 node --test "$2" > "$OUT/$1.out" 2>&1; echo "$?" > "$OUT/$1.rc"
|
||
echo "$1: exit $(cat "$OUT/$1.rc"); $(grep -E '^ℹ (tests|pass|fail)' "$OUT/$1.out" | tr '\n' ' ')"
|
||
grep -E '^✖' "$OUT/$1.out" | sort -u | sed 's/ ([0-9.]*ms)$//' | sed "s/^/ failed: /"
|
||
}
|
||
mutant() { # name testfile file old new
|
||
edit "$3" "$4" "$5" || { echo "$1: edit failed"; return; }
|
||
diff -u "$BAK/$3" "$3" > "$OUT/$1.patch"
|
||
run "$1" "$2"
|
||
cp "$BAK/$3" "$3"
|
||
}
|
||
H=packages/harness; TT=$H/tests/tools.test.mjs; BT=$H/tests/bundle.test.mjs; RT=$H/tests/runner.test.mjs
|
||
T=$H/src/tools.mjs; BD=$H/src/bundle.mjs; RN=$H/src/runner.mjs; B=packages/bus/src/broker.mjs
|
||
for t in "$TT" "$BT" "$RT"; do run "control-$(basename "$t" .test.mjs)" "$t"; done
|
||
|
||
# The chief gets both tools; a role without the grants gets neither
|
||
# m1: end_run is offered to every role
|
||
mutant m1-end-run-always "$TT" "$T" 'if (held.has("role.end")) {' 'if (true) {'
|
||
# m2: change_organization is offered to every role
|
||
mutant m2-change-organization-always "$TT" "$T" 'if (held.has("organization.change")) {' 'if (true) {'
|
||
# m3: the chief gets no end_run
|
||
mutant m3-end-run-never "$TT" "$T" 'if (held.has("role.end")) {' 'if (false) {'
|
||
mutant m3b-end-run-never-bundle "$BT" "$T" 'if (held.has("role.end")) {' 'if (false) {'
|
||
# m4: the chief gets no change_organization
|
||
mutant m4-change-organization-never "$TT" "$T" 'if (held.has("organization.change")) {' 'if (false) {'
|
||
mutant m4b-change-organization-never-bundle "$BT" "$T" 'if (held.has("organization.change")) {' 'if (false) {'
|
||
|
||
# What each tool sends
|
||
# m5: end_run goes to the wrong verb
|
||
mutant m5-end-run-verb "$TT" "$T" 'add("end_run", "role.revoke", "role.end", ' 'add("end_run", "role.end", "role.end", '
|
||
mutant m5b-end-run-verb-runner "$RT" "$T" 'add("end_run", "role.revoke", "role.end", ' 'add("end_run", "role.end", "role.end", '
|
||
# m6: end_run has no action, so the manifest lists role.end as untyped
|
||
mutant m6-end-run-action "$TT" "$T" 'add("end_run", "role.revoke", "role.end", ' 'add("end_run", "role.revoke", null, '
|
||
# m7: change_organization goes to the human's verb
|
||
mutant m7-change-organization-verb "$TT" "$T" 'add("change_organization", "organization.set", "organization.change",' 'add("change_organization", "limits.set", "organization.change",'
|
||
mutant m7b-change-organization-verb-runner "$RT" "$T" 'add("change_organization", "organization.set", "organization.change",' 'add("change_organization", "limits.set", "organization.change",'
|
||
# m8: change_organization has no action, so the manifest lists organization.change as untyped
|
||
mutant m8-change-organization-action "$TT" "$T" 'add("change_organization", "organization.set", "organization.change",' 'add("change_organization", "organization.set", null,'
|
||
# m9: reason isn't required
|
||
mutant m9-reason-optional "$TT" "$T" '}, ["role", "reason"]));' '}, ["role"]));'
|
||
# m10: end_run takes a decision
|
||
mutant m10-end-run-decision "$TT" "$T" ' }, ["role", "reason"]));' ' decision: COMMON.decision,
|
||
}, ["role", "reason"]));'
|
||
# m11: the reason has no useful bound
|
||
mutant m11-reason-unbounded "$TT" "$T" 'maxLength: 500 }' 'maxLength: 50000 }'
|
||
# m12: revision isn't required
|
||
mutant m12-revision-optional "$TT" "$T" '}, ["revision"]));' '}, []));'
|
||
# m13: revision takes any string
|
||
mutant m13-revision-pattern "$TT" "$T" 'pattern: "^[a-f0-9]{64}$" }' 'pattern: "^.*$" }'
|
||
# m14: the description drops the own-run and assistant limits
|
||
mutant m14-description-limits "$TT" "$T" "You can't end your own run or the assistant's; the broker refuses with role-end-refused." 'The broker refuses some runs.'
|
||
|
||
# One key list
|
||
# m15: the harness list drops a key
|
||
mutant m15-harness-key-dropped "$TT" "$T" '"revision", "limits", "models", "instances"])' '"revision", "limits", "instances"])'
|
||
# m16: the broker list gains a key
|
||
mutant m16-broker-key-added "$TT" "$B" "export const ORGANIZATION_KEYS = ['revision', 'limits', 'models', 'instances'];" "export const ORGANIZATION_KEYS = ['revision', 'limits', 'models', 'instances', 'plugins'];"
|
||
|
||
# The bundle text
|
||
# m17: a role.end holder keeps the human-only text
|
||
mutant m17-bundle-text-unchanged "$BT" "$BD" 'lines.push(named.has("end_run")' 'lines.push(false'
|
||
# m18: every role gets the role.end text
|
||
mutant m18-bundle-text-for-all "$BT" "$BD" 'lines.push(named.has("end_run")' 'lines.push(true'
|
||
# m19: the text drops the reason requirement
|
||
mutant m19-text-reason "$BT" "$BD" '; the reason is required and goes into the log' ''
|
||
# m20: the text drops the own-run and assistant limits
|
||
mutant m20-text-limits "$BT" "$BD" " You can't end your own run or the assistant's." ''
|
||
# m21: the text drops the decision fallback
|
||
mutant m21-text-fallback "$BT" "$BD" 'For anything the broker refuses, raise a role.revoke decision' 'For anything else, raise a role.revoke decision'
|
||
|
||
# Through the runner
|
||
# m22: the runner drops the tool's arguments
|
||
mutant m22-runner-drops-args "$RT" "$RN" 'await client.call(tool.verb, args)' 'await client.call(tool.verb, {})'
|
||
# m23: the runner sends the tool's name as the verb
|
||
mutant m23-runner-name-as-verb "$RT" "$RN" 'await client.call(tool.verb, args)' 'await client.call(tool.name, args)'
|
||
# m24: the runner hides the broker's refusal code
|
||
mutant m24-runner-hides-code "$RT" "$RN" 'error: /^[a-z-]{1,64}$/.test(e.code ?? "") ? e.code : "tool-failed"' 'error: "tool-failed"'
|
||
echo done
|