#!/usr/bin/env bash # mutate-push-guard.sh -- regenerate the README's mutation table from MEASUREMENT. # # WHY THIS EXISTS. The README carried a hand-written mutation table quoting # "32/32" style results. Those numbers were true when typed and went stale in # silence as the suite grew. A README is what a reader trusts when the tool # misbehaves, so a confidently-wrong one is worse than none. Every number the # README prints about mutation now comes out of this script. # # ============================ WHAT THIS TOOL GOT WRONG ======================== # Three defects, all found by review, all of the same shape: A TOOL WHOSE ENTIRE # OUTPUT IS A COVERAGE CLAIM MUST BE HARDER TO FOOL THAN THE CODE IT MEASURES. # # 1. IT REPORTED FULL COVERAGE ON A RED BASELINE. A mutant was "killed" whenever # the suite reported any failure at all, and the baseline was run only at the # END and never required to be green. So ONE pre-existing suite failure -- # changing no guard behaviour whatsoever -- satisfied EVERY mutant: 13 killed, # 0 survived, a confident table generated and pasted into the README, exit 0. # Now: the baseline runs FIRST and must be exit-0 with zero failures, and a # kill requires the mutant to break a case THE BASELINE PASSED, recorded BY # NAME. A tally is not evidence; a named delta is. # # 2. IT MUTATED THE REVIEWED SOURCE IN PLACE. Restoration leaned on an EXIT trap. # A TRAP IS CLEANUP, NOT ISOLATION -- SIGKILL cannot run it. An interrupted run # left push-guard.sh mutated in the working tree, and the reviewer's NEXT # suite run silently inherited it. A verification tool that alters its subject # can leave the subject wrong in a way the next measurement believes. # Now: the subject is copied into a temp dir and only the COPY is ever # written to. The source is untouched BY CONSTRUCTION rather than by cleanup, # which is the only version of this that survives kill -9. # # 3. ITS WORK DIR WAS SHARED. Concurrent runs interfered through the suite's # default .work directory. Each run now gets its own. # # (Note the deliberate asymmetry with verify-clean-clone.sh, which forbids cp: # there the copy LAUNDERED the property under measurement, so measuring a copy # was the defect. Here mutation is destructive by design, so copying is what # PROTECTS the subject. The rule is not "never copy" -- it is "know whether the # copy preserves the property you are about to measure.") # # ================== THREE WAYS A MUTATION RUN LIES, AND THE GUARD FOR EACH ==== # A. THE ANCHOR NO LONGER MATCHES. The mutant is never applied, the suite is # green, and the report says SURVIVED -- the same word a real coverage gap # gets. Guarded: ANCHOR MISSING is a loud failure. # B. THE ANCHOR MATCHES PROSE. This one landed on the first run: a mutant aimed # at a branch matched inside the usage() heredoc, edited a help string, # changed no behaviour, and duly reported SURVIVED. A documentation edit was # one step from being recorded as an uncovered branch. A MUTATION THAT CANNOT # CHANGE BEHAVIOUR IS NOT A SURVIVING MUTANT, IT IS A NON-MEASUREMENT. # Guarded: anchors resolving inside usage() are refused. # C. THE ANCHOR IS AMBIGUOUS. Two unrelated branches here are both the line # `if (( status != 0 )); then`; a first-match replace would credit the kill # to the wrong branch. Guarded: a match count != 1 refuses rather than guesses. set -uo pipefail SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" while (( $# )); do case "$1" in # --dir exists so this tool can be pointed at a FIXTURE copy and tested. --dir) SRC_DIR="$(cd "$2" && pwd)"; shift 2 ;; *) printf 'usage error: unknown argument: %s\n' "$1" >&2; exit 64 ;; esac done SRC_TARGET="$SRC_DIR/push-guard.sh" SRC_SUITE="$SRC_DIR/test-push-guard.sh" for f in "$SRC_TARGET" "$SRC_SUITE"; do [[ -r "$f" ]] || { printf 'REFUSING: cannot read %s\n' "$f" >&2; exit 1; } done # --- ISOLATION, NOT CLEANUP -------------------------------------------------- # Everything below writes only inside WORK. The trap is a courtesy for disk # space; correctness does not depend on it running. WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT install -m 755 "$SRC_TARGET" "$WORK/push-guard.sh" install -m 755 "$SRC_SUITE" "$WORK/test-push-guard.sh" TARGET="$WORK/push-guard.sh" SUITE="$WORK/test-push-guard.sh" BAK="$WORK/push-guard.sh.orig" cp "$TARGET" "$BAK" # Per-run work dir: the suite otherwise defaults to a shared .work beside itself, # and two concurrent runs corrupt each other's fixtures. export MOSAIC_TEST_WORK_DIR="$WORK/.work" # --- where the prose lives: usage() { ... EOF --------------------------------- PROSE_LO="$(grep -n '^usage() {' "$BAK" | head -1 | cut -d: -f1)" PROSE_HI="$(awk -v lo="$PROSE_LO" 'NR > lo && /^EOF$/ { print NR; exit }' "$BAK")" if [[ -z "$PROSE_LO" || -z "$PROSE_HI" ]]; then echo "!! cannot locate the usage() heredoc -- the prose guard would be inert; refusing" >&2 exit 1 fi # passing_cases -- names of cases that PASSED, one per line passing_cases() { printf '%s\n' "$1" | sed -n 's/^ PASS \[[^]]*\] \(.*\) (exit [0-9]*)$/\1/p'; } tally() { printf '%s\n' "$1" | grep -E 'needles: [0-9]+ passed' | tail -1; } # --- THE BASELINE MUST BE GREEN, AND IT IS ESTABLISHED FIRST ------------------ printf '=== baseline (must be exit 0 with zero failures) ===\n' BASE_OUT="$("$SUITE" 2>&1)"; BASE_RC=$? BASE_LINE="$(tally "$BASE_OUT")" BASE_FAILED="$(printf '%s\n' "$BASE_LINE" | sed -n 's/.*, \([0-9]*\) failed.*/\1/p')" if (( BASE_RC != 0 )) || [[ -z "$BASE_LINE" || "$BASE_FAILED" != "0" ]]; then printf 'REFUSING: baseline is not green -- exit %s, tally: %s\n' \ "$BASE_RC" "${BASE_LINE:-}" >&2 printf '\nEvery mutant would be scored KILLED by the pre-existing failure, and this\n' >&2 printf 'tool would publish a confident coverage table that measured nothing. Fix the\n' >&2 printf 'suite first. NO TABLE IS EMITTED.\n' >&2 exit 1 fi printf ' %s\n' "$BASE_LINE" mapfile -t BASE_PASSING < <(passing_cases "$BASE_OUT") printf ' %d named cases passing at baseline\n' "${#BASE_PASSING[@]}" if (( ${#BASE_PASSING[@]} == 0 )); then printf 'REFUSING: could not parse any case names -- kills could not be attributed.\n' >&2 exit 1 fi printf ' prose (usage heredoc) is lines %s-%s -- anchors there are refused, not scored\n\n' \ "$PROSE_LO" "$PROSE_HI" rc_all=0 KILLED=0; SURVIVED=0 declare -a ROWS=() mutate() { local name="$1" find="$2" repl="$3" # Count and locate in python so MULTI-LINE anchors work. They are required: # two unrelated branches in this file are both the single line # `if (( status != 0 )); then`, and a one-line anchor cannot say which one a # result belongs to. Guessing would attribute a kill to the wrong branch. local loc; loc="$(python3 - "$BAK" "$find" <<'LOCPY' import sys s = open(sys.argv[1]).read(); find = sys.argv[2] n = s.count(find) print(n, (s[:s.index(find)].count("\n") + 1) if n else 0) LOCPY )" local n="${loc%% *}" ln="${loc##* }" if (( n == 0 )); then printf ' !! ANCHOR MISSING %-46s NOT APPLIED -- result would be meaningless\n' "$name" rc_all=1; return fi if (( n != 1 )); then printf ' !! ANCHOR AMBIGUOUS %-46s %d matches -- refusing to guess which branch\n' "$name" "$n" rc_all=1; return fi if (( ln >= PROSE_LO && ln <= PROSE_HI )); then printf ' !! ANCHOR IS PROSE %-46s line %d is inside usage() -- not a branch\n' "$name" "$ln" rc_all=1; return fi python3 - "$BAK" "$TARGET" "$find" "$repl" <<'MUTPY' import sys src, dst, find, repl = sys.argv[1:5] open(dst, "w").write(open(src).read().replace(find, repl, 1)) MUTPY local out; out="$("$SUITE" 2>&1)" cp "$BAK" "$TARGET" local line; line="$(tally "$out")" if [[ -z "$line" ]]; then printf ' !! NO TALLY %-46s suite produced no needle count\n' "$name" rc_all=1; return fi # A KILL IS A NAMED DELTA, NOT A TALLY. Cases that passed at baseline and no # longer pass are the evidence; anything else (a case that was already # failing, a suite that died early) cannot be credited to this mutant. local now; now="$(passing_cases "$out")" local -a broke=() local c for c in "${BASE_PASSING[@]}"; do grep -qxF -- "$c" <<<"$now" || broke+=("$c") done local total="${#BASE_PASSING[@]}" if (( ${#broke[@]} > 0 )); then printf ' KILLED L%-5s %-46s %d/%d fail\n' "$ln" "$name" "${#broke[@]}" "$total" printf ' by: %s\n' "${broke[0]}" (( ${#broke[@]} > 1 )) && printf ' +%d more\n' "$(( ${#broke[@]} - 1 ))" ROWS+=("| \`$name\` (L$ln) | ${#broke[@]}/$total fail | killed |") KILLED=$(( KILLED + 1 )) else printf ' SURVIVED L%-5s %-46s 0/%d fail <-- UNCOVERED BRANCH\n' "$ln" "$name" "$total" ROWS+=("| \`$name\` (L$ln) | 0/$total fail | **SURVIVED** |") SURVIVED=$(( SURVIVED + 1 )); rc_all=1 fi } echo "=== push-guard mutation run ===" # EVERY ANCHOR BELOW IS VERBATIM SOURCE TEXT OF THE GUARD, so the single quotes # are load-bearing: these strings must reach `mutate` as the CHARACTERS that # appear in push-guard.sh. Expanding them would search for THIS shell's (unset) # $rel, $cmode, $EX_CONFIG and match nothing -- which the anchor guards would # report as ANCHOR MISSING rather than silently, but the intent is still to # forbid expansion. The directive is scoped to this function so it cannot mask a # genuine unintended-literal anywhere else in the file. # shellcheck disable=SC2016 run_mutants() { mutate "json decision requirement bypassed" \ ' if (( ${#JSON_PATHS[@]} == 0 )); then' ' if false; then' mutate "opt-out accepted with no written reason" \ 'if not isinstance(reason, str) or not reason.strip():' 'if False:' mutate "committed re-read of the opt-out skipped" \ ' [[ "$CFG_MODE" == "none" ]] || return 0' ' return 0' mutate "untracked config honoured as an opt-out" \ ' if [[ -z "$rel" ]]; then' ' if false; then' mutate "staged-but-uncommitted opt-out honoured" \ ' if [[ -z "$cmode" ]]; then' ' if false; then' mutate "committed SYMLINK config honoured" \ ' if [[ "$cmode" == "120000" ]]; then' ' if false; then' mutate "unparseable committed config ignored" \ ' if (( cstatus != 0 )); then' ' if false; then' mutate "local-only opt-out (HEAD says ON) honoured" \ ' if [[ "$cmode_val" != "none" ]]; then' ' if false; then' mutate "empty MERGE exempted" \ ' if [[ "$all_same" == yes ]]; then' ' if false; then' mutate "empty ROOT exempted" \ 'if [[ -z "$(git diff-tree --root -r --name-only --no-commit-id HEAD)" ]]; then' \ 'if false; then' mutate "--since-head ancestry check removed" \ 'if ! git merge-base --is-ancestor "$since_head" "$head"; then' 'if false; then' # guard's own source text, matched verbatim. Expanding them here would search for # this shell's (empty) $EX_CONFIG instead of the characters in the file. mutate "staged-file enumeration ignores git failure" \ "$(printf 'if (( status != 0 )); then\n local msg')" \ "$(printf 'if false; then\n local msg')" mutate "malformed config degrades to absent instead of refusing" \ "$(printf 'if (( status != 0 )); then\n fail "$EX_CONFIG"')" \ "$(printf 'if false; then\n fail "$EX_CONFIG"')" } run_mutants printf '\nbaseline: %s\n' "$BASE_LINE" printf '%d killed, %d survived\n' "$KILLED" "$SURVIVED" printf '\n--- README TABLE (paste verbatim) ---\n' printf '| mutation | suite result | verdict |\n|---|---|---|\n' printf '| *unmodified* | %s | baseline |\n' "$(printf '%s' "$BASE_LINE" | sed 's/push-guard needles: //')" printf '%s\n' "${ROWS[@]}" exit "$rc_all"