242 lines
12 KiB
Bash
Executable File
242 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# mutate-push-guard.sh -- regenerate the README's mutation table from MEASUREMENT.
|
|
#
|
|
# WHY THIS EXISTS. The README carried a hand-written mutation table quoting
|
|
# "32/32" style results. Those numbers were true when typed and went stale in
|
|
# silence as the suite grew. A README is what a reader trusts when the tool
|
|
# misbehaves, so a confidently-wrong one is worse than none. Every number the
|
|
# README prints about mutation now comes out of this script.
|
|
#
|
|
# ============================ WHAT THIS TOOL GOT WRONG ========================
|
|
# Three defects, all found by review, all of the same shape: A TOOL WHOSE ENTIRE
|
|
# OUTPUT IS A COVERAGE CLAIM MUST BE HARDER TO FOOL THAN THE CODE IT MEASURES.
|
|
#
|
|
# 1. IT REPORTED FULL COVERAGE ON A RED BASELINE. A mutant was "killed" whenever
|
|
# the suite reported any failure at all, and the baseline was run only at the
|
|
# END and never required to be green. So ONE pre-existing suite failure --
|
|
# changing no guard behaviour whatsoever -- satisfied EVERY mutant: 13 killed,
|
|
# 0 survived, a confident table generated and pasted into the README, exit 0.
|
|
# Now: the baseline runs FIRST and must be exit-0 with zero failures, and a
|
|
# kill requires the mutant to break a case THE BASELINE PASSED, recorded BY
|
|
# NAME. A tally is not evidence; a named delta is.
|
|
#
|
|
# 2. IT MUTATED THE REVIEWED SOURCE IN PLACE. Restoration leaned on an EXIT trap.
|
|
# A TRAP IS CLEANUP, NOT ISOLATION -- SIGKILL cannot run it. An interrupted run
|
|
# left push-guard.sh mutated in the working tree, and the reviewer's NEXT
|
|
# suite run silently inherited it. A verification tool that alters its subject
|
|
# can leave the subject wrong in a way the next measurement believes.
|
|
# Now: the subject is copied into a temp dir and only the COPY is ever
|
|
# written to. The source is untouched BY CONSTRUCTION rather than by cleanup,
|
|
# which is the only version of this that survives kill -9.
|
|
#
|
|
# 3. ITS WORK DIR WAS SHARED. Concurrent runs interfered through the suite's
|
|
# default .work directory. Each run now gets its own.
|
|
#
|
|
# (Note the deliberate asymmetry with verify-clean-clone.sh, which forbids cp:
|
|
# there the copy LAUNDERED the property under measurement, so measuring a copy
|
|
# was the defect. Here mutation is destructive by design, so copying is what
|
|
# PROTECTS the subject. The rule is not "never copy" -- it is "know whether the
|
|
# copy preserves the property you are about to measure.")
|
|
#
|
|
# ================== THREE WAYS A MUTATION RUN LIES, AND THE GUARD FOR EACH ====
|
|
# A. THE ANCHOR NO LONGER MATCHES. The mutant is never applied, the suite is
|
|
# green, and the report says SURVIVED -- the same word a real coverage gap
|
|
# gets. Guarded: ANCHOR MISSING is a loud failure.
|
|
# B. THE ANCHOR MATCHES PROSE. This one landed on the first run: a mutant aimed
|
|
# at a branch matched inside the usage() heredoc, edited a help string,
|
|
# changed no behaviour, and duly reported SURVIVED. A documentation edit was
|
|
# one step from being recorded as an uncovered branch. A MUTATION THAT CANNOT
|
|
# CHANGE BEHAVIOUR IS NOT A SURVIVING MUTANT, IT IS A NON-MEASUREMENT.
|
|
# Guarded: anchors resolving inside usage() are refused.
|
|
# C. THE ANCHOR IS AMBIGUOUS. Two unrelated branches here are both the line
|
|
# `if (( status != 0 )); then`; a first-match replace would credit the kill
|
|
# to the wrong branch. Guarded: a match count != 1 refuses rather than guesses.
|
|
set -uo pipefail
|
|
|
|
SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
while (( $# )); do
|
|
case "$1" in
|
|
# --dir exists so this tool can be pointed at a FIXTURE copy and tested.
|
|
--dir) SRC_DIR="$(cd "$2" && pwd)"; shift 2 ;;
|
|
*) printf 'usage error: unknown argument: %s\n' "$1" >&2; exit 64 ;;
|
|
esac
|
|
done
|
|
|
|
SRC_TARGET="$SRC_DIR/push-guard.sh"
|
|
SRC_SUITE="$SRC_DIR/test-push-guard.sh"
|
|
for f in "$SRC_TARGET" "$SRC_SUITE"; do
|
|
[[ -r "$f" ]] || { printf 'REFUSING: cannot read %s\n' "$f" >&2; exit 1; }
|
|
done
|
|
|
|
# --- ISOLATION, NOT CLEANUP --------------------------------------------------
|
|
# Everything below writes only inside WORK. The trap is a courtesy for disk
|
|
# space; correctness does not depend on it running.
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
install -m 755 "$SRC_TARGET" "$WORK/push-guard.sh"
|
|
install -m 755 "$SRC_SUITE" "$WORK/test-push-guard.sh"
|
|
TARGET="$WORK/push-guard.sh"
|
|
SUITE="$WORK/test-push-guard.sh"
|
|
BAK="$WORK/push-guard.sh.orig"
|
|
cp "$TARGET" "$BAK"
|
|
# Per-run work dir: the suite otherwise defaults to a shared .work beside itself,
|
|
# and two concurrent runs corrupt each other's fixtures.
|
|
export MOSAIC_TEST_WORK_DIR="$WORK/.work"
|
|
|
|
# --- where the prose lives: usage() { ... EOF ---------------------------------
|
|
PROSE_LO="$(grep -n '^usage() {' "$BAK" | head -1 | cut -d: -f1)"
|
|
PROSE_HI="$(awk -v lo="$PROSE_LO" 'NR > lo && /^EOF$/ { print NR; exit }' "$BAK")"
|
|
if [[ -z "$PROSE_LO" || -z "$PROSE_HI" ]]; then
|
|
echo "!! cannot locate the usage() heredoc -- the prose guard would be inert; refusing" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# passing_cases <output> -- names of cases that PASSED, one per line
|
|
passing_cases() { printf '%s\n' "$1" | sed -n 's/^ PASS \[[^]]*\] \(.*\) (exit [0-9]*)$/\1/p'; }
|
|
tally() { printf '%s\n' "$1" | grep -E 'needles: [0-9]+ passed' | tail -1; }
|
|
|
|
# --- THE BASELINE MUST BE GREEN, AND IT IS ESTABLISHED FIRST ------------------
|
|
printf '=== baseline (must be exit 0 with zero failures) ===\n'
|
|
BASE_OUT="$("$SUITE" 2>&1)"; BASE_RC=$?
|
|
BASE_LINE="$(tally "$BASE_OUT")"
|
|
BASE_FAILED="$(printf '%s\n' "$BASE_LINE" | sed -n 's/.*, \([0-9]*\) failed.*/\1/p')"
|
|
if (( BASE_RC != 0 )) || [[ -z "$BASE_LINE" || "$BASE_FAILED" != "0" ]]; then
|
|
printf 'REFUSING: baseline is not green -- exit %s, tally: %s\n' \
|
|
"$BASE_RC" "${BASE_LINE:-<no tally emitted>}" >&2
|
|
printf '\nEvery mutant would be scored KILLED by the pre-existing failure, and this\n' >&2
|
|
printf 'tool would publish a confident coverage table that measured nothing. Fix the\n' >&2
|
|
printf 'suite first. NO TABLE IS EMITTED.\n' >&2
|
|
exit 1
|
|
fi
|
|
printf ' %s\n' "$BASE_LINE"
|
|
mapfile -t BASE_PASSING < <(passing_cases "$BASE_OUT")
|
|
printf ' %d named cases passing at baseline\n' "${#BASE_PASSING[@]}"
|
|
if (( ${#BASE_PASSING[@]} == 0 )); then
|
|
printf 'REFUSING: could not parse any case names -- kills could not be attributed.\n' >&2
|
|
exit 1
|
|
fi
|
|
printf ' prose (usage heredoc) is lines %s-%s -- anchors there are refused, not scored\n\n' \
|
|
"$PROSE_LO" "$PROSE_HI"
|
|
|
|
rc_all=0
|
|
KILLED=0; SURVIVED=0
|
|
declare -a ROWS=()
|
|
|
|
mutate() {
|
|
local name="$1" find="$2" repl="$3"
|
|
# Count and locate in python so MULTI-LINE anchors work. They are required:
|
|
# two unrelated branches in this file are both the single line
|
|
# `if (( status != 0 )); then`, and a one-line anchor cannot say which one a
|
|
# result belongs to. Guessing would attribute a kill to the wrong branch.
|
|
local loc; loc="$(python3 - "$BAK" "$find" <<'LOCPY'
|
|
import sys
|
|
s = open(sys.argv[1]).read(); find = sys.argv[2]
|
|
n = s.count(find)
|
|
print(n, (s[:s.index(find)].count("\n") + 1) if n else 0)
|
|
LOCPY
|
|
)"
|
|
local n="${loc%% *}" ln="${loc##* }"
|
|
if (( n == 0 )); then
|
|
printf ' !! ANCHOR MISSING %-46s NOT APPLIED -- result would be meaningless\n' "$name"
|
|
rc_all=1; return
|
|
fi
|
|
if (( n != 1 )); then
|
|
printf ' !! ANCHOR AMBIGUOUS %-46s %d matches -- refusing to guess which branch\n' "$name" "$n"
|
|
rc_all=1; return
|
|
fi
|
|
if (( ln >= PROSE_LO && ln <= PROSE_HI )); then
|
|
printf ' !! ANCHOR IS PROSE %-46s line %d is inside usage() -- not a branch\n' "$name" "$ln"
|
|
rc_all=1; return
|
|
fi
|
|
|
|
python3 - "$BAK" "$TARGET" "$find" "$repl" <<'MUTPY'
|
|
import sys
|
|
src, dst, find, repl = sys.argv[1:5]
|
|
open(dst, "w").write(open(src).read().replace(find, repl, 1))
|
|
MUTPY
|
|
local out; out="$("$SUITE" 2>&1)"
|
|
cp "$BAK" "$TARGET"
|
|
|
|
local line; line="$(tally "$out")"
|
|
if [[ -z "$line" ]]; then
|
|
printf ' !! NO TALLY %-46s suite produced no needle count\n' "$name"
|
|
rc_all=1; return
|
|
fi
|
|
# A KILL IS A NAMED DELTA, NOT A TALLY. Cases that passed at baseline and no
|
|
# longer pass are the evidence; anything else (a case that was already
|
|
# failing, a suite that died early) cannot be credited to this mutant.
|
|
local now; now="$(passing_cases "$out")"
|
|
local -a broke=()
|
|
local c
|
|
for c in "${BASE_PASSING[@]}"; do
|
|
grep -qxF -- "$c" <<<"$now" || broke+=("$c")
|
|
done
|
|
local total="${#BASE_PASSING[@]}"
|
|
|
|
if (( ${#broke[@]} > 0 )); then
|
|
printf ' KILLED L%-5s %-46s %d/%d fail\n' "$ln" "$name" "${#broke[@]}" "$total"
|
|
printf ' by: %s\n' "${broke[0]}"
|
|
(( ${#broke[@]} > 1 )) && printf ' +%d more\n' "$(( ${#broke[@]} - 1 ))"
|
|
ROWS+=("| \`$name\` (L$ln) | ${#broke[@]}/$total fail | killed |")
|
|
KILLED=$(( KILLED + 1 ))
|
|
else
|
|
printf ' SURVIVED L%-5s %-46s 0/%d fail <-- UNCOVERED BRANCH\n' "$ln" "$name" "$total"
|
|
ROWS+=("| \`$name\` (L$ln) | 0/$total fail | **SURVIVED** |")
|
|
SURVIVED=$(( SURVIVED + 1 )); rc_all=1
|
|
fi
|
|
}
|
|
|
|
echo "=== push-guard mutation run ==="
|
|
# EVERY ANCHOR BELOW IS VERBATIM SOURCE TEXT OF THE GUARD, so the single quotes
|
|
# are load-bearing: these strings must reach `mutate` as the CHARACTERS that
|
|
# appear in push-guard.sh. Expanding them would search for THIS shell's (unset)
|
|
# $rel, $cmode, $EX_CONFIG and match nothing -- which the anchor guards would
|
|
# report as ANCHOR MISSING rather than silently, but the intent is still to
|
|
# forbid expansion. The directive is scoped to this function so it cannot mask a
|
|
# genuine unintended-literal anywhere else in the file.
|
|
# shellcheck disable=SC2016
|
|
run_mutants() {
|
|
mutate "json decision requirement bypassed" \
|
|
' if (( ${#JSON_PATHS[@]} == 0 )); then' ' if false; then'
|
|
mutate "opt-out accepted with no written reason" \
|
|
'if not isinstance(reason, str) or not reason.strip():' 'if False:'
|
|
mutate "committed re-read of the opt-out skipped" \
|
|
' [[ "$CFG_MODE" == "none" ]] || return 0' ' return 0'
|
|
mutate "untracked config honoured as an opt-out" \
|
|
' if [[ -z "$rel" ]]; then' ' if false; then'
|
|
mutate "staged-but-uncommitted opt-out honoured" \
|
|
' if [[ -z "$cmode" ]]; then' ' if false; then'
|
|
mutate "committed SYMLINK config honoured" \
|
|
' if [[ "$cmode" == "120000" ]]; then' ' if false; then'
|
|
mutate "unparseable committed config ignored" \
|
|
' if (( cstatus != 0 )); then' ' if false; then'
|
|
mutate "local-only opt-out (HEAD says ON) honoured" \
|
|
' if [[ "$cmode_val" != "none" ]]; then' ' if false; then'
|
|
mutate "empty MERGE exempted" \
|
|
' if [[ "$all_same" == yes ]]; then' ' if false; then'
|
|
mutate "empty ROOT exempted" \
|
|
'if [[ -z "$(git diff-tree --root -r --name-only --no-commit-id HEAD)" ]]; then' \
|
|
'if false; then'
|
|
mutate "--since-head ancestry check removed" \
|
|
'if ! git merge-base --is-ancestor "$since_head" "$head"; then' 'if false; then'
|
|
# guard's own source text, matched verbatim. Expanding them here would search for
|
|
# this shell's (empty) $EX_CONFIG instead of the characters in the file.
|
|
mutate "staged-file enumeration ignores git failure" \
|
|
"$(printf 'if (( status != 0 )); then\n local msg')" \
|
|
"$(printf 'if false; then\n local msg')"
|
|
mutate "malformed config degrades to absent instead of refusing" \
|
|
"$(printf 'if (( status != 0 )); then\n fail "$EX_CONFIG"')" \
|
|
"$(printf 'if false; then\n fail "$EX_CONFIG"')"
|
|
}
|
|
|
|
run_mutants
|
|
|
|
printf '\nbaseline: %s\n' "$BASE_LINE"
|
|
printf '%d killed, %d survived\n' "$KILLED" "$SURVIVED"
|
|
|
|
printf '\n--- README TABLE (paste verbatim) ---\n'
|
|
printf '| mutation | suite result | verdict |\n|---|---|---|\n'
|
|
printf '| *unmodified* | %s | baseline |\n' "$(printf '%s' "$BASE_LINE" | sed 's/push-guard needles: //')"
|
|
printf '%s\n' "${ROWS[@]}"
|
|
exit "$rc_all"
|