#!/usr/bin/env bash # test-mutate-push-guard.sh -- needles for the mutation generator. # # The generator's entire output is a COVERAGE CLAIM, and the README publishes it. # That makes it the most dangerous file here: when it is wrong it does not fail, # it reassures. Two of its three defects were found by review rather than by any # test, so these are the cases that had to exist. # # g1 a RED BASELINE must be refused before any mutant runs. Previously ONE # pre-existing suite failure -- changing no guard behaviour at all -- # satisfied every mutant: 13 killed, 0 survived, table emitted, exit 0. # g3 an INTERRUPTED run must leave the subject byte-identical. Restoration # used to lean on an EXIT trap, and A TRAP IS CLEANUP, NOT ISOLATION: # SIGKILL cannot run it, so an interrupted run stranded a mutated # push-guard.sh that the next suite run silently inherited. # g2 is the positive control. Without it every case here could be passing # because the generator refuses unconditionally, which is a wall, not a gate. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GEN="$HERE/mutate-push-guard.sh" PASS=0; FAIL=0 TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT ok() { printf ' ok [%-16s] %s\n' "$1" "${2:-}"; PASS=$(( PASS + 1 )); } bad() { printf ' FAIL [%-16s] %s\n' "$1" "$2"; FAIL=$(( FAIL + 1 )); } # fixture -- a directory holding an independent copy of guard + suite fixture() { local d="$TMP/$1"; mkdir -p "$d" install -m 755 "$HERE/push-guard.sh" "$d/push-guard.sh" install -m 755 "$HERE/test-push-guard.sh" "$d/test-push-guard.sh" printf '%s\n' "$d" } echo "== g1: a RED BASELINE must be refused, with NO table emitted ==" # The injected case asserts exit 99 from `true`. It fails always, and it changes # NO guard behaviour -- which is the whole point: a defect anywhere in the suite # used to be enough to certify every branch as covered. g1="$(fixture red)" python3 - "$g1/test-push-guard.sh" <<'PY' import sys p = sys.argv[1]; s = open(p).read() anchor = 'mkdir -p "$WORK_DIR"\n' assert s.count(anchor) == 1, "injection anchor not unique -- fixture would not be the red baseline" s = s.replace(anchor, anchor + '\nexpect NEEDLE 99 "injected always-failing case" -- true\n', 1) open(p, "w").write(s) PY # Prove the fixture really IS red before asserting the generator notices, or g1 # could pass against a green suite and test nothing. if "$g1/test-push-guard.sh" >/dev/null 2>&1; then bad g1-fixture "injected suite still passes -- fixture is not a red baseline" else ok g1-fixture "fixture suite is red, as required" fi out="$("$GEN" --dir "$g1" 2>&1)"; rc=$? if (( rc != 0 )) && [[ "$out" == *"REFUSING: baseline is not green"* ]]; then ok g1-refused "exit $rc, refused before mutating" else bad g1-refused "wanted nonzero + refusal; got rc=$rc" fi if [[ "$out" != *"README TABLE"* && "$out" != *"killed,"* ]]; then ok g1-no-table "no coverage table emitted from a red baseline" else bad g1-no-table "a table or kill count was published despite the red baseline" fi echo echo "== g3: an INTERRUPTED run must not alter the subject ==" # THE KILL MUST LAND INSIDE A MUTATION WINDOW OR THIS CASE PROVES NOTHING. # First attempt used `timeout -s KILL 3`. At three seconds the generator is still # running its BASELINE, so no mutation has been applied yet and the subject is # trivially unchanged -- for the ORIGINAL in-place generator too, which I # confirmed by running it. The control passed for a reason unrelated to the fix: # a vacuous control, in the control written for blocker 3. # # So the kill is now driven from INSIDE the run. The fixture's suite counts its # own invocations and SIGKILLs the generator on the second one -- invocation 1 is # the baseline, invocation 2 happens with mutant #1 APPLIED. That is exactly the # window where an in-place generator strands a mutated subject. instrument_kill_at_second_run() { python3 - "$1" <<'PY' import sys p = sys.argv[1]; s = open(p).read() anchor = 'PASS=0\nFAIL=0\n' assert s.count(anchor) == 1, "instrumentation anchor not unique" inject = anchor + ''' if [[ -n "${G3_COUNTER:-}" ]]; then n=$(( $(cat "$G3_COUNTER" 2>/dev/null || echo 0) + 1 )) printf '%s' "$n" > "$G3_COUNTER" # Invocation 2 = first mutant applied. Kill the generator where it hurts. # `kill -9 0` targets the whole PROCESS GROUP, not $PPID: the generator runs # the suite inside $( ), which forks, so $PPID is that subshell and killing # it merely ends the command substitution -- the generator carries on and # exits 0. The caller puts the generator in its OWN group via setsid, so the # group is exactly the generator and its children, and this harness is not # in it. (( n == 2 )) && kill -9 0 fi ''' open(p, "w").write(s.replace(anchor, inject, 1)) PY } # run_killed -> echoes " " run_killed() { local d="$1" gen="$2" before after rc instrument_kill_at_second_run "$d/test-push-guard.sh" before="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)" G3_COUNTER="$d/.count" setsid --wait "$gen" --dir "$d" >/dev/null 2>&1; rc=$? after="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)" printf '%s %s %s\n' "$rc" "$before" "$after" } g3="$(fixture killed)" read -r krc before after <<<"$(run_killed "$g3" "$GEN")" if (( krc != 0 )); then ok g3-was-killed "generator died mid-mutation (exit $krc)" else bad g3-was-killed "generator exited 0 -- the kill never landed, so the check below is vacuous" fi if [[ "$before" == "$after" ]]; then ok g3-subject-intact "push-guard.sh byte-identical after the kill" else bad g3-subject-intact "SUBJECT MUTATED AND STRANDED: $before -> $after" fi # PROVE THE NEEDLE BITES. Reconstruct the pre-fix mechanism -- mutate the source # in place, restore from an EXIT trap -- and put it through the identical kill. # If this does NOT strand a mutated file, g3-subject-intact is measuring nothing # and the isolation fix is unevidenced. g3o="$(fixture killed-inplace)" python3 - "$GEN" "$g3o/gen-inplace.sh" <<'PY' import sys s = open(sys.argv[1]).read() old = '''install -m 755 "$SRC_TARGET" "$WORK/push-guard.sh" install -m 755 "$SRC_SUITE" "$WORK/test-push-guard.sh" TARGET="$WORK/push-guard.sh" SUITE="$WORK/test-push-guard.sh" BAK="$WORK/push-guard.sh.orig" cp "$TARGET" "$BAK"''' new = '''TARGET="$SRC_TARGET" SUITE="$SRC_SUITE" BAK="$WORK/push-guard.sh.orig" cp "$TARGET" "$BAK" trap 'cp "$BAK" "$TARGET"; rm -rf "$WORK"' EXIT''' assert s.count(old) == 1, "cannot reconstruct the in-place mechanism -- bite proof would be fake" open(sys.argv[2], "w").write(s.replace(old, new, 1)) PY chmod +x "$g3o/gen-inplace.sh" read -r _orc obefore oafter <<<"$(run_killed "$g3o" "$g3o/gen-inplace.sh")" if [[ "$obefore" != "$oafter" ]]; then ok g3-needle-bites "in-place generator strands a mutated subject, as it must" else bad g3-needle-bites "the OLD mechanism also left the subject intact -- g3 is vacuous" fi echo echo "== g2: POSITIVE CONTROL -- a clean subject must produce a full green run ==" g2="$(fixture clean)" out2="$("$GEN" --dir "$g2" 2>&1)"; rc2=$? if (( rc2 == 0 )) && [[ "$out2" == *"0 survived"* ]]; then ok g2-control "$(printf '%s' "$out2" | grep -E '^[0-9]+ killed')" else bad g2-control "wanted exit 0 with 0 survived; got rc=$rc2" fi # A kill must be attributed to a NAMED case, not to a bare tally -- that is the # fix for blocker 2 and it needs its own assertion. if [[ "$out2" == *" by: "* ]]; then ok g2-attributed "kills name the case they broke" else bad g2-attributed "no per-mutant case attribution in the output" fi echo printf '%d passed, %d failed\n' "$PASS" "$FAIL" (( FAIL == 0 ))