179 lines
7.6 KiB
Bash
Executable File
179 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-mutate-push-guard.sh -- needles for the mutation generator.
|
|
#
|
|
# The generator's entire output is a COVERAGE CLAIM, and the README publishes it.
|
|
# That makes it the most dangerous file here: when it is wrong it does not fail,
|
|
# it reassures. Two of its three defects were found by review rather than by any
|
|
# test, so these are the cases that had to exist.
|
|
#
|
|
# g1 a RED BASELINE must be refused before any mutant runs. Previously ONE
|
|
# pre-existing suite failure -- changing no guard behaviour at all --
|
|
# satisfied every mutant: 13 killed, 0 survived, table emitted, exit 0.
|
|
# g3 an INTERRUPTED run must leave the subject byte-identical. Restoration
|
|
# used to lean on an EXIT trap, and A TRAP IS CLEANUP, NOT ISOLATION:
|
|
# SIGKILL cannot run it, so an interrupted run stranded a mutated
|
|
# push-guard.sh that the next suite run silently inherited.
|
|
# g2 is the positive control. Without it every case here could be passing
|
|
# because the generator refuses unconditionally, which is a wall, not a gate.
|
|
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
GEN="$HERE/mutate-push-guard.sh"
|
|
PASS=0; FAIL=0
|
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
|
|
ok() { printf ' ok [%-16s] %s\n' "$1" "${2:-}"; PASS=$(( PASS + 1 )); }
|
|
bad() { printf ' FAIL [%-16s] %s\n' "$1" "$2"; FAIL=$(( FAIL + 1 )); }
|
|
|
|
# fixture <name> -- a directory holding an independent copy of guard + suite
|
|
fixture() {
|
|
local d="$TMP/$1"; mkdir -p "$d"
|
|
install -m 755 "$HERE/push-guard.sh" "$d/push-guard.sh"
|
|
install -m 755 "$HERE/test-push-guard.sh" "$d/test-push-guard.sh"
|
|
printf '%s\n' "$d"
|
|
}
|
|
|
|
echo "== g1: a RED BASELINE must be refused, with NO table emitted =="
|
|
# The injected case asserts exit 99 from `true`. It fails always, and it changes
|
|
# NO guard behaviour -- which is the whole point: a defect anywhere in the suite
|
|
# used to be enough to certify every branch as covered.
|
|
g1="$(fixture red)"
|
|
python3 - "$g1/test-push-guard.sh" <<'PY'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
anchor = 'mkdir -p "$WORK_DIR"\n'
|
|
assert s.count(anchor) == 1, "injection anchor not unique -- fixture would not be the red baseline"
|
|
s = s.replace(anchor, anchor + '\nexpect NEEDLE 99 "injected always-failing case" -- true\n', 1)
|
|
open(p, "w").write(s)
|
|
PY
|
|
# Prove the fixture really IS red before asserting the generator notices, or g1
|
|
# could pass against a green suite and test nothing.
|
|
if "$g1/test-push-guard.sh" >/dev/null 2>&1; then
|
|
bad g1-fixture "injected suite still passes -- fixture is not a red baseline"
|
|
else
|
|
ok g1-fixture "fixture suite is red, as required"
|
|
fi
|
|
out="$("$GEN" --dir "$g1" 2>&1)"; rc=$?
|
|
if (( rc != 0 )) && [[ "$out" == *"REFUSING: baseline is not green"* ]]; then
|
|
ok g1-refused "exit $rc, refused before mutating"
|
|
else
|
|
bad g1-refused "wanted nonzero + refusal; got rc=$rc"
|
|
fi
|
|
if [[ "$out" != *"README TABLE"* && "$out" != *"killed,"* ]]; then
|
|
ok g1-no-table "no coverage table emitted from a red baseline"
|
|
else
|
|
bad g1-no-table "a table or kill count was published despite the red baseline"
|
|
fi
|
|
|
|
echo
|
|
echo "== g3: an INTERRUPTED run must not alter the subject =="
|
|
# THE KILL MUST LAND INSIDE A MUTATION WINDOW OR THIS CASE PROVES NOTHING.
|
|
# First attempt used `timeout -s KILL 3`. At three seconds the generator is still
|
|
# running its BASELINE, so no mutation has been applied yet and the subject is
|
|
# trivially unchanged -- for the ORIGINAL in-place generator too, which I
|
|
# confirmed by running it. The control passed for a reason unrelated to the fix:
|
|
# a vacuous control, in the control written for blocker 3.
|
|
#
|
|
# So the kill is now driven from INSIDE the run. The fixture's suite counts its
|
|
# own invocations and SIGKILLs the generator on the second one -- invocation 1 is
|
|
# the baseline, invocation 2 happens with mutant #1 APPLIED. That is exactly the
|
|
# window where an in-place generator strands a mutated subject.
|
|
instrument_kill_at_second_run() {
|
|
python3 - "$1" <<'PY'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
anchor = 'PASS=0\nFAIL=0\n'
|
|
assert s.count(anchor) == 1, "instrumentation anchor not unique"
|
|
inject = anchor + '''
|
|
if [[ -n "${G3_COUNTER:-}" ]]; then
|
|
n=$(( $(cat "$G3_COUNTER" 2>/dev/null || echo 0) + 1 ))
|
|
printf '%s' "$n" > "$G3_COUNTER"
|
|
# Invocation 2 = first mutant applied. Kill the generator where it hurts.
|
|
# `kill -9 0` targets the whole PROCESS GROUP, not $PPID: the generator runs
|
|
# the suite inside $( ), which forks, so $PPID is that subshell and killing
|
|
# it merely ends the command substitution -- the generator carries on and
|
|
# exits 0. The caller puts the generator in its OWN group via setsid, so the
|
|
# group is exactly the generator and its children, and this harness is not
|
|
# in it.
|
|
(( n == 2 )) && kill -9 0
|
|
fi
|
|
'''
|
|
open(p, "w").write(s.replace(anchor, inject, 1))
|
|
PY
|
|
}
|
|
|
|
# run_killed <dir> -> echoes "<rc> <before> <after>"
|
|
run_killed() {
|
|
local d="$1" gen="$2" before after rc
|
|
instrument_kill_at_second_run "$d/test-push-guard.sh"
|
|
before="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)"
|
|
G3_COUNTER="$d/.count" setsid --wait "$gen" --dir "$d" >/dev/null 2>&1; rc=$?
|
|
after="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)"
|
|
printf '%s %s %s\n' "$rc" "$before" "$after"
|
|
}
|
|
|
|
g3="$(fixture killed)"
|
|
read -r krc before after <<<"$(run_killed "$g3" "$GEN")"
|
|
if (( krc != 0 )); then
|
|
ok g3-was-killed "generator died mid-mutation (exit $krc)"
|
|
else
|
|
bad g3-was-killed "generator exited 0 -- the kill never landed, so the check below is vacuous"
|
|
fi
|
|
if [[ "$before" == "$after" ]]; then
|
|
ok g3-subject-intact "push-guard.sh byte-identical after the kill"
|
|
else
|
|
bad g3-subject-intact "SUBJECT MUTATED AND STRANDED: $before -> $after"
|
|
fi
|
|
|
|
# PROVE THE NEEDLE BITES. Reconstruct the pre-fix mechanism -- mutate the source
|
|
# in place, restore from an EXIT trap -- and put it through the identical kill.
|
|
# If this does NOT strand a mutated file, g3-subject-intact is measuring nothing
|
|
# and the isolation fix is unevidenced.
|
|
g3o="$(fixture killed-inplace)"
|
|
python3 - "$GEN" "$g3o/gen-inplace.sh" <<'PY'
|
|
import sys
|
|
s = open(sys.argv[1]).read()
|
|
old = '''install -m 755 "$SRC_TARGET" "$WORK/push-guard.sh"
|
|
install -m 755 "$SRC_SUITE" "$WORK/test-push-guard.sh"
|
|
TARGET="$WORK/push-guard.sh"
|
|
SUITE="$WORK/test-push-guard.sh"
|
|
BAK="$WORK/push-guard.sh.orig"
|
|
cp "$TARGET" "$BAK"'''
|
|
new = '''TARGET="$SRC_TARGET"
|
|
SUITE="$SRC_SUITE"
|
|
BAK="$WORK/push-guard.sh.orig"
|
|
cp "$TARGET" "$BAK"
|
|
trap 'cp "$BAK" "$TARGET"; rm -rf "$WORK"' EXIT'''
|
|
assert s.count(old) == 1, "cannot reconstruct the in-place mechanism -- bite proof would be fake"
|
|
open(sys.argv[2], "w").write(s.replace(old, new, 1))
|
|
PY
|
|
chmod +x "$g3o/gen-inplace.sh"
|
|
read -r _orc obefore oafter <<<"$(run_killed "$g3o" "$g3o/gen-inplace.sh")"
|
|
if [[ "$obefore" != "$oafter" ]]; then
|
|
ok g3-needle-bites "in-place generator strands a mutated subject, as it must"
|
|
else
|
|
bad g3-needle-bites "the OLD mechanism also left the subject intact -- g3 is vacuous"
|
|
fi
|
|
|
|
echo
|
|
echo "== g2: POSITIVE CONTROL -- a clean subject must produce a full green run =="
|
|
g2="$(fixture clean)"
|
|
out2="$("$GEN" --dir "$g2" 2>&1)"; rc2=$?
|
|
if (( rc2 == 0 )) && [[ "$out2" == *"0 survived"* ]]; then
|
|
ok g2-control "$(printf '%s' "$out2" | grep -E '^[0-9]+ killed')"
|
|
else
|
|
bad g2-control "wanted exit 0 with 0 survived; got rc=$rc2"
|
|
fi
|
|
# A kill must be attributed to a NAMED case, not to a bare tally -- that is the
|
|
# fix for blocker 2 and it needs its own assertion.
|
|
if [[ "$out2" == *" by: "* ]]; then
|
|
ok g2-attributed "kills name the case they broke"
|
|
else
|
|
bad g2-attributed "no per-mutant case attribution in the output"
|
|
fi
|
|
|
|
echo
|
|
printf '%d passed, %d failed\n' "$PASS" "$FAIL"
|
|
(( FAIL == 0 ))
|