installer-7andClaude Opus 5
8fdc8738ed
ci/woodpecker/pr/ci Pipeline was successful
Authored by installer-7; committed by mos-claude (no credential for this remote). push-guard.sh and test-push-guard.sh are BYTE-IDENTICAL to the previously cleared versions — every change this round is in the harness, confirming the reviewer's framing that none of the three blockers was in the guard itself. B1 verify-clean-clone.sh could not verify the artifact in its real monorepo location: it resolved ROOT but kept artifacts as bare basenames, so running it in place reported all artifacts NOT TRACKED. Its own suite missed this because every fixture installed artifacts at fixture ROOT — a fixture encoding a layout the real subject does not have. PREFIX now comes from 'git rev-parse --show-prefix' and is threaded through the ls-tree pathspec, the cloned stat, and the suite cwd; the verifier PRINTS the prefix. Three needles: nested-layout pass, prefix-reported (else a green only means the prefix was ignored harmlessly), and mode-needle-still- bites-nested (a prefix threaded into the clone but not ls-tree would silently stop checking modes). B2 the generator reported full coverage and exited 0 on a RED baseline — any pre-existing failure marked every mutant killed. Now refuses unless baseline is exit 0 with zero failures, prints the actual tally on refusal, emits no table, and scores kills by NAMED DELTA rather than a raw red count. B3 the generator mutated the reviewed source in place; SIGKILL stranded a mutant and contaminated a following run. Guard and suite are now copied into a temp dir and mutations apply to that copy — no restore step to fail. The author's first control for this was itself vacuous (a 3s kill lands during baseline, before any mutation, so it passed against the unfixed mechanism too); the real control drives the kill from inside the run on the second suite invocation, plus a needle proving the reconstructed pre-fix mechanism DOES strand. Corrections: 'shellcheck clean' had been measured at -S warning and published unqualified — a filtered measurement stated as an unfiltered claim. Now clean at DEFAULT severity across six files with one scoped, documented SC2016 disable. ARTIFACTS extended five -> six so the test files no longer omit themselves. Verified before commit: six files sha256-matched to the author's hashes; shellcheck exit 0 at default severity; push-guard 46/46 and verifier 9/9 run directly; the in-place verifier resolved the real nested prefix against this tree. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01YKj59Qadrb2WBLaePvkM7H
179 lines
7.6 KiB
Bash
Executable File
179 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-mutate-push-guard.sh -- needles for the mutation generator.
|
|
#
|
|
# The generator's entire output is a COVERAGE CLAIM, and the README publishes it.
|
|
# That makes it the most dangerous file here: when it is wrong it does not fail,
|
|
# it reassures. Two of its three defects were found by review rather than by any
|
|
# test, so these are the cases that had to exist.
|
|
#
|
|
# g1 a RED BASELINE must be refused before any mutant runs. Previously ONE
|
|
# pre-existing suite failure -- changing no guard behaviour at all --
|
|
# satisfied every mutant: 13 killed, 0 survived, table emitted, exit 0.
|
|
# g3 an INTERRUPTED run must leave the subject byte-identical. Restoration
|
|
# used to lean on an EXIT trap, and A TRAP IS CLEANUP, NOT ISOLATION:
|
|
# SIGKILL cannot run it, so an interrupted run stranded a mutated
|
|
# push-guard.sh that the next suite run silently inherited.
|
|
# g2 is the positive control. Without it every case here could be passing
|
|
# because the generator refuses unconditionally, which is a wall, not a gate.
|
|
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
GEN="$HERE/mutate-push-guard.sh"
|
|
PASS=0; FAIL=0
|
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
|
|
ok() { printf ' ok [%-16s] %s\n' "$1" "${2:-}"; PASS=$(( PASS + 1 )); }
|
|
bad() { printf ' FAIL [%-16s] %s\n' "$1" "$2"; FAIL=$(( FAIL + 1 )); }
|
|
|
|
# fixture <name> -- a directory holding an independent copy of guard + suite
|
|
fixture() {
|
|
local d="$TMP/$1"; mkdir -p "$d"
|
|
install -m 755 "$HERE/push-guard.sh" "$d/push-guard.sh"
|
|
install -m 755 "$HERE/test-push-guard.sh" "$d/test-push-guard.sh"
|
|
printf '%s\n' "$d"
|
|
}
|
|
|
|
echo "== g1: a RED BASELINE must be refused, with NO table emitted =="
|
|
# The injected case asserts exit 99 from `true`. It fails always, and it changes
|
|
# NO guard behaviour -- which is the whole point: a defect anywhere in the suite
|
|
# used to be enough to certify every branch as covered.
|
|
g1="$(fixture red)"
|
|
python3 - "$g1/test-push-guard.sh" <<'PY'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
anchor = 'mkdir -p "$WORK_DIR"\n'
|
|
assert s.count(anchor) == 1, "injection anchor not unique -- fixture would not be the red baseline"
|
|
s = s.replace(anchor, anchor + '\nexpect NEEDLE 99 "injected always-failing case" -- true\n', 1)
|
|
open(p, "w").write(s)
|
|
PY
|
|
# Prove the fixture really IS red before asserting the generator notices, or g1
|
|
# could pass against a green suite and test nothing.
|
|
if "$g1/test-push-guard.sh" >/dev/null 2>&1; then
|
|
bad g1-fixture "injected suite still passes -- fixture is not a red baseline"
|
|
else
|
|
ok g1-fixture "fixture suite is red, as required"
|
|
fi
|
|
out="$("$GEN" --dir "$g1" 2>&1)"; rc=$?
|
|
if (( rc != 0 )) && [[ "$out" == *"REFUSING: baseline is not green"* ]]; then
|
|
ok g1-refused "exit $rc, refused before mutating"
|
|
else
|
|
bad g1-refused "wanted nonzero + refusal; got rc=$rc"
|
|
fi
|
|
if [[ "$out" != *"README TABLE"* && "$out" != *"killed,"* ]]; then
|
|
ok g1-no-table "no coverage table emitted from a red baseline"
|
|
else
|
|
bad g1-no-table "a table or kill count was published despite the red baseline"
|
|
fi
|
|
|
|
echo
|
|
echo "== g3: an INTERRUPTED run must not alter the subject =="
|
|
# THE KILL MUST LAND INSIDE A MUTATION WINDOW OR THIS CASE PROVES NOTHING.
|
|
# First attempt used `timeout -s KILL 3`. At three seconds the generator is still
|
|
# running its BASELINE, so no mutation has been applied yet and the subject is
|
|
# trivially unchanged -- for the ORIGINAL in-place generator too, which I
|
|
# confirmed by running it. The control passed for a reason unrelated to the fix:
|
|
# a vacuous control, in the control written for blocker 3.
|
|
#
|
|
# So the kill is now driven from INSIDE the run. The fixture's suite counts its
|
|
# own invocations and SIGKILLs the generator on the second one -- invocation 1 is
|
|
# the baseline, invocation 2 happens with mutant #1 APPLIED. That is exactly the
|
|
# window where an in-place generator strands a mutated subject.
|
|
instrument_kill_at_second_run() {
|
|
python3 - "$1" <<'PY'
|
|
import sys
|
|
p = sys.argv[1]; s = open(p).read()
|
|
anchor = 'PASS=0\nFAIL=0\n'
|
|
assert s.count(anchor) == 1, "instrumentation anchor not unique"
|
|
inject = anchor + '''
|
|
if [[ -n "${G3_COUNTER:-}" ]]; then
|
|
n=$(( $(cat "$G3_COUNTER" 2>/dev/null || echo 0) + 1 ))
|
|
printf '%s' "$n" > "$G3_COUNTER"
|
|
# Invocation 2 = first mutant applied. Kill the generator where it hurts.
|
|
# `kill -9 0` targets the whole PROCESS GROUP, not $PPID: the generator runs
|
|
# the suite inside $( ), which forks, so $PPID is that subshell and killing
|
|
# it merely ends the command substitution -- the generator carries on and
|
|
# exits 0. The caller puts the generator in its OWN group via setsid, so the
|
|
# group is exactly the generator and its children, and this harness is not
|
|
# in it.
|
|
(( n == 2 )) && kill -9 0
|
|
fi
|
|
'''
|
|
open(p, "w").write(s.replace(anchor, inject, 1))
|
|
PY
|
|
}
|
|
|
|
# run_killed <dir> -> echoes "<rc> <before> <after>"
|
|
run_killed() {
|
|
local d="$1" gen="$2" before after rc
|
|
instrument_kill_at_second_run "$d/test-push-guard.sh"
|
|
before="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)"
|
|
G3_COUNTER="$d/.count" setsid --wait "$gen" --dir "$d" >/dev/null 2>&1; rc=$?
|
|
after="$(sha256sum "$d/push-guard.sh" | cut -d' ' -f1)"
|
|
printf '%s %s %s\n' "$rc" "$before" "$after"
|
|
}
|
|
|
|
g3="$(fixture killed)"
|
|
read -r krc before after <<<"$(run_killed "$g3" "$GEN")"
|
|
if (( krc != 0 )); then
|
|
ok g3-was-killed "generator died mid-mutation (exit $krc)"
|
|
else
|
|
bad g3-was-killed "generator exited 0 -- the kill never landed, so the check below is vacuous"
|
|
fi
|
|
if [[ "$before" == "$after" ]]; then
|
|
ok g3-subject-intact "push-guard.sh byte-identical after the kill"
|
|
else
|
|
bad g3-subject-intact "SUBJECT MUTATED AND STRANDED: $before -> $after"
|
|
fi
|
|
|
|
# PROVE THE NEEDLE BITES. Reconstruct the pre-fix mechanism -- mutate the source
|
|
# in place, restore from an EXIT trap -- and put it through the identical kill.
|
|
# If this does NOT strand a mutated file, g3-subject-intact is measuring nothing
|
|
# and the isolation fix is unevidenced.
|
|
g3o="$(fixture killed-inplace)"
|
|
python3 - "$GEN" "$g3o/gen-inplace.sh" <<'PY'
|
|
import sys
|
|
s = open(sys.argv[1]).read()
|
|
old = '''install -m 755 "$SRC_TARGET" "$WORK/push-guard.sh"
|
|
install -m 755 "$SRC_SUITE" "$WORK/test-push-guard.sh"
|
|
TARGET="$WORK/push-guard.sh"
|
|
SUITE="$WORK/test-push-guard.sh"
|
|
BAK="$WORK/push-guard.sh.orig"
|
|
cp "$TARGET" "$BAK"'''
|
|
new = '''TARGET="$SRC_TARGET"
|
|
SUITE="$SRC_SUITE"
|
|
BAK="$WORK/push-guard.sh.orig"
|
|
cp "$TARGET" "$BAK"
|
|
trap 'cp "$BAK" "$TARGET"; rm -rf "$WORK"' EXIT'''
|
|
assert s.count(old) == 1, "cannot reconstruct the in-place mechanism -- bite proof would be fake"
|
|
open(sys.argv[2], "w").write(s.replace(old, new, 1))
|
|
PY
|
|
chmod +x "$g3o/gen-inplace.sh"
|
|
read -r _orc obefore oafter <<<"$(run_killed "$g3o" "$g3o/gen-inplace.sh")"
|
|
if [[ "$obefore" != "$oafter" ]]; then
|
|
ok g3-needle-bites "in-place generator strands a mutated subject, as it must"
|
|
else
|
|
bad g3-needle-bites "the OLD mechanism also left the subject intact -- g3 is vacuous"
|
|
fi
|
|
|
|
echo
|
|
echo "== g2: POSITIVE CONTROL -- a clean subject must produce a full green run =="
|
|
g2="$(fixture clean)"
|
|
out2="$("$GEN" --dir "$g2" 2>&1)"; rc2=$?
|
|
if (( rc2 == 0 )) && [[ "$out2" == *"0 survived"* ]]; then
|
|
ok g2-control "$(printf '%s' "$out2" | grep -E '^[0-9]+ killed')"
|
|
else
|
|
bad g2-control "wanted exit 0 with 0 survived; got rc=$rc2"
|
|
fi
|
|
# A kill must be attributed to a NAMED case, not to a bare tally -- that is the
|
|
# fix for blocker 2 and it needs its own assertion.
|
|
if [[ "$out2" == *" by: "* ]]; then
|
|
ok g2-attributed "kills name the case they broke"
|
|
else
|
|
bad g2-attributed "no per-mutant case attribution in the output"
|
|
fi
|
|
|
|
echo
|
|
printf '%d passed, %d failed\n' "$PASS" "$FAIL"
|
|
(( FAIL == 0 ))
|