Mechanical closure of 'a verification that passes when the thing it verifies never happened.'
300 lines
11 KiB
Bash
300 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
# test-push-guard.sh - Needle harness for push-guard.sh.
|
|
#
|
|
# Every check gets BOTH polarities:
|
|
# NEEDLE a deliberately-broken fixture that MUST trip the guard.
|
|
# CONTROL a clean fixture that MUST pass.
|
|
#
|
|
# The controls are not decoration. A guard that failed unconditionally would
|
|
# satisfy every needle and look fully covered — which is the same class of defect
|
|
# (an assertion satisfied by the null case) that push-guard.sh exists to prevent.
|
|
# A needle set without controls cannot tell "working guard" from "broken guard".
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
GUARD="$SCRIPT_DIR/push-guard.sh"
|
|
WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$SCRIPT_DIR/.work}"
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
# Fresh repo + bare "remote" for each case, so no case can inherit another's state.
|
|
new_repo() {
|
|
local name="$1"
|
|
local dir="$WORK_DIR/$name"
|
|
rm -rf "$dir"
|
|
mkdir -p "$dir"
|
|
git init -q -b main "$dir/repo"
|
|
git init -q --bare "$dir/remote.git"
|
|
git -C "$dir/repo" remote add origin "$dir/remote.git"
|
|
git -C "$dir/repo" config user.name "Needle Harness"
|
|
git -C "$dir/repo" config user.email "[email protected]"
|
|
printf 'seed\n' > "$dir/repo/seed.txt"
|
|
git -C "$dir/repo" add seed.txt
|
|
git -C "$dir/repo" commit -q -m "seed"
|
|
printf '%s' "$dir/repo"
|
|
}
|
|
|
|
# expect <kind> <expected_exit> <description> [--out <substring>] -- <command...>
|
|
#
|
|
# --out asserts a substring of the guard's own output. It exists because exit 0
|
|
# alone cannot distinguish "checked the files and they were fine" from "matched
|
|
# no files and had nothing to check". Two controls in an earlier revision of this
|
|
# harness were passing vacuously for exactly that reason — the pathspec silently
|
|
# matched nothing. A control that cannot fail is not a control.
|
|
expect() {
|
|
local kind="$1" want="$2" desc="$3"; shift 3
|
|
local need_out=""
|
|
while (( $# )); do
|
|
case "$1" in
|
|
--out) need_out="$2"; shift 2 ;;
|
|
--) shift; break ;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
local got=0 out
|
|
out="$("$@" 2>&1)" || got=$?
|
|
|
|
local why=""
|
|
[[ "$got" == "$want" ]] || why="wanted exit $want, got $got"
|
|
if [[ -z "$why" && -n "$need_out" && "$out" != *"$need_out"* ]]; then
|
|
why="exit $got as expected, but output never said: $need_out"
|
|
fi
|
|
|
|
if [[ -z "$why" ]]; then
|
|
printf ' PASS [%-7s] %s (exit %s)\n' "$kind" "$desc" "$got"
|
|
PASS=$((PASS + 1))
|
|
else
|
|
printf ' FAIL [%-7s] %s — %s\n' "$kind" "$desc" "$why"
|
|
printf '%s\n' "$out" | sed 's/^/ | /'
|
|
FAIL=$((FAIL + 1))
|
|
fi
|
|
}
|
|
|
|
rm -rf "$WORK_DIR"
|
|
mkdir -p "$WORK_DIR"
|
|
|
|
echo "=== (a) conflict markers / unmerged index ==="
|
|
|
|
# NEEDLE: staged content carrying real conflict markers.
|
|
R="$(new_repo a1)"
|
|
cat > "$R/conflicted.txt" <<'EOF'
|
|
intro line
|
|
<<<<<<< HEAD
|
|
ours
|
|
=======
|
|
theirs
|
|
>>>>>>> feature/x
|
|
tail line
|
|
EOF
|
|
git -C "$R" add conflicted.txt
|
|
expect NEEDLE 2 "staged conflict markers are refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# NEEDLE: a genuine unmerged index (merge in progress).
|
|
R="$(new_repo a2)"
|
|
git -C "$R" checkout -q -b other
|
|
printf 'from-other\n' > "$R/clash.txt"
|
|
git -C "$R" add clash.txt && git -C "$R" commit -q -m other
|
|
git -C "$R" checkout -q main
|
|
printf 'from-main\n' > "$R/clash.txt"
|
|
git -C "$R" add clash.txt && git -C "$R" commit -q -m main
|
|
git -C "$R" merge other -q >/dev/null 2>&1 || true
|
|
expect NEEDLE 2 "unmerged index paths are refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# CONTROL: ordinary staged content passes.
|
|
R="$(new_repo a3)"
|
|
printf 'just some code\n' > "$R/clean.txt"
|
|
git -C "$R" add clean.txt
|
|
expect CONTROL 0 "clean staged content passes (1 file actually scanned)" \
|
|
--out "no conflict markers in 1 staged file(s)" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# CONTROL (false-positive guard): prose using ======= as an underline must NOT trip.
|
|
# This is why the marker pattern deliberately ignores a bare '======='.
|
|
R="$(new_repo a4)"
|
|
cat > "$R/README.rst" <<'EOF'
|
|
Section Title
|
|
=============
|
|
|
|
Body text.
|
|
|
|
Another Heading
|
|
=======
|
|
EOF
|
|
git -C "$R" add README.rst
|
|
expect CONTROL 0 "prose using ======= underlines does not trip the guard" \
|
|
--out "no conflict markers in 1 staged file(s)" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# NEEDLE (fault injection): if the scan itself ERRORS, the guard must refuse
|
|
# rather than report a clean result. git grep exits 1 for "no match" but >=2 for
|
|
# a real failure; a blanket '|| true' would collapse the two. We inject the
|
|
# failure with a PATH shim rather than trying to corrupt an index.
|
|
R="$(new_repo a5)"
|
|
SHIM="$WORK_DIR/a5/bin"
|
|
mkdir -p "$SHIM"
|
|
REAL_GIT="$(command -v git)"
|
|
cat > "$SHIM/git" <<SH
|
|
#!/usr/bin/env bash
|
|
if [[ "\$1" == "grep" ]]; then exit 2; fi
|
|
exec "$REAL_GIT" "\$@"
|
|
SH
|
|
chmod +x "$SHIM/git"
|
|
printf 'ordinary content\n' > "$R/thing.txt"
|
|
git -C "$R" add thing.txt
|
|
expect NEEDLE 2 "a conflict scan that ERRORS is refused, not reported clean" \
|
|
--out "did not run" -- \
|
|
bash -c "cd '$R' && export PATH=\"$SHIM:\$PATH\" && '$GUARD' check-staged"
|
|
|
|
# NEEDLE: RENAME + MODIFY. Git reports a `git mv` plus a small edit as a single
|
|
# 'R' entry, which --diff-filter=ACM does not match, making the file invisible to
|
|
# every content check. A clean file is co-staged deliberately: without it the
|
|
# file list is empty and the nothing-staged guard fires by ACCIDENT, which would
|
|
# make this needle pass for the wrong reason and hide the real defect.
|
|
R="$(new_repo a6)"
|
|
mkdir -p "$R/data"
|
|
seq 1 200 | sed 's/^/line /' > "$R/data/canon.txt"
|
|
printf 'original\n' > "$R/other.txt"
|
|
git -C "$R" add -A && git -C "$R" commit -q -m seed
|
|
git -C "$R" mv data/canon.txt data/canon2.txt
|
|
printf '<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> b\n' >> "$R/data/canon2.txt"
|
|
printf 'an ordinary innocent change\n' > "$R/other.txt"
|
|
git -C "$R" add -A
|
|
expect NEEDLE 2 "conflict markers in a RENAMED file are refused (rename+modify)" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# NEEDLE: a path marked binary in .gitattributes. `git grep -I` honours that
|
|
# classification and skips the blob entirely while the summary still counts the
|
|
# file as scanned — a clean pass AND a false count. Hence -a, not -I.
|
|
R="$(new_repo a7)"
|
|
printf 'notes.txt binary\n' > "$R/.gitattributes"
|
|
printf 'x\n<<<<<<< HEAD\nours\n=======\ntheirs\n>>>>>>> b\ny\n' > "$R/notes.txt"
|
|
git -C "$R" add -A
|
|
expect NEEDLE 2 "conflict markers in a .gitattributes-binary file are refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
echo "=== (c) staged JSON parses ==="
|
|
|
|
# NEEDLE: malformed JSON, the shape a broken generator emits.
|
|
R="$(new_repo c1)"
|
|
mkdir -p "$R/data"
|
|
printf '{"a": 1,\n' > "$R/data/broken.json"
|
|
git -C "$R" add data/broken.json
|
|
expect NEEDLE 3 "staged unparseable JSON under --json-path is refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged --json-path 'data/**/*.json'"
|
|
|
|
# NEEDLE: conflict markers inside a generated JSON file — the 70-file incident.
|
|
R="$(new_repo c2)"
|
|
mkdir -p "$R/data"
|
|
cat > "$R/data/canon.json" <<'EOF'
|
|
{
|
|
<<<<<<< HEAD
|
|
"v": 1
|
|
=======
|
|
"v": 2
|
|
>>>>>>> main
|
|
}
|
|
EOF
|
|
git -C "$R" add data/canon.json
|
|
expect NEEDLE 2 "conflict markers in generated JSON are refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
# CONTROL: valid JSON passes.
|
|
R="$(new_repo c3)"
|
|
mkdir -p "$R/data"
|
|
printf '{"a": 1, "b": [2, 3]}\n' > "$R/data/good.json"
|
|
git -C "$R" add data/good.json
|
|
expect CONTROL 0 "valid staged JSON passes (and was actually parsed)" \
|
|
--out "1 staged .json file(s)" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged --json-path 'data/**/*.json'"
|
|
|
|
# CONTROL: an explicit exemption works (and is reported, never silent).
|
|
R="$(new_repo c4)"
|
|
mkdir -p "$R/fixtures"
|
|
printf 'not json at all' > "$R/fixtures/malformed.json"
|
|
git -C "$R" add fixtures/malformed.json
|
|
expect CONTROL 0 "deliberate malformed fixture can be exempted (exemption announced)" \
|
|
--out "EXEMPTED by --allow-invalid-json" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged --json-path 'fixtures/*' --allow-invalid-json 'fixtures/*'"
|
|
|
|
# CONTROL (false-positive regression): JSONC is the common real-world case.
|
|
# tsconfig.json legally carries comments and does NOT parse strictly. Measured on
|
|
# a real repo: 17 of 119 tracked .json files are like this. An on-by-default
|
|
# check would fire on all of them; scoping by --json-path is what prevents it.
|
|
R="$(new_repo c5)"
|
|
cat > "$R/tsconfig.json" <<'EOF'
|
|
{
|
|
// JSONC: comments are legal here and strict json.load() rejects them.
|
|
"compilerOptions": { "strict": true }
|
|
}
|
|
EOF
|
|
mkdir -p "$R/data"
|
|
printf '{"generated": true}\n' > "$R/data/view.json"
|
|
git -C "$R" add tsconfig.json data/view.json
|
|
expect CONTROL 0 "JSONC outside --json-path does not trip the JSON check" \
|
|
--out "1 staged .json file(s)" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged --json-path 'data/**/*.json'"
|
|
|
|
# NEEDLE: the same tsconfig DOES fail if someone wrongly nominates it, proving
|
|
# the check is genuinely running and the control above is not a vacuous pass.
|
|
expect NEEDLE 3 "the same JSONC file fails when explicitly nominated" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged --json-path 'tsconfig.json'"
|
|
|
|
echo "=== null case: nothing staged ==="
|
|
|
|
# NEEDLE: the index equals HEAD. Committing here produces the empty commit.
|
|
R="$(new_repo n1)"
|
|
expect NEEDLE 5 "empty index is refused before a commit happens" -- \
|
|
bash -c "cd '$R' && '$GUARD' check-staged"
|
|
|
|
echo "=== (b) push actually happened ==="
|
|
|
|
# CONTROL: a real push that moves the remote.
|
|
R="$(new_repo b1)"
|
|
printf 'work\n' > "$R/work.txt"
|
|
git -C "$R" add work.txt && git -C "$R" commit -q -m "real work"
|
|
expect CONTROL 0 "a push that moves the remote is confirmed" \
|
|
--out "PUSH CONFIRMED" -- \
|
|
bash -c "cd '$R' && '$GUARD' push --remote origin --branch main"
|
|
|
|
# NEEDLE: THE ORIGINAL FALSE POSITIVE. Remote already equals local HEAD, so the
|
|
# naive 'remote == local' assertion succeeds while nothing is transferred.
|
|
R="$(new_repo b2)"
|
|
printf 'work\n' > "$R/work.txt"
|
|
git -C "$R" add work.txt && git -C "$R" commit -q -m "real work"
|
|
git -C "$R" push -q origin HEAD:refs/heads/main
|
|
expect NEEDLE 4 "second push transferring nothing is refused (remote did not move)" -- \
|
|
bash -c "cd '$R' && '$GUARD' push --remote origin --branch main"
|
|
|
|
# NEEDLE: the commit step aborted, so HEAD never advanced.
|
|
R="$(new_repo b3)"
|
|
HEAD_BEFORE="$(git -C "$R" rev-parse HEAD)"
|
|
expect NEEDLE 5 "push after an aborted commit is refused (HEAD did not advance)" -- \
|
|
bash -c "cd '$R' && '$GUARD' push --remote origin --branch main --since-head '$HEAD_BEFORE'"
|
|
|
|
# NEEDLE: an empty commit carrying a real message.
|
|
R="$(new_repo b4)"
|
|
git -C "$R" commit -q --allow-empty -m "feat: important-sounding message"
|
|
expect NEEDLE 5 "pushing an empty commit is refused" -- \
|
|
bash -c "cd '$R' && '$GUARD' push --remote origin --branch main"
|
|
|
|
# CONTROL: --since-head is satisfied when a real commit was made.
|
|
R="$(new_repo b5)"
|
|
HEAD_BEFORE="$(git -C "$R" rev-parse HEAD)"
|
|
printf 'work\n' > "$R/work.txt"
|
|
git -C "$R" add work.txt && git -C "$R" commit -q -m "real work"
|
|
# --out is required here. Without it this control is VACUOUS: deleting the whole
|
|
# --since-head validation block would still yield exit 0, because the empty-commit
|
|
# and remote-moved checks independently succeed. It would prove nothing about the
|
|
# code path it names.
|
|
expect CONTROL 0 "--since-head passes when a real commit was created" \
|
|
--out "HEAD advanced" -- \
|
|
bash -c "cd '$R' && '$GUARD' push --remote origin --branch main --since-head '$HEAD_BEFORE'"
|
|
|
|
echo
|
|
printf 'push-guard needles: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
(( FAIL == 0 )) || exit 1
|