167 lines
8.3 KiB
Bash
Executable File
167 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-check-test-enumeration.sh — needles for the enumeration guard (#1017).
|
|
#
|
|
# Every failure mode the guard promises gets BOTH polarities:
|
|
# NEEDLE a fixture that MUST trip the guard, asserted on the guard's OWN
|
|
# words (--out) — exit 1 alone cannot distinguish "caught the rogue
|
|
# file" from "choked on the fixture".
|
|
# CONTROL a fixture that MUST pass. A guard that failed unconditionally
|
|
# would satisfy every needle here — the null-case defect the guard's
|
|
# own subject matter (#1017) exists to make impossible.
|
|
#
|
|
# The needles encode the specific errors that produced #1017's thread:
|
|
# n6 is the 20124 boundary file (a suite the strict prefix cannot name);
|
|
# n2b proves surface 2 is PARSED, not line-ranged (three seats mis-scoped
|
|
# hand-written ranges against ci.yml);
|
|
# n5/n7 keep the exclusions file honest so it cannot become the next silent cap;
|
|
# n8/c4 are F1 (20155): a commented-out ci.yml line is NOT enumeration —
|
|
# commenting-out is the most common way a suite actually gets disabled,
|
|
# and it must fail loud in one direction without false-staling the other.
|
|
|
|
set -uo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
GUARD="$HERE/check-test-enumeration.sh"
|
|
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
|
|
PASS=0; FAIL=0
|
|
|
|
# fixture <name> — a minimal repo root the guard accepts via --root:
|
|
# one suite enumerated on S1 (plus a naming-outlier suite, so the S1 parser's
|
|
# handling of non-prefix names is always exercised), one on S2, one check-script
|
|
# named on S2 that is outside the population, and an empty exclusions file.
|
|
fixture() {
|
|
local r="$TMP/$1"
|
|
mkdir -p "$r/packages/mosaic/framework/tools/git" \
|
|
"$r/packages/mosaic/framework/tools/tmux" \
|
|
"$r/packages/mosaic/framework/tools/quality/scripts" \
|
|
"$r/.woodpecker"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/git/test-a.sh"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/tmux/outlier.test.sh"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/test-ci.sh"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/verify-thing.sh"
|
|
cat > "$r/packages/mosaic/package.json" <<'JSON'
|
|
{"scripts": {"test:framework-shell": "bash framework/tools/git/test-a.sh && bash framework/tools/tmux/outlier.test.sh"}}
|
|
JSON
|
|
cat > "$r/.woodpecker/ci.yml" <<'YML'
|
|
steps:
|
|
sanitize:
|
|
commands:
|
|
- bash packages/mosaic/framework/tools/quality/scripts/verify-thing.sh
|
|
guard:
|
|
commands:
|
|
- bash packages/mosaic/framework/tools/quality/scripts/test-ci.sh
|
|
YML
|
|
: > "$r/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt"
|
|
printf '%s' "$r"
|
|
}
|
|
|
|
# expect <kind> <want-exit> <desc> [--out <substring>] -- <root>
|
|
expect() {
|
|
local kind="$1" want="$2" desc="$3"; shift 3
|
|
local need_out=""
|
|
while (( $# )); do
|
|
case "$1" in
|
|
--out) need_out="$2"; shift 2 ;;
|
|
--) shift; break ;;
|
|
esac
|
|
done
|
|
local root="$1" got=0 out
|
|
out="$(bash "$GUARD" --root "$root" 2>&1)" || got=$?
|
|
local why=""
|
|
[[ "$got" == "$want" ]] || why="wanted exit $want, got $got"
|
|
if [[ -z "$why" && -n "$need_out" && "$out" != *"$need_out"* ]]; then
|
|
why="exit $got as expected, but output never said: $need_out"
|
|
fi
|
|
if [[ -z "$why" ]]; then
|
|
printf ' PASS [%-7s] %s (exit %s)\n' "$kind" "$desc" "$got"
|
|
PASS=$(( PASS + 1 ))
|
|
else
|
|
printf ' FAIL [%-7s] %s — %s\n' "$kind" "$desc" "$why"
|
|
printf '%s\n' "$out" | sed 's/^/ | /'
|
|
FAIL=$(( FAIL + 1 ))
|
|
fi
|
|
}
|
|
|
|
excl() { printf '%s\n' "$2" >> "$1/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt"; }
|
|
|
|
echo "=== c1: a fully consistent fixture passes ==="
|
|
R="$(fixture c1)"
|
|
expect CONTROL 0 "consistent tree: both surfaces enumerated, nothing unlisted" \
|
|
--out "enumeration guard: OK" -- "$R"
|
|
|
|
echo "=== n1: an on-disk suite reachable from no surface must fail ==="
|
|
R="$(fixture n1)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
|
|
expect NEEDLE 1 "unlisted suite is named in the failure" \
|
|
--out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-rogue.sh'" -- "$R"
|
|
|
|
echo "=== n6: the 20124 boundary file — a suite the strict prefix cannot name ==="
|
|
R="$(fixture n6)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/rogue.test.sh"
|
|
expect NEEDLE 1 "naming-outlier suite (*.test.sh) is a population member, not invisible" \
|
|
--out "UNENUMERATED: 'packages/mosaic/framework/tools/git/rogue.test.sh'" -- "$R"
|
|
|
|
echo "=== c3: a non-suite script outside the pattern is outside it on BOTH sides ==="
|
|
R="$(fixture c3)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/check-unrelated.sh"
|
|
expect CONTROL 0 "check-script on disk, unlisted, outside population: not the guard's business" \
|
|
--out "enumeration guard: OK" -- "$R"
|
|
|
|
echo "=== n2/n2b: a surface naming a path absent from disk must fail — both surfaces ==="
|
|
R="$(fixture n2)"
|
|
rm "$R/packages/mosaic/framework/tools/git/test-a.sh"
|
|
expect NEEDLE 1 "S1 (package.json) stale entry" \
|
|
--out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/git/test-a.sh'" -- "$R"
|
|
R="$(fixture n2b)"
|
|
rm "$R/packages/mosaic/framework/tools/quality/scripts/test-ci.sh"
|
|
expect NEEDLE 1 "S2 (ci.yml) stale entry — proves ci.yml is parsed, not line-ranged" \
|
|
--out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/quality/scripts/test-ci.sh'" -- "$R"
|
|
|
|
echo "=== c2: a rogue suite with a SIGNED exclusion passes, and is counted ==="
|
|
R="$(fixture c2)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
|
|
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh | non-hermetic pending fixture work (needle-suite specimen)"
|
|
expect CONTROL 0 "signed exclusion is honoured and visible in the summary" \
|
|
--out "excluded (signed) 1" -- "$R"
|
|
|
|
echo "=== n3: an exclusion with no reason is not a decision ==="
|
|
R="$(fixture n3)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
|
|
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh | "
|
|
expect NEEDLE 1 "empty reason rejected" --out "EXCLUSION MISSING REASON" -- "$R"
|
|
R="$(fixture n3b)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
|
|
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh"
|
|
expect NEEDLE 1 "missing separator rejected (the path alone is not a signature)" \
|
|
--out "EXCLUSION MISSING REASON" -- "$R"
|
|
|
|
echo "=== n4: an exclusion whose path is gone is stale, not satisfied ==="
|
|
R="$(fixture n4)"
|
|
excl "$R" "packages/mosaic/framework/tools/git/test-vanished.sh | was excluded once, then deleted"
|
|
expect NEEDLE 1 "stale exclusion rejected" --out "STALE EXCLUSION" -- "$R"
|
|
|
|
echo "=== n5: excluding an enumerated suite is a contradiction, not belt-and-braces ==="
|
|
R="$(fixture n5)"
|
|
excl "$R" "packages/mosaic/framework/tools/git/test-a.sh | already in CI but excluded anyway"
|
|
expect NEEDLE 1 "contradictory exclusion rejected" --out "CONTRADICTORY EXCLUSION" -- "$R"
|
|
|
|
echo "=== n8/c4: a commented-out ci.yml line is not enumeration (F1, 20155) ==="
|
|
R="$(fixture n8)"
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-disabled.sh"
|
|
printf ' # - bash packages/mosaic/framework/tools/git/test-disabled.sh\n' >> "$R/.woodpecker/ci.yml"
|
|
expect NEEDLE 1 "suite named only in a commented-out invocation is UNENUMERATED" \
|
|
--out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-disabled.sh'" -- "$R"
|
|
R="$(fixture c4)"
|
|
printf ' # - bash packages/mosaic/framework/tools/git/test-vanished.sh\n' >> "$R/.woodpecker/ci.yml"
|
|
expect CONTROL 0 "comment naming an absent path raises no false stale-enumeration" \
|
|
--out "enumeration guard: OK" -- "$R"
|
|
|
|
echo "=== n7: excluding a file outside the population is dead weight, not coverage ==="
|
|
R="$(fixture n7)"
|
|
excl "$R" "packages/mosaic/framework/tools/quality/scripts/verify-thing.sh | not a suite but signing it anyway"
|
|
expect NEEDLE 1 "out-of-population exclusion rejected" --out "EXCLUSION OUTSIDE POPULATION" -- "$R"
|
|
|
|
echo
|
|
printf 'enumeration-guard needles: %d passed, %d failed\n' "$PASS" "$FAIL"
|
|
(( FAIL == 0 ))
|