#!/usr/bin/env bash # test-check-test-enumeration.sh — needles for the enumeration guard (#1017). # # Every failure mode the guard promises gets BOTH polarities: # NEEDLE a fixture that MUST trip the guard, asserted on the guard's OWN # words (--out) — exit 1 alone cannot distinguish "caught the rogue # file" from "choked on the fixture". # CONTROL a fixture that MUST pass. A guard that failed unconditionally # would satisfy every needle here — the null-case defect the guard's # own subject matter (#1017) exists to make impossible. # # The needles encode the specific errors that produced #1017's thread: # n6 is the 20124 boundary file (a suite the strict prefix cannot name); # n2b proves surface 2 is PARSED, not line-ranged (three seats mis-scoped # hand-written ranges against ci.yml); # n5/n7 keep the exclusions file honest so it cannot become the next silent cap; # n8/c4 are F1 (20155): a commented-out ci.yml line is NOT enumeration — # commenting-out is the most common way a suite actually gets disabled, # and it must fail loud in one direction without false-staling the other. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GUARD="$HERE/check-test-enumeration.sh" TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT PASS=0; FAIL=0 # fixture — a minimal repo root the guard accepts via --root: # one suite enumerated on S1 (plus a naming-outlier suite, so the S1 parser's # handling of non-prefix names is always exercised), one on S2, one check-script # named on S2 that is outside the population, and an empty exclusions file. fixture() { local r="$TMP/$1" mkdir -p "$r/packages/mosaic/framework/tools/git" \ "$r/packages/mosaic/framework/tools/tmux" \ "$r/packages/mosaic/framework/tools/quality/scripts" \ "$r/.woodpecker" printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/git/test-a.sh" printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/tmux/outlier.test.sh" printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/test-ci.sh" printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/verify-thing.sh" cat > "$r/packages/mosaic/package.json" <<'JSON' {"scripts": {"test:framework-shell": "bash framework/tools/git/test-a.sh && bash framework/tools/tmux/outlier.test.sh"}} JSON cat > "$r/.woodpecker/ci.yml" <<'YML' steps: sanitize: commands: - bash packages/mosaic/framework/tools/quality/scripts/verify-thing.sh guard: commands: - bash packages/mosaic/framework/tools/quality/scripts/test-ci.sh YML : > "$r/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt" printf '%s' "$r" } # expect [--out ] -- expect() { local kind="$1" want="$2" desc="$3"; shift 3 local need_out="" while (( $# )); do case "$1" in --out) need_out="$2"; shift 2 ;; --) shift; break ;; esac done local root="$1" got=0 out out="$(bash "$GUARD" --root "$root" 2>&1)" || got=$? local why="" [[ "$got" == "$want" ]] || why="wanted exit $want, got $got" if [[ -z "$why" && -n "$need_out" && "$out" != *"$need_out"* ]]; then why="exit $got as expected, but output never said: $need_out" fi if [[ -z "$why" ]]; then printf ' PASS [%-7s] %s (exit %s)\n' "$kind" "$desc" "$got" PASS=$(( PASS + 1 )) else printf ' FAIL [%-7s] %s — %s\n' "$kind" "$desc" "$why" printf '%s\n' "$out" | sed 's/^/ | /' FAIL=$(( FAIL + 1 )) fi } excl() { printf '%s\n' "$2" >> "$1/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt"; } echo "=== c1: a fully consistent fixture passes ===" R="$(fixture c1)" expect CONTROL 0 "consistent tree: both surfaces enumerated, nothing unlisted" \ --out "enumeration guard: OK" -- "$R" echo "=== n1: an on-disk suite reachable from no surface must fail ===" R="$(fixture n1)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh" expect NEEDLE 1 "unlisted suite is named in the failure" \ --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-rogue.sh'" -- "$R" echo "=== n6: the 20124 boundary file — a suite the strict prefix cannot name ===" R="$(fixture n6)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/rogue.test.sh" expect NEEDLE 1 "naming-outlier suite (*.test.sh) is a population member, not invisible" \ --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/rogue.test.sh'" -- "$R" echo "=== c3: a non-suite script outside the pattern is outside it on BOTH sides ===" R="$(fixture c3)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/check-unrelated.sh" expect CONTROL 0 "check-script on disk, unlisted, outside population: not the guard's business" \ --out "enumeration guard: OK" -- "$R" echo "=== n2/n2b: a surface naming a path absent from disk must fail — both surfaces ===" R="$(fixture n2)" rm "$R/packages/mosaic/framework/tools/git/test-a.sh" expect NEEDLE 1 "S1 (package.json) stale entry" \ --out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/git/test-a.sh'" -- "$R" R="$(fixture n2b)" rm "$R/packages/mosaic/framework/tools/quality/scripts/test-ci.sh" expect NEEDLE 1 "S2 (ci.yml) stale entry — proves ci.yml is parsed, not line-ranged" \ --out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/quality/scripts/test-ci.sh'" -- "$R" echo "=== c2: a rogue suite with a SIGNED exclusion passes, and is counted ===" R="$(fixture c2)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh" excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh | non-hermetic pending fixture work (needle-suite specimen)" expect CONTROL 0 "signed exclusion is honoured and visible in the summary" \ --out "excluded (signed) 1" -- "$R" echo "=== n3: an exclusion with no reason is not a decision ===" R="$(fixture n3)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh" excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh | " expect NEEDLE 1 "empty reason rejected" --out "EXCLUSION MISSING REASON" -- "$R" R="$(fixture n3b)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh" excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh" expect NEEDLE 1 "missing separator rejected (the path alone is not a signature)" \ --out "EXCLUSION MISSING REASON" -- "$R" echo "=== n4: an exclusion whose path is gone is stale, not satisfied ===" R="$(fixture n4)" excl "$R" "packages/mosaic/framework/tools/git/test-vanished.sh | was excluded once, then deleted" expect NEEDLE 1 "stale exclusion rejected" --out "STALE EXCLUSION" -- "$R" echo "=== n5: excluding an enumerated suite is a contradiction, not belt-and-braces ===" R="$(fixture n5)" excl "$R" "packages/mosaic/framework/tools/git/test-a.sh | already in CI but excluded anyway" expect NEEDLE 1 "contradictory exclusion rejected" --out "CONTRADICTORY EXCLUSION" -- "$R" echo "=== n8/c4: a commented-out ci.yml line is not enumeration (F1, 20155) ===" R="$(fixture n8)" printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-disabled.sh" printf ' # - bash packages/mosaic/framework/tools/git/test-disabled.sh\n' >> "$R/.woodpecker/ci.yml" expect NEEDLE 1 "suite named only in a commented-out invocation is UNENUMERATED" \ --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-disabled.sh'" -- "$R" R="$(fixture c4)" printf ' # - bash packages/mosaic/framework/tools/git/test-vanished.sh\n' >> "$R/.woodpecker/ci.yml" expect CONTROL 0 "comment naming an absent path raises no false stale-enumeration" \ --out "enumeration guard: OK" -- "$R" echo "=== n7: excluding a file outside the population is dead weight, not coverage ===" R="$(fixture n7)" excl "$R" "packages/mosaic/framework/tools/quality/scripts/verify-thing.sh | not a suite but signing it anyway" expect NEEDLE 1 "out-of-population exclusion rejected" --out "EXCLUSION OUTSIDE POPULATION" -- "$R" echo printf 'enumeration-guard needles: %d passed, %d failed\n' "$PASS" "$FAIL" (( FAIL == 0 ))