stack/packages/mosaic/framework/tools/quality/scripts/test-check-test-enumeration.sh

#!/usr/bin/env bash
# test-check-test-enumeration.sh — needles for the enumeration guard (#1017).
#
# Every failure mode the guard promises gets BOTH polarities:
#   NEEDLE   a fixture that MUST trip the guard, asserted on the guard's OWN
#            words (--out) — exit 1 alone cannot distinguish "caught the rogue
#            file" from "choked on the fixture".
#   CONTROL  a fixture that MUST pass. A guard that failed unconditionally
#            would satisfy every needle here — the null-case defect the guard's
#            own subject matter (#1017) exists to make impossible.
#
# The needles encode the specific errors that produced #1017's thread:
#   n6 is the 20124 boundary file (a suite the strict prefix cannot name);
#   n2b proves surface 2 is PARSED, not line-ranged (three seats mis-scoped
#       hand-written ranges against ci.yml);
#   n5/n7 keep the exclusions file honest so it cannot become the next silent cap;
#   n8/c4 are F1 (20155): a commented-out ci.yml line is NOT enumeration —
#       commenting-out is the most common way a suite actually gets disabled,
#       and it must fail loud in one direction without false-staling the other.

set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GUARD="$HERE/check-test-enumeration.sh"
TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT
PASS=0; FAIL=0

# fixture <name> — a minimal repo root the guard accepts via --root:
# one suite enumerated on S1 (plus a naming-outlier suite, so the S1 parser's
# handling of non-prefix names is always exercised), one on S2, one check-script
# named on S2 that is outside the population, and an empty exclusions file.
fixture() {
    local r="$TMP/$1"
    mkdir -p "$r/packages/mosaic/framework/tools/git" \
             "$r/packages/mosaic/framework/tools/tmux" \
             "$r/packages/mosaic/framework/tools/quality/scripts" \
             "$r/.woodpecker"
    printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/git/test-a.sh"
    printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/tmux/outlier.test.sh"
    printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/test-ci.sh"
    printf '#!/usr/bin/env bash\nexit 0\n' > "$r/packages/mosaic/framework/tools/quality/scripts/verify-thing.sh"
    cat > "$r/packages/mosaic/package.json" <<'JSON'
{"scripts": {"test:framework-shell": "bash framework/tools/git/test-a.sh && bash framework/tools/tmux/outlier.test.sh"}}
JSON
    cat > "$r/.woodpecker/ci.yml" <<'YML'
steps:
  sanitize:
    commands:
      - bash packages/mosaic/framework/tools/quality/scripts/verify-thing.sh
  guard:
    commands:
      - bash packages/mosaic/framework/tools/quality/scripts/test-ci.sh
YML
    : > "$r/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt"
    printf '%s' "$r"
}

# expect <kind> <want-exit> <desc> [--out <substring>] -- <root>
expect() {
    local kind="$1" want="$2" desc="$3"; shift 3
    local need_out=""
    while (( $# )); do
        case "$1" in
            --out) need_out="$2"; shift 2 ;;
            --)    shift; break ;;
        esac
    done
    local root="$1" got=0 out
    out="$(bash "$GUARD" --root "$root" 2>&1)" || got=$?
    local why=""
    [[ "$got" == "$want" ]] || why="wanted exit $want, got $got"
    if [[ -z "$why" && -n "$need_out" && "$out" != *"$need_out"* ]]; then
        why="exit $got as expected, but output never said: $need_out"
    fi
    if [[ -z "$why" ]]; then
        printf '  PASS  [%-7s] %s (exit %s)\n' "$kind" "$desc" "$got"
        PASS=$(( PASS + 1 ))
    else
        printf '  FAIL  [%-7s] %s — %s\n' "$kind" "$desc" "$why"
        printf '%s\n' "$out" | sed 's/^/          | /'
        FAIL=$(( FAIL + 1 ))
    fi
}

excl() { printf '%s\n' "$2" >> "$1/packages/mosaic/framework/tools/quality/test-enumeration-exclusions.txt"; }

echo "=== c1: a fully consistent fixture passes ==="
R="$(fixture c1)"
expect CONTROL 0 "consistent tree: both surfaces enumerated, nothing unlisted" \
    --out "enumeration guard: OK" -- "$R"

echo "=== n1: an on-disk suite reachable from no surface must fail ==="
R="$(fixture n1)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
expect NEEDLE 1 "unlisted suite is named in the failure" \
    --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-rogue.sh'" -- "$R"

echo "=== n6: the 20124 boundary file — a suite the strict prefix cannot name ==="
R="$(fixture n6)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/rogue.test.sh"
expect NEEDLE 1 "naming-outlier suite (*.test.sh) is a population member, not invisible" \
    --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/rogue.test.sh'" -- "$R"

echo "=== c3: a non-suite script outside the pattern is outside it on BOTH sides ==="
R="$(fixture c3)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/check-unrelated.sh"
expect CONTROL 0 "check-script on disk, unlisted, outside population: not the guard's business" \
    --out "enumeration guard: OK" -- "$R"

echo "=== n2/n2b: a surface naming a path absent from disk must fail — both surfaces ==="
R="$(fixture n2)"
rm "$R/packages/mosaic/framework/tools/git/test-a.sh"
expect NEEDLE 1 "S1 (package.json) stale entry" \
    --out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/git/test-a.sh'" -- "$R"
R="$(fixture n2b)"
rm "$R/packages/mosaic/framework/tools/quality/scripts/test-ci.sh"
expect NEEDLE 1 "S2 (ci.yml) stale entry — proves ci.yml is parsed, not line-ranged" \
    --out "STALE ENUMERATION: surfaces name 'packages/mosaic/framework/tools/quality/scripts/test-ci.sh'" -- "$R"

echo "=== c2: a rogue suite with a SIGNED exclusion passes, and is counted ==="
R="$(fixture c2)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh | non-hermetic pending fixture work (needle-suite specimen)"
expect CONTROL 0 "signed exclusion is honoured and visible in the summary" \
    --out "excluded (signed) 1" -- "$R"

echo "=== n3: an exclusion with no reason is not a decision ==="
R="$(fixture n3)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh |   "
expect NEEDLE 1 "empty reason rejected" --out "EXCLUSION MISSING REASON" -- "$R"
R="$(fixture n3b)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-rogue.sh"
excl "$R" "packages/mosaic/framework/tools/git/test-rogue.sh"
expect NEEDLE 1 "missing separator rejected (the path alone is not a signature)" \
    --out "EXCLUSION MISSING REASON" -- "$R"

echo "=== n4: an exclusion whose path is gone is stale, not satisfied ==="
R="$(fixture n4)"
excl "$R" "packages/mosaic/framework/tools/git/test-vanished.sh | was excluded once, then deleted"
expect NEEDLE 1 "stale exclusion rejected" --out "STALE EXCLUSION" -- "$R"

echo "=== n5: excluding an enumerated suite is a contradiction, not belt-and-braces ==="
R="$(fixture n5)"
excl "$R" "packages/mosaic/framework/tools/git/test-a.sh | already in CI but excluded anyway"
expect NEEDLE 1 "contradictory exclusion rejected" --out "CONTRADICTORY EXCLUSION" -- "$R"

echo "=== n8/c4: a commented-out ci.yml line is not enumeration (F1, 20155) ==="
R="$(fixture n8)"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/tools/git/test-disabled.sh"
printf '      # - bash packages/mosaic/framework/tools/git/test-disabled.sh\n' >> "$R/.woodpecker/ci.yml"
expect NEEDLE 1 "suite named only in a commented-out invocation is UNENUMERATED" \
    --out "UNENUMERATED: 'packages/mosaic/framework/tools/git/test-disabled.sh'" -- "$R"
R="$(fixture c4)"
printf '      # - bash packages/mosaic/framework/tools/git/test-vanished.sh\n' >> "$R/.woodpecker/ci.yml"
expect CONTROL 0 "comment naming an absent path raises no false stale-enumeration" \
    --out "enumeration guard: OK" -- "$R"

echo "=== n7: excluding a file outside the population is dead weight, not coverage ==="
R="$(fixture n7)"
excl "$R" "packages/mosaic/framework/tools/quality/scripts/verify-thing.sh | not a suite but signing it anyway"
expect NEEDLE 1 "out-of-population exclusion rejected" --out "EXCLUSION OUTSIDE POPULATION" -- "$R"

echo "=== n9/c5: a suite in a SIBLING of tools/ is in the population (@scooby, 2026-08-16) ==="
# Every other fixture here lives under framework/tools/, which is how the guard came to
# scan TOOLS_DIR while claiming a population defined by basename alone. The real specimen
# was framework/systemd/user/test-fleet-units.sh: a member by the guard's own definition,
# on no CI surface, in no exclusion, and structurally unreachable by the scan. n9 is that
# blind spot; without it a future narrowing back to TOOLS_DIR passes all fourteen needles.
R="$(fixture n9)"
mkdir -p "$R/packages/mosaic/framework/systemd/user"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/systemd/user/test-sibling.sh"
expect NEEDLE 1 "suite outside tools/ but inside framework/ is enumerable, not invisible" \
    --out "UNENUMERATED: 'packages/mosaic/framework/systemd/user/test-sibling.sh'" -- "$R"
# c5 is why the S2 regex had to widen WITH the scan: detecting the file is useless if the
# fix for it cannot be recognised. Enumerating a sibling-directory suite on ci.yml must
# clear the finding — under a tools/-scoped S2 it stays UNENUMERATED forever and the only
# reachable disposition is an exclusion.
#
# Measured scope of what c5 catches, because it is narrower than it looks: against the
# ORIGINAL guard (both hunks absent) c5 passes vacuously — the scan never sees the file
# and S2 never matches it, so nothing is asserted. It discriminates against the HALF-patch
# — scan widened, S2 narrowed back — which is the realistic future regression, and it was
# confirmed red in exactly that state. n9 is the one that fails on the original.
R="$(fixture c5)"
mkdir -p "$R/packages/mosaic/framework/systemd/user"
printf '#!/usr/bin/env bash\nexit 0\n' > "$R/packages/mosaic/framework/systemd/user/test-sibling.sh"
printf '      - bash packages/mosaic/framework/systemd/user/test-sibling.sh\n' >> "$R/.woodpecker/ci.yml"
expect CONTROL 0 "enumerating a sibling-directory suite on ci.yml actually clears it" \
    --out "enumeration guard: OK" -- "$R"

echo
printf 'enumeration-guard needles: %d passed, %d failed\n' "$PASS" "$FAIL"
(( FAIL == 0 ))