166 lines
7.7 KiB
Bash
Executable File
166 lines
7.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# check-test-enumeration.sh — CI test-membership guard (#1017).
|
|
#
|
|
# CI reaches shell suites through two hand-enumerated surfaces:
|
|
# S1 packages/mosaic/package.json scripts."test:framework-shell"
|
|
# S2 .woodpecker/ci.yml direct `bash packages/mosaic/framework/tools/...` commands
|
|
#
|
|
# A hand-enumerated allowlist re-arms its own gap: a new suite never auto-joins,
|
|
# so the list silently under-runs the disk (17 of 39 suites were invisible when
|
|
# #1017 was filed). This guard makes that under-run impossible to do silently:
|
|
#
|
|
# FAIL when a suite-shaped file exists on disk and is neither enumerated on
|
|
# the UNION of both surfaces nor listed in the exclusions file.
|
|
# ("Enumerated", deliberately — F1/F2 on PR #1018 proved this guard sees
|
|
# NAMING, not reachability, and its words must not claim otherwise.)
|
|
# FAIL when either surface names a path that does not exist on disk
|
|
# (a rename manufactures a stale entry silently — checked BOTH directions).
|
|
# FAIL when an exclusion entry has no reason, names a path that is gone,
|
|
# names a path that is also enumerated (contradiction), or names a path
|
|
# outside the population (dead weight that looks like coverage).
|
|
#
|
|
# POPULATION PATTERN — a deliberate decision, stated per #1017's record:
|
|
# basename matches *test*.sh (contains "test", ends ".sh"). Deliberately BROAD:
|
|
# the strict `test-*.sh` prefix cannot even name three real boundary files
|
|
# (tmux/agent-send.test.sh — CI-run; orchestrator/smoke-test.sh;
|
|
# wake/validate-973/microtest-wake-assert.sh), and three independent censuses
|
|
# handled that last file three different ways with no trace of the judgement.
|
|
# The broad pattern makes such files MEMBERS, so their disposition must be a
|
|
# signed exclusion, not an accident of the glob. The SAME pattern is applied to
|
|
# both sides of the comparison (disk and enumeration) — a comparison globbed two
|
|
# ways runs on two different populations. Scripts outside the pattern on both
|
|
# sides symmetrically (e.g. check-resident-budget.sh, verify-sanitized.sh) are
|
|
# check-scripts, not suites; their existence is still verified via the
|
|
# both-directions rule because every surface-named path must exist on disk.
|
|
#
|
|
# The surfaces are PARSED, never line-ranged: three seats independently
|
|
# mis-scoped hand-written line ranges against these files (#1017 thread). S1 is
|
|
# read via JSON + command-chain tokenization; S2 by extracting every
|
|
# packages/mosaic/framework/tools/ token wherever it appears in the file.
|
|
#
|
|
# Exclusions file format (framework/tools/quality/test-enumeration-exclusions.txt):
|
|
# <repo-relative-path> | <non-empty reason>
|
|
# Lines starting with # and blank lines are ignored. An exclusion is a recorded
|
|
# decision someone signed, not an omission nobody made.
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT="$(cd "$SCRIPT_DIR/../../../../../.." && pwd)"
|
|
while (( $# )); do
|
|
case "$1" in
|
|
--root) ROOT="$(cd "$2" && pwd)"; shift 2 ;;
|
|
*) echo "usage: check-test-enumeration.sh [--root <repo-root>]" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
PKG_JSON="$ROOT/packages/mosaic/package.json"
|
|
CI_YML="$ROOT/.woodpecker/ci.yml"
|
|
TOOLS_DIR="$ROOT/packages/mosaic/framework/tools"
|
|
EXCLUSIONS="$TOOLS_DIR/quality/test-enumeration-exclusions.txt"
|
|
|
|
for f in "$PKG_JSON" "$CI_YML"; do
|
|
[[ -f "$f" ]] || { echo "FAIL: required surface file missing: $f" >&2; exit 2; }
|
|
done
|
|
[[ -d "$TOOLS_DIR" ]] || { echo "FAIL: tools dir missing: $TOOLS_DIR" >&2; exit 2; }
|
|
|
|
fail_count=0
|
|
fail() { printf 'FAIL %s\n' "$1"; fail_count=$(( fail_count + 1 )); }
|
|
|
|
# in_population <repo-relative path> — the single pattern, used for BOTH sides.
|
|
in_population() {
|
|
local base; base="$(basename "$1")"
|
|
[[ "$base" == *test*.sh ]]
|
|
}
|
|
|
|
# --- Surface 1: package.json test:framework-shell, parsed, repo-relative -----
|
|
# Tokens are script paths iff they contain "/" and end .sh/.py; interpreter
|
|
# names and flags are skipped. Paths are relative to packages/mosaic/.
|
|
mapfile -t S1 < <(python3 - "$PKG_JSON" <<'PY'
|
|
import json, shlex, sys
|
|
cmd = json.load(open(sys.argv[1]))["scripts"].get("test:framework-shell", "")
|
|
seen = []
|
|
for seg in cmd.split("&&"):
|
|
for tok in shlex.split(seg):
|
|
if "/" in tok and (tok.endswith(".sh") or tok.endswith(".py")):
|
|
path = "packages/mosaic/" + tok
|
|
if path not in seen:
|
|
seen.append(path)
|
|
print("\n".join(seen))
|
|
PY
|
|
)
|
|
|
|
# --- Surface 2: ci.yml, every framework/tools token wherever it appears ------
|
|
# Comment lines (first non-whitespace char is #) are skipped BEFORE matching:
|
|
# commenting an invocation out is the most common way a suite actually gets
|
|
# disabled, and a raw-text regex would keep calling it enumerated (F1, 20155 on
|
|
# PR #1018 — demonstrated, not argued). Known residual limit: a path named only
|
|
# in a TRAILING comment on a live line still matches; no such line exists today
|
|
# and full fidelity would need a YAML parser the CI image does not ship.
|
|
mapfile -t S2 < <(grep -vE '^[[:space:]]*#' "$CI_YML" \
|
|
| grep -oE 'packages/mosaic/framework/tools/[A-Za-z0-9_./-]+\.(sh|py)' | sort -u)
|
|
|
|
# --- Union, and its population-restricted view -------------------------------
|
|
declare -A ENUM=() ENUM_POP=()
|
|
for p in "${S1[@]:-}" "${S2[@]:-}"; do
|
|
[[ -n "$p" ]] || continue
|
|
ENUM["$p"]=1
|
|
in_population "$p" && ENUM_POP["$p"]=1
|
|
done
|
|
|
|
# --- Direction B: every surface-named path must exist on disk ----------------
|
|
for p in "${!ENUM[@]}"; do
|
|
[[ -f "$ROOT/$p" ]] || fail "STALE ENUMERATION: surfaces name '$p' but it does not exist on disk"
|
|
done
|
|
|
|
# --- Exclusions: parsed with the same rigor the enumeration gets -------------
|
|
declare -A EXCLUDED=()
|
|
if [[ -f "$EXCLUSIONS" ]]; then
|
|
lineno=0
|
|
while IFS= read -r line; do
|
|
lineno=$(( lineno + 1 ))
|
|
[[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
|
|
path="${line%%|*}"; reason="${line#*|}"
|
|
path="$(echo "$path" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
|
|
reason="$(echo "$reason" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
|
|
if [[ "$line" != *"|"* || -z "$reason" ]]; then
|
|
fail "EXCLUSION MISSING REASON: line $lineno ('$path') — an exclusion is a recorded decision someone signed"
|
|
continue
|
|
fi
|
|
if [[ ! -f "$ROOT/$path" ]]; then
|
|
fail "STALE EXCLUSION: line $lineno excludes '$path' which does not exist on disk"
|
|
continue
|
|
fi
|
|
if ! in_population "$path"; then
|
|
fail "EXCLUSION OUTSIDE POPULATION: line $lineno excludes '$path' which the population pattern does not name — dead weight that reads as coverage"
|
|
continue
|
|
fi
|
|
if [[ -n "${ENUM[$path]:-}" ]]; then
|
|
fail "CONTRADICTORY EXCLUSION: line $lineno excludes '$path' which the surfaces already enumerate"
|
|
continue
|
|
fi
|
|
EXCLUDED["$path"]=1
|
|
done < "$EXCLUSIONS"
|
|
fi
|
|
|
|
# --- Direction A: disk population must be enumerated or signed-excluded ------
|
|
disk_total=0
|
|
unlisted=0
|
|
while IFS= read -r f; do
|
|
rel="${f#"$ROOT"/}"
|
|
in_population "$rel" || continue
|
|
disk_total=$(( disk_total + 1 ))
|
|
if [[ -z "${ENUM_POP[$rel]:-}" && -z "${EXCLUDED[$rel]:-}" ]]; then
|
|
fail "UNENUMERATED: '$rel' exists on disk but is neither enumerated on any CI surface nor signed in the exclusions file"
|
|
unlisted=$(( unlisted + 1 ))
|
|
fi
|
|
done < <(find "$TOOLS_DIR" -type f -name '*.sh' | sort)
|
|
|
|
if (( fail_count > 0 )); then
|
|
printf 'enumeration guard: %d failure(s) — population %d, enumerated (in-population) %d, excluded %d\n' \
|
|
"$fail_count" "$disk_total" "${#ENUM_POP[@]}" "${#EXCLUDED[@]}"
|
|
exit 1
|
|
fi
|
|
printf 'enumeration guard: OK — population %d, enumerated (in-population) %d, excluded (signed) %d, surfaces name %d path(s), all present on disk\n' \
|
|
"$disk_total" "${#ENUM_POP[@]}" "${#EXCLUDED[@]}" "${#ENUM[@]}"
|