ci(installer): enforce greenfield case coverage
This commit is contained in:
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
echo "usage: $0 <init|mark|check> <manifest> <state-root> <run-id> [case]" >&2
|
||||
}
|
||||
|
||||
[[ "$#" -ge 4 ]] || { usage; exit 2; }
|
||||
mode="$1"
|
||||
manifest="$2"
|
||||
state_root="$3"
|
||||
run_id="$4"
|
||||
case_name="${5:-}"
|
||||
|
||||
[[ -s "$manifest" ]] || { echo "[fixture-suite] manifest missing or empty: $manifest" >&2; exit 2; }
|
||||
[[ -n "$state_root" && "$state_root" != / ]] \
|
||||
|| { echo "[fixture-suite] unsafe state root" >&2; exit 2; }
|
||||
[[ "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||
|| { echo "[fixture-suite] invalid run id: $run_id" >&2; exit 2; }
|
||||
|
||||
run_dir="$state_root/$run_id"
|
||||
|
||||
expected_cases() {
|
||||
awk -F '\t' 'NF && $0 !~ /^[[:space:]]*#/ && !seen[$1]++ { print $1 }' "$manifest" | LC_ALL=C sort
|
||||
}
|
||||
|
||||
validate_expected_cases() {
|
||||
local found=0 expected
|
||||
while IFS= read -r expected; do
|
||||
[[ -n "$expected" ]] || continue
|
||||
found=1
|
||||
[[ "$expected" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||
|| { echo "[fixture-suite] invalid manifest case name: $expected" >&2; return 1; }
|
||||
done < <(expected_cases)
|
||||
[[ "$found" -eq 1 ]] \
|
||||
|| { echo "[fixture-suite] manifest defines no cases" >&2; return 1; }
|
||||
}
|
||||
|
||||
validate_expected_cases
|
||||
|
||||
case "$mode" in
|
||||
init)
|
||||
[[ "$#" -eq 4 ]] || { usage; exit 2; }
|
||||
mkdir -p "$run_dir"
|
||||
find "$run_dir" -mindepth 1 -delete
|
||||
;;
|
||||
mark)
|
||||
[[ "$#" -eq 5 ]] || { usage; exit 2; }
|
||||
[[ "$case_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||
|| { echo "[fixture-suite] invalid case marker name: $case_name" >&2; exit 2; }
|
||||
[[ -d "$run_dir" ]] \
|
||||
|| { echo "[fixture-suite] run state was not initialized: $run_id" >&2; exit 1; }
|
||||
expected_cases | grep -Fxq -- "$case_name" \
|
||||
|| { echo "[fixture-suite] case is not defined by manifest: $case_name" >&2; exit 1; }
|
||||
: > "$run_dir/$case_name.ran"
|
||||
;;
|
||||
check)
|
||||
[[ "$#" -eq 4 ]] || { usage; exit 2; }
|
||||
mkdir -p "$run_dir"
|
||||
expected_file="$(mktemp "$run_dir/.expected.XXXXXX")"
|
||||
actual_file="$(mktemp "$run_dir/.actual.XXXXXX")"
|
||||
cleanup() { rm -f "$expected_file" "$actual_file"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
expected_cases > "$expected_file"
|
||||
find "$run_dir" -mindepth 1 -maxdepth 1 -type f -name '*.ran' -printf '%f\n' \
|
||||
| sed 's/\.ran$//' | LC_ALL=C sort -u > "$actual_file"
|
||||
|
||||
cases_defined="$(wc -l < "$expected_file" | tr -d ' ')"
|
||||
cases_executed="$(wc -l < "$actual_file" | tr -d ' ')"
|
||||
printf '[fixture-suite] cases_defined=%s cases_executed=%s\n' \
|
||||
"$cases_defined" "$cases_executed"
|
||||
|
||||
if ! cmp -s "$expected_file" "$actual_file"; then
|
||||
while IFS= read -r missing; do
|
||||
[[ -n "$missing" ]] && printf '[fixture-suite] missing_case=%s\n' "$missing" >&2
|
||||
done < <(comm -23 "$expected_file" "$actual_file")
|
||||
while IFS= read -r unexpected; do
|
||||
[[ -n "$unexpected" ]] && printf '[fixture-suite] unexpected_case=%s\n' "$unexpected" >&2
|
||||
done < <(comm -13 "$expected_file" "$actual_file")
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SUBJECT="$ROOT/tools/verify-greenfield-case-coverage.sh"
|
||||
WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml"
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
manifest="$TMP/expected-red.tsv"
|
||||
state_root="$TMP/state"
|
||||
run_id="pipeline-1"
|
||||
printf '%s\n' \
|
||||
'# representative manifest comment' \
|
||||
$'# case\tkind\tkey/value' \
|
||||
'' \
|
||||
$'next-git-present\texit\t1' \
|
||||
$'main-git-present\texit\t1' \
|
||||
$'next-git-absent\texit\t1' > "$manifest"
|
||||
|
||||
bash "$SUBJECT" init "$manifest" "$state_root" "$run_id"
|
||||
|
||||
# Positive firing control: a skipped expected arm makes the final gate red.
|
||||
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" next-git-present
|
||||
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" main-git-present
|
||||
set +e
|
||||
skipped_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||
skipped_status=$?
|
||||
set -e
|
||||
[[ "$skipped_status" -eq 1 ]]
|
||||
grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$skipped_output"
|
||||
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$skipped_output"
|
||||
|
||||
# Count inflation cannot pass: equal counts with one missing and one unexpected remain red.
|
||||
: > "$state_root/$run_id/unexpected-case.ran"
|
||||
set +e
|
||||
inflated_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||
inflated_status=$?
|
||||
set -e
|
||||
[[ "$inflated_status" -eq 1 ]]
|
||||
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$inflated_output"
|
||||
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$inflated_output"
|
||||
grep -qF '[fixture-suite] unexpected_case=unexpected-case' <<<"$inflated_output"
|
||||
|
||||
# Re-initializing the same run clears stale markers instead of certifying a later run.
|
||||
bash "$SUBJECT" init "$manifest" "$state_root" "$run_id"
|
||||
set +e
|
||||
stale_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||
stale_status=$?
|
||||
set -e
|
||||
[[ "$stale_status" -eq 1 ]]
|
||||
grep -qF '[fixture-suite] cases_defined=3 cases_executed=0' <<<"$stale_output"
|
||||
|
||||
# Exact set equality is the sole green disposition.
|
||||
for case_name in next-git-present main-git-present next-git-absent; do
|
||||
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" "$case_name"
|
||||
done
|
||||
complete_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id")"
|
||||
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$complete_output"
|
||||
|
||||
# The repository's production manifest, including its comments, initializes and checks.
|
||||
production_manifest="$ROOT/tools/fixtures/greenfield-expected-red.tsv"
|
||||
production_state="$TMP/production-state"
|
||||
bash "$SUBJECT" init "$production_manifest" "$production_state" production-1
|
||||
for case_name in next-git-present main-git-present next-git-absent; do
|
||||
bash "$SUBJECT" mark "$production_manifest" "$production_state" production-1 "$case_name"
|
||||
done
|
||||
production_output="$(bash "$SUBJECT" check "$production_manifest" "$production_state" production-1)"
|
||||
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output"
|
||||
|
||||
# A future manifest case changes the derived set and cannot silently escape coverage.
|
||||
printf '%s\n' $'future-case\texit\t1' >> "$manifest"
|
||||
set +e
|
||||
future_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||
future_status=$?
|
||||
set -e
|
||||
[[ "$future_status" -eq 1 ]]
|
||||
grep -qF '[fixture-suite] cases_defined=4 cases_executed=3' <<<"$future_output"
|
||||
grep -qF '[fixture-suite] missing_case=future-case' <<<"$future_output"
|
||||
|
||||
# The workflow writes each marker only after that case's per-case verifier succeeds,
|
||||
# and the final check is eligible after either success or failure.
|
||||
python3 - "$WORKFLOW" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
steps = {
|
||||
match.group(1): match.group(2)
|
||||
for match in re.finditer(r"(?ms)^ ([A-Za-z0-9_-]+):\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text)
|
||||
}
|
||||
case_steps = (
|
||||
("greenfield-git-present", "next-git-present"),
|
||||
("greenfield-main-git-present", "main-git-present"),
|
||||
("greenfield-git-absent", "next-git-absent"),
|
||||
)
|
||||
for step, case in case_steps:
|
||||
block = steps[step]
|
||||
verify = block.index("bash tools/verify-greenfield-expected-red.sh")
|
||||
mark = block.index(f"mark \\\n tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \\\n \"$coverage_run\" {case}")
|
||||
assert verify < mark, f"{step} marks execution before successful verification"
|
||||
|
||||
for step in (
|
||||
"greenfield-git-present",
|
||||
"greenfield-main-git-present",
|
||||
"greenfield-remote-installer-contract",
|
||||
"greenfield-git-absent",
|
||||
):
|
||||
assert re.search(
|
||||
r"(?m)^ depends_on:\n - greenfield-case-denominator-init$", steps[step]
|
||||
), f"{step} can race marker initialization"
|
||||
|
||||
final = steps["greenfield-case-denominator"]
|
||||
assert "status: [success, failure]" in final
|
||||
assert "bash tools/verify-greenfield-case-coverage.sh check" in final
|
||||
for dependency in (
|
||||
"greenfield-git-present",
|
||||
"greenfield-main-git-present",
|
||||
"greenfield-remote-installer-contract",
|
||||
"greenfield-git-absent",
|
||||
):
|
||||
assert f" - {dependency}\n" in final, f"final gate can race {dependency}"
|
||||
PY
|
||||
|
||||
printf 'greenfield case coverage tests passed\n'
|
||||
Reference in New Issue
Block a user