289 lines
13 KiB
Bash
Executable File
289 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
SUBJECT="$ROOT/tools/verify-greenfield-execution-coverage.sh"
|
|
GATE="${GREENFIELD_COVERAGE_GATE:-$ROOT/tools/verify-greenfield-execution-coverage-gate.sh}"
|
|
WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml"
|
|
INSTALLER="${GREENFIELD_E2E_INSTALLER:-$ROOT/tools/e2e-install-test.sh}"
|
|
mkdir -p "$ROOT/.mosaic-test-work"
|
|
TMP="$(mktemp -d "$ROOT/.mosaic-test-work/execution-coverage.XXXXXX")"
|
|
selector_control="$ROOT/greenfield-coverage-selector-control.ran"
|
|
cleanup() {
|
|
rm -f "$selector_control"
|
|
rm -rf "$TMP"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
manifest="$TMP/expected-red.tsv"
|
|
state_root="$TMP/state"
|
|
run_id="pipeline-1"
|
|
printf '%s\n' \
|
|
'# representative manifest comment' \
|
|
$'# case\tkind\tkey/value' \
|
|
'' \
|
|
$'next-git-present\texit\t1' \
|
|
$'main-git-present\texit\t1' \
|
|
$'next-git-absent\texit\t1' > "$manifest"
|
|
|
|
bash "$SUBJECT" init cases "$manifest" "$state_root" "$run_id"
|
|
|
|
# Positive firing control: a skipped expected case makes the final gate red.
|
|
bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" next-git-present
|
|
bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" main-git-present
|
|
set +e
|
|
skipped_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)"
|
|
skipped_status=$?
|
|
set -e
|
|
[[ "$skipped_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$skipped_output"
|
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$skipped_output"
|
|
|
|
# Count inflation cannot pass: equal counts with one missing and one unexpected remain red.
|
|
: > "$state_root/$run_id/cases/unexpected-case.ran"
|
|
set +e
|
|
inflated_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)"
|
|
inflated_status=$?
|
|
set -e
|
|
[[ "$inflated_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$inflated_output"
|
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$inflated_output"
|
|
grep -qF '[fixture-suite] unexpected_case=unexpected-case' <<<"$inflated_output"
|
|
|
|
# Re-initializing the same run clears stale markers instead of certifying a later run.
|
|
bash "$SUBJECT" init cases "$manifest" "$state_root" "$run_id"
|
|
set +e
|
|
stale_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)"
|
|
stale_status=$?
|
|
set -e
|
|
[[ "$stale_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=0' <<<"$stale_output"
|
|
|
|
# Exact case-set equality is green.
|
|
for case_name in next-git-present main-git-present next-git-absent; do
|
|
bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" "$case_name"
|
|
done
|
|
complete_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id")"
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$complete_output"
|
|
|
|
# The production manifest, including comments, initializes and checks.
|
|
production_manifest="$ROOT/tools/fixtures/greenfield-expected-red.tsv"
|
|
production_arms="$ROOT/tools/fixtures/greenfield-expected-arms.txt"
|
|
production_state="$TMP/production-state"
|
|
bash "$SUBJECT" init cases "$production_manifest" "$production_state" production-1
|
|
for case_name in next-git-present main-git-present next-git-absent; do
|
|
bash "$SUBJECT" mark cases "$production_manifest" "$production_state" production-1 "$case_name"
|
|
done
|
|
production_output="$(bash "$SUBJECT" check cases "$production_manifest" "$production_state" production-1)"
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output"
|
|
|
|
mark_all_cases() {
|
|
local root="$1" run="$2"
|
|
for case_name in next-git-present main-git-present next-git-absent; do
|
|
bash "$SUBJECT" mark cases "$production_manifest" "$root" "$run" "$case_name"
|
|
done
|
|
}
|
|
|
|
mark_all_arms() {
|
|
local root="$1" run="$2"
|
|
for arm_name in \
|
|
greenfield-git-present \
|
|
greenfield-main-git-present \
|
|
greenfield-remote-installer-contract \
|
|
greenfield-git-absent; do
|
|
bash "$SUBJECT" mark arms "$production_arms" "$root" "$run" "$arm_name"
|
|
done
|
|
}
|
|
|
|
gate_state="$TMP/gate-state"
|
|
gate_run="truth-table"
|
|
|
|
# Final aggregation truth table: cases PASS / arms FAIL is red.
|
|
bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run"
|
|
bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run"
|
|
mark_all_cases "$gate_state" "$gate_run"
|
|
for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do
|
|
bash "$SUBJECT" mark arms "$production_arms" "$gate_state" "$gate_run" "$arm_name"
|
|
done
|
|
set +e
|
|
gate_cases_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)"
|
|
gate_cases_pass_status=$?
|
|
set -e
|
|
[[ "$gate_cases_pass_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$gate_cases_pass_output"
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=3' <<<"$gate_cases_pass_output"
|
|
grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$gate_cases_pass_output"
|
|
|
|
# Final aggregation truth table: cases FAIL / arms PASS is red.
|
|
bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run"
|
|
bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run"
|
|
for case_name in next-git-present main-git-present; do
|
|
bash "$SUBJECT" mark cases "$production_manifest" "$gate_state" "$gate_run" "$case_name"
|
|
done
|
|
mark_all_arms "$gate_state" "$gate_run"
|
|
set +e
|
|
gate_arms_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)"
|
|
gate_arms_pass_status=$?
|
|
set -e
|
|
[[ "$gate_arms_pass_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$gate_arms_pass_output"
|
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$gate_arms_pass_output"
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$gate_arms_pass_output"
|
|
|
|
# Final aggregation truth table: both FAIL emits both failures and is red.
|
|
bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run"
|
|
bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run"
|
|
for case_name in next-git-present main-git-present; do
|
|
bash "$SUBJECT" mark cases "$production_manifest" "$gate_state" "$gate_run" "$case_name"
|
|
done
|
|
for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do
|
|
bash "$SUBJECT" mark arms "$production_arms" "$gate_state" "$gate_run" "$arm_name"
|
|
done
|
|
set +e
|
|
gate_both_fail_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)"
|
|
gate_both_fail_status=$?
|
|
set -e
|
|
[[ "$gate_both_fail_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$gate_both_fail_output"
|
|
grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$gate_both_fail_output"
|
|
|
|
# Final aggregation truth table: both PASS is green.
|
|
bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run"
|
|
bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run"
|
|
mark_all_cases "$gate_state" "$gate_run"
|
|
mark_all_arms "$gate_state" "$gate_run"
|
|
gate_both_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run")"
|
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$gate_both_pass_output"
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$gate_both_pass_output"
|
|
|
|
# A future manifest case changes the derived set and cannot silently escape coverage.
|
|
printf '%s\n' $'future-case\texit\t1' >> "$manifest"
|
|
set +e
|
|
future_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)"
|
|
future_status=$?
|
|
set -e
|
|
[[ "$future_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] cases_defined=4 cases_executed=3' <<<"$future_output"
|
|
grep -qF '[fixture-suite] missing_case=future-case' <<<"$future_output"
|
|
|
|
# A skipped remote arm is red even while the three case identities are complete.
|
|
bash "$SUBJECT" init arms "$production_arms" "$production_state" production-1
|
|
for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do
|
|
bash "$SUBJECT" mark arms "$production_arms" "$production_state" production-1 "$arm_name"
|
|
done
|
|
set +e
|
|
remote_skipped_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1 2>&1)"
|
|
remote_skipped_status=$?
|
|
set -e
|
|
[[ "$remote_skipped_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=3' <<<"$remote_skipped_output"
|
|
grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$remote_skipped_output"
|
|
|
|
# Arm counts cannot be inflated either.
|
|
: > "$production_state/production-1/arms/unexpected-arm.ran"
|
|
set +e
|
|
arm_inflated_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1 2>&1)"
|
|
arm_inflated_status=$?
|
|
set -e
|
|
[[ "$arm_inflated_status" -eq 1 ]]
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$arm_inflated_output"
|
|
grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$arm_inflated_output"
|
|
grep -qF '[fixture-suite] unexpected_arm=unexpected-arm' <<<"$arm_inflated_output"
|
|
|
|
# Exact arm-set equality is green.
|
|
bash "$SUBJECT" init arms "$production_arms" "$production_state" production-1
|
|
for arm_name in \
|
|
greenfield-git-present \
|
|
greenfield-main-git-present \
|
|
greenfield-remote-installer-contract \
|
|
greenfield-git-absent; do
|
|
bash "$SUBJECT" mark arms "$production_arms" "$production_state" production-1 "$arm_name"
|
|
done
|
|
arm_complete_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1)"
|
|
grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$arm_complete_output"
|
|
|
|
# Bind archive purity to the production checkout-archive selector before proving its effect.
|
|
python3 - "$INSTALLER" <<'PY'
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
|
archive = re.search(
|
|
r'(?ms)^ tar -C "\$repo_parent" \\\n(?P<selectors>.*?)^ -czf "\$checkout_archive" "\$repo_name"$',
|
|
text,
|
|
)
|
|
assert archive, "production checkout-archive command not found"
|
|
assert "--exclude='*/.mosaic-test-work'" in archive.group("selectors"), (
|
|
"production checkout archive does not exclude .mosaic-test-work"
|
|
)
|
|
PY
|
|
|
|
# The production-bound checkout-archive selector excludes coverage state under .mosaic-test-work.
|
|
: > "$selector_control"
|
|
archive="$TMP/checkout.tar.gz"
|
|
archive_list="$TMP/checkout.list"
|
|
repo_parent="$(dirname "$ROOT")"
|
|
repo_name="$(basename "$ROOT")"
|
|
tar -C "$repo_parent" \
|
|
--exclude='*/.git' --exclude='*/node_modules' --exclude='*/dist' \
|
|
--exclude='*/coverage' --exclude='*/.turbo' --exclude='*/.mosaic-test-work' \
|
|
--exclude='*/.env' --exclude='*/.env.*' \
|
|
-czf "$archive" "$repo_name"
|
|
tar -tzf "$archive" > "$archive_list"
|
|
grep -qF "$repo_name/greenfield-coverage-selector-control.ran" "$archive_list"
|
|
if grep -qF "$repo_name/.mosaic-test-work/" "$archive_list"; then
|
|
echo 'coverage state leaked into checkout archive' >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Workflow contract: success-only case+arm markers, excluded state root, and complete ordering.
|
|
python3 - "$WORKFLOW" <<'PY'
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
|
steps = {
|
|
match.group(1): match.group(2)
|
|
for match in re.finditer(r"(?ms)^ ([A-Za-z0-9_-]+):\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text)
|
|
}
|
|
case_steps = (
|
|
("greenfield-git-present", "next-git-present"),
|
|
("greenfield-main-git-present", "main-git-present"),
|
|
("greenfield-git-absent", "next-git-absent"),
|
|
)
|
|
for step, case in case_steps:
|
|
block = steps[step]
|
|
verify = block.index("bash tools/verify-greenfield-expected-red.sh")
|
|
mark = block.index(f'"$coverage_run" {case}')
|
|
assert verify < mark, f"{step} marks its case before successful verification"
|
|
|
|
arm_steps = (
|
|
"greenfield-git-present",
|
|
"greenfield-main-git-present",
|
|
"greenfield-remote-installer-contract",
|
|
"greenfield-git-absent",
|
|
)
|
|
for step in arm_steps:
|
|
block = steps[step]
|
|
verify = block.index("bash tools/verify-greenfield-expected-red.sh")
|
|
mark = block.index(f'"$coverage_run" {step}')
|
|
assert verify < mark, f"{step} marks its arm before successful verification"
|
|
assert re.search(
|
|
r"(?m)^ depends_on:\n - greenfield-case-denominator-init$", block
|
|
), f"{step} can race marker initialization"
|
|
|
|
assert ".greenfield-case-state" not in text
|
|
assert ".mosaic-test-work/greenfield-execution-coverage" in text
|
|
final = steps["greenfield-case-denominator"]
|
|
assert "status: [success, failure]" in final
|
|
assert "bash tools/verify-greenfield-execution-coverage-gate.sh" in final
|
|
assert "check cases" not in final and "check arms" not in final
|
|
assert "cases_status" not in final and "arms_status" not in final
|
|
for dependency in arm_steps:
|
|
assert f" - {dependency}\n" in final, f"final gate can race {dependency}"
|
|
PY
|
|
|
|
printf 'greenfield execution coverage tests passed\n'
|