From d66e91b1f22489b911eef982aec3faa62c5edeb3 Mon Sep 17 00:00:00 2001 From: be-coder-05 Date: Thu, 6 Aug 2026 03:48:03 -0500 Subject: [PATCH] test(installer): bind execution coverage callers --- .woodpecker/greenfield-install.yml | 18 +-- docs/PRD.md | 2 +- ...d6-caller-coupling-and-archive-binding.txt | 60 ++++++++++ .../documentation-checklist.md | 3 +- .../1050-install-state-machine-red-fixture.md | 8 ++ ...rify-greenfield-execution-coverage-gate.sh | 27 +++++ ...rify-greenfield-execution-coverage.test.sh | 105 +++++++++++++++++- 7 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 docs/reports/verification/1050-c1-fix-round/12-round6-caller-coupling-and-archive-binding.txt create mode 100755 tools/verify-greenfield-execution-coverage-gate.sh diff --git a/.woodpecker/greenfield-install.yml b/.woodpecker/greenfield-install.yml index a648cba1..4dc33962 100644 --- a/.woodpecker/greenfield-install.yml +++ b/.woodpecker/greenfield-install.yml @@ -132,16 +132,8 @@ steps: - | coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" coverage_root=.mosaic-test-work/greenfield-execution-coverage - set +e - bash tools/verify-greenfield-execution-coverage.sh check cases \ - tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ - "$coverage_run" - cases_status=$? - bash tools/verify-greenfield-execution-coverage.sh check arms \ - tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ - "$coverage_run" - arms_status=$? - set -e - if [ "$cases_status" -ne 0 ] || [ "$arms_status" -ne 0 ]; then - exit 1 - fi + bash tools/verify-greenfield-execution-coverage-gate.sh \ + tools/verify-greenfield-execution-coverage.sh \ + tools/fixtures/greenfield-expected-red.tsv \ + tools/fixtures/greenfield-expected-arms.txt \ + "$coverage_root" "$coverage_run" diff --git a/docs/PRD.md b/docs/PRD.md index b330aadb..314688cb 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1393,7 +1393,7 @@ A from-zero install can report success while leaving the target host unusable be 12. The expected-RED comparator SHALL validate the complete manifest before selecting a case: exact case population, one exit and P0–P9 disposition per case, pinned require/forbid classes, and no malformed, duplicate, or unknown rows. 13. The published installer contract SHALL reject failed fetches, HTTP-success empty bodies, and digest mismatch, then execute the exact digest-verified body. The remote CI arm SHALL enumerate every payload-acquisition path and report a bound/found denominator. It SHALL bind both the installer body and the downstream stack framework/source it consumes to the same immutable CI commit, while retaining `--next` as the lane selector. Source resolution/acquisition SHALL fail closed, and the arm SHALL verify the realised source commit and archive digest after installation. Any out-of-scope unpinned sibling SHALL be named and counted rather than silently included in a broader pinning claim. `ASSUMPTION:` the configured repository's authenticated exact-commit endpoint is trusted to map that commit ID to the returned archive bytes; independent signed provenance/authenticity against repository or TLS trust-root compromise remains excluded by canonical greenfield-install PRD v2 §3. 14. Phase diagnostics SHALL be redacted before terminal or durable-log output. A seeded positive-control canary SHALL remain absent from observed argv, output, command logs, npm configuration, generated files, and shell history. -15. The CI fixture SHALL publish pipeline-level execution coverage for both the unconditional case set derived from the expected-RED manifest and the explicitly declared required pipeline-arm set, including the immutable remote-installer contract. Per-case and per-arm success markers SHALL be run-scoped, stored beneath an existing checkout-archive exclusion, and written only after that arm's verifier passes. A final step SHALL depend on the complete fixture matrix, run after prior success or failure, emit both `cases_defined=N cases_executed=M` and `arms_defined=N arms_executed=M`, and fail unless both expected/executed name sets are exactly equal; missing, unexpected, stale, newly added unexecuted, or checkout-contaminating state SHALL fail closed. +15. The CI fixture SHALL publish pipeline-level execution coverage for both the unconditional case set derived from the expected-RED manifest and the explicitly declared required pipeline-arm set, including the immutable remote-installer contract. Per-case and per-arm success markers SHALL be run-scoped, stored beneath an existing checkout-archive exclusion, and written only after that arm's verifier passes. A final step SHALL depend on the complete fixture matrix, run after prior success or failure, emit both `cases_defined=N cases_executed=M` and `arms_defined=N arms_executed=M`, and invoke the same testable aggregation helper whose complete case/arm PASS/FAIL truth table is covered. It SHALL fail unless both expected/executed name sets are exactly equal; missing, unexpected, stale, newly added unexecuted, or checkout-contaminating state SHALL fail closed. The archive-purity control SHALL bind to the production checkout-archive selector so deleting the production exclusion makes the control RED. ### C1 acceptance criteria diff --git a/docs/reports/verification/1050-c1-fix-round/12-round6-caller-coupling-and-archive-binding.txt b/docs/reports/verification/1050-c1-fix-round/12-round6-caller-coupling-and-archive-binding.txt new file mode 100644 index 00000000..6b24ccd8 --- /dev/null +++ b/docs/reports/verification/1050-c1-fix-round/12-round6-caller-coupling-and-archive-binding.txt @@ -0,0 +1,60 @@ +# #1050 C1 fix round — Round 6 caller coupling and production archive binding + +BASE HEAD +- df705828a439c6795cd47c938ed7f838b956ebf3 +- Reviews 114 (rev-security-02) and 115 (be-coder-06) independently reproduced the same caller-level mutant: ignoring arms_status in the workflow left the focused helper suite green. +- Both reviewers confirmed the production aggregation was correct and requested test reachability, not a logic rewrite. +- rev-974 separately deleted only the production e2e-install-test.sh .mosaic-test-work tar exclusion; the copied-selector suite remained green. + +RED +- After adding the four-cell caller truth-table expectations but before creating the workflow helper: + bash tools/verify-greenfield-execution-coverage.test.sh => rc=1 + The expected helper did not exist, so the test could not reach a passing aggregation implementation. +- Prior independent controls: + ignore-arms workflow mutant => rc=0 before this round (reviews 114/115) + delete-production-archive-exclusion mutant => rc=0 before this round (rev-974 exact-head sandbox) + +IMPLEMENTATION +- Extracted the existing set +e / two checks / status capture / final OR into tools/verify-greenfield-execution-coverage-gate.sh without changing its semantics. +- .woodpecker/greenfield-install.yml now invokes that helper directly; its inline duplicate was removed. +- The focused suite invokes the same helper used by the workflow across all four cells: + cases PASS / arms FAIL => RED + cases FAIL / arms PASS => RED + both FAIL => both diagnostic outputs and RED + both PASS => GREEN +- The archive-purity test now first locates the production checkout-archive command in tools/e2e-install-test.sh and requires its .mosaic-test-work exclusion, then proves the exclusion's archive effect. +- GREENFIELD_COVERAGE_GATE and GREENFIELD_E2E_INSTALLER are test-only subject injection seams used to prove the requested mutants; production paths remain the defaults. + +MUTATION CONTROLS +- ignore-arms aggregation helper => focused suite rc=1 +- ignore-cases aggregation helper => focused suite rc=1 +- always-RED aggregation helper => focused suite rc=1 +- delete only production .mosaic-test-work archive exclusion => focused suite rc=1 +- restored production subjects => greenfield execution coverage tests passed +- final mutant workspace: /tmp/c1-round6-final-mutants.u0ZaL6 +- logs: /tmp/c1-round6-final-gate-ignore-arms.log, /tmp/c1-round6-final-gate-ignore-cases.log, /tmp/c1-round6-final-gate-always-red.log, /tmp/c1-round6-final-archive-mutant.log + +FENCED FILES +- tools/e2e-install-test.sh HEAD/worktree SHA-256: a93113565aa69f2c6f3d792b78251021bb3bbe3f7813d5fed547ab0099fa3b98 (byte-identical) +- tools/fixtures/greenfield-expected-red.tsv HEAD/worktree SHA-256: baae06c664d0ab7ffe4c32ec106bfcb6f9dd74b46461d3c4440b73dc582af1a2 (byte-identical) +- No per-case disposition or #869 wiring changed. + +LOCAL BASELINES +- /home free before build: 11G +- bash -n focused scripts: PASS +- ShellCheck focused scripts: PASS +- bash tools/verify-greenfield-execution-coverage.test.sh: PASS +- pnpm test:installer: PASS +- pnpm typecheck: PASS (45/45) +- pnpm lint: PASS (25/25) +- pnpm format:check: PASS +- woodpecker-cli lint --strict .woodpecker/greenfield-install.yml: PASS +- git diff --check: PASS + +INDEPENDENT REVIEW +- Codex code review: APPROVE, confidence 0.94, seven files reviewed, zero findings. It confirmed the workflow invokes the exact four-cell-tested helper, both diagnostics remain observable, and archive purity binds to the production selector. +- Codex security review: risk NONE, confidence 0.97, seven files reviewed, zero findings. It confirmed fail-closed behavior, quoted inputs, repository-controlled production arguments, and no new secret/injection/access-control exposure. +- The code-review sandbox could not execute the dynamic suite because its filesystem is read-only; the local measured baselines and mutation controls above provide dynamic evidence. + +PENDING +- commit, queue guard, one lease-pinned push, provider attribution read-back diff --git a/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md b/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md index a672a3f0..a70f60b8 100644 --- a/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md +++ b/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md @@ -2,7 +2,7 @@ ## Required artifacts -- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 binds the complete in-scope remote stack source chain, and requirement 15 binds pipeline-level exact case+arm execution coverage without checkout-state contamination. +- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 binds the complete in-scope remote stack source chain, and requirement 15 binds pipeline-level exact case+arm execution coverage, executable aggregation, and production-bound archive purity. - [x] User guide: not applicable; no user-facing production installer behavior changed. - [x] Admin guide: not applicable; no operator procedure or deployment behavior changed. - [x] Developer guide: existing `docs/guides/installer-state-machine.md` defines the fail-closed installer model; the C1-specific remote-arm contract and evidence live in the PRD, scratchpad, and scoped verification report. @@ -22,4 +22,5 @@ - [x] Round-3 Codex code review APPROVE (confidence 0.92, zero findings) and security re-review risk NONE (confidence 0.96, zero findings). The initial HIGH trust-root finding remains documented as the canonical signed-provenance deferral, not hidden. - [x] Round-4 review cycle closed: two blocking Codex findings (missing dependency ordering; production manifest comments misparsed) were accepted and fixed. Final code review APPROVE (confidence 0.94, zero findings) and security risk NONE (confidence 0.96, zero findings). - [x] Round-5 exact arm coverage and archive-purity controls reviewed: Codex code APPROVE (confidence 0.93, zero findings) and security risk NONE (confidence 0.96, zero findings). +- [x] Round-6 executable aggregation and production archive-selector binding reviewed: Codex code APPROVE (confidence 0.94, zero findings) and security risk NONE (confidence 0.97, zero findings). - [x] Canonical evidence remains in-repo. No external publishing action was requested or performed. diff --git a/docs/scratchpads/1050-install-state-machine-red-fixture.md b/docs/scratchpads/1050-install-state-machine-red-fixture.md index fbe05e01..16f67cbf 100644 --- a/docs/scratchpads/1050-install-state-machine-red-fixture.md +++ b/docs/scratchpads/1050-install-state-machine-red-fixture.md @@ -160,3 +160,11 @@ Denominator corrections were sent to and accepted by the lane lead before implem - State moved beneath `.mosaic-test-work/greenfield-execution-coverage`, which the existing checkout selector excludes. The regression control proves a non-excluded root marker is archived while no `.mosaic-test-work` path is archived. - Final review: Codex code APPROVE confidence 0.93 and security risk NONE confidence 0.96, both with zero findings. Full installer tests, typecheck 45/45, lint 25/25, format, Bash syntax, ShellCheck, Woodpecker strict lint, and diff check pass with 11G free before/after. - Full evidence: `docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt`. + +## Round 6 — executable aggregation and production archive binding + +- Reviews 114/115 independently reproduced the same caller-level mutant at `df705828`: ignoring `arms_status` in the workflow still left the helper-bounded focused suite green. The production OR was correct and was not rewritten; its aggregation moved byte-for-byte into `verify-greenfield-execution-coverage-gate.sh`, which the workflow invokes directly. +- The focused suite now exercises that exact helper across the complete truth table: cases PASS/arms FAIL RED; cases FAIL/arms PASS RED; both FAIL emits both outputs and is RED; both PASS GREEN. Local ignore-arms, ignore-cases, and always-RED mutants each make the suite RED. +- Review `rev-974` separately proved that deleting only the production `e2e-install-test.sh` `.mosaic-test-work` tar exclusion left the copied-selector control green. The control now structurally binds its semantic archive test to the production checkout-archive command; the delete-production-binding mutant is RED. The fenced production installer remains byte-unchanged. +- Final review: Codex code APPROVE confidence 0.94 and security risk NONE confidence 0.97, both with zero findings. Full installer tests, typecheck 45/45, lint 25/25, format, Bash syntax, ShellCheck, Woodpecker strict lint, focused truth-table and four mutant controls, and diff check pass with 11G free before validation. +- Full evidence: `docs/reports/verification/1050-c1-fix-round/12-round6-caller-coupling-and-archive-binding.txt`. diff --git a/tools/verify-greenfield-execution-coverage-gate.sh b/tools/verify-greenfield-execution-coverage-gate.sh new file mode 100755 index 00000000..a717bc68 --- /dev/null +++ b/tools/verify-greenfield-execution-coverage-gate.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 +} + +[[ "$#" -eq 5 ]] || { usage; exit 2; } +checker="$1" +case_manifest="$2" +arm_manifest="$3" +state_root="$4" +run_id="$5" + +[[ -f "$checker" ]] \ + || { echo "[fixture-suite] coverage checker missing: $checker" >&2; exit 2; } + +set +e +bash "$checker" check cases "$case_manifest" "$state_root" "$run_id" +cases_status=$? +bash "$checker" check arms "$arm_manifest" "$state_root" "$run_id" +arms_status=$? +set -e + +if [ "$cases_status" -ne 0 ] || [ "$arms_status" -ne 0 ]; then + exit 1 +fi diff --git a/tools/verify-greenfield-execution-coverage.test.sh b/tools/verify-greenfield-execution-coverage.test.sh index 187ed8fb..ee441eb3 100755 --- a/tools/verify-greenfield-execution-coverage.test.sh +++ b/tools/verify-greenfield-execution-coverage.test.sh @@ -3,7 +3,9 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SUBJECT="$ROOT/tools/verify-greenfield-execution-coverage.sh" +GATE="${GREENFIELD_COVERAGE_GATE:-$ROOT/tools/verify-greenfield-execution-coverage-gate.sh}" WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml" +INSTALLER="${GREENFIELD_E2E_INSTALLER:-$ROOT/tools/e2e-install-test.sh}" mkdir -p "$ROOT/.mosaic-test-work" TMP="$(mktemp -d "$ROOT/.mosaic-test-work/execution-coverage.XXXXXX")" selector_control="$ROOT/greenfield-coverage-selector-control.ran" @@ -75,6 +77,85 @@ done production_output="$(bash "$SUBJECT" check cases "$production_manifest" "$production_state" production-1)" grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output" +mark_all_cases() { + local root="$1" run="$2" + for case_name in next-git-present main-git-present next-git-absent; do + bash "$SUBJECT" mark cases "$production_manifest" "$root" "$run" "$case_name" + done +} + +mark_all_arms() { + local root="$1" run="$2" + for arm_name in \ + greenfield-git-present \ + greenfield-main-git-present \ + greenfield-remote-installer-contract \ + greenfield-git-absent; do + bash "$SUBJECT" mark arms "$production_arms" "$root" "$run" "$arm_name" + done +} + +gate_state="$TMP/gate-state" +gate_run="truth-table" + +# Final aggregation truth table: cases PASS / arms FAIL is red. +bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run" +bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run" +mark_all_cases "$gate_state" "$gate_run" +for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do + bash "$SUBJECT" mark arms "$production_arms" "$gate_state" "$gate_run" "$arm_name" +done +set +e +gate_cases_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)" +gate_cases_pass_status=$? +set -e +[[ "$gate_cases_pass_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$gate_cases_pass_output" +grep -qF '[fixture-suite] arms_defined=4 arms_executed=3' <<<"$gate_cases_pass_output" +grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$gate_cases_pass_output" + +# Final aggregation truth table: cases FAIL / arms PASS is red. +bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run" +bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run" +for case_name in next-git-present main-git-present; do + bash "$SUBJECT" mark cases "$production_manifest" "$gate_state" "$gate_run" "$case_name" +done +mark_all_arms "$gate_state" "$gate_run" +set +e +gate_arms_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)" +gate_arms_pass_status=$? +set -e +[[ "$gate_arms_pass_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$gate_arms_pass_output" +grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$gate_arms_pass_output" +grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$gate_arms_pass_output" + +# Final aggregation truth table: both FAIL emits both failures and is red. +bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run" +bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run" +for case_name in next-git-present main-git-present; do + bash "$SUBJECT" mark cases "$production_manifest" "$gate_state" "$gate_run" "$case_name" +done +for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do + bash "$SUBJECT" mark arms "$production_arms" "$gate_state" "$gate_run" "$arm_name" +done +set +e +gate_both_fail_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run" 2>&1)" +gate_both_fail_status=$? +set -e +[[ "$gate_both_fail_status" -eq 1 ]] +grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$gate_both_fail_output" +grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$gate_both_fail_output" + +# Final aggregation truth table: both PASS is green. +bash "$SUBJECT" init cases "$production_manifest" "$gate_state" "$gate_run" +bash "$SUBJECT" init arms "$production_arms" "$gate_state" "$gate_run" +mark_all_cases "$gate_state" "$gate_run" +mark_all_arms "$gate_state" "$gate_run" +gate_both_pass_output="$(bash "$GATE" "$SUBJECT" "$production_manifest" "$production_arms" "$gate_state" "$gate_run")" +grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$gate_both_pass_output" +grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$gate_both_pass_output" + # A future manifest case changes the derived set and cannot silently escape coverage. printf '%s\n' $'future-case\texit\t1' >> "$manifest" set +e @@ -121,7 +202,24 @@ done arm_complete_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1)" grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$arm_complete_output" -# The exact checkout-archive selector excludes coverage state under .mosaic-test-work. +# Bind archive purity to the production checkout-archive selector before proving its effect. +python3 - "$INSTALLER" <<'PY' +import re +import sys +from pathlib import Path + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +archive = re.search( + r'(?ms)^ tar -C "\$repo_parent" \\\n(?P.*?)^ -czf "\$checkout_archive" "\$repo_name"$', + text, +) +assert archive, "production checkout-archive command not found" +assert "--exclude='*/.mosaic-test-work'" in archive.group("selectors"), ( + "production checkout archive does not exclude .mosaic-test-work" +) +PY + +# The production-bound checkout-archive selector excludes coverage state under .mosaic-test-work. : > "$selector_control" archive="$TMP/checkout.tar.gz" archive_list="$TMP/checkout.list" @@ -180,8 +278,9 @@ assert ".greenfield-case-state" not in text assert ".mosaic-test-work/greenfield-execution-coverage" in text final = steps["greenfield-case-denominator"] assert "status: [success, failure]" in final -assert "check cases" in final and "check arms" in final -assert "cases_status" in final and "arms_status" in final +assert "bash tools/verify-greenfield-execution-coverage-gate.sh" in final +assert "check cases" not in final and "check arms" not in final +assert "cases_status" not in final and "arms_status" not in final for dependency in arm_steps: assert f" - {dependency}\n" in final, f"final gate can race {dependency}" PY