From df705828a439c6795cd47c938ed7f838b956ebf3 Mon Sep 17 00:00:00 2001 From: be-coder-05 Date: Thu, 6 Aug 2026 03:14:19 -0500 Subject: [PATCH] ci(installer): cover all greenfield execution arms --- .woodpecker/greenfield-install.yml | 52 ++++- docs/PRD.md | 4 +- .../11-round5-arm-coverage-and-purity.txt | 81 ++++++++ .../documentation-checklist.md | 3 +- .../1050-install-state-machine-red-fixture.md | 9 + package.json | 2 +- tools/fixtures/greenfield-expected-arms.txt | 5 + tools/verify-greenfield-case-coverage.sh | 88 -------- tools/verify-greenfield-case-coverage.test.sh | 126 ------------ tools/verify-greenfield-execution-coverage.sh | 96 +++++++++ ...rify-greenfield-execution-coverage.test.sh | 189 ++++++++++++++++++ 11 files changed, 427 insertions(+), 228 deletions(-) create mode 100644 docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt create mode 100644 tools/fixtures/greenfield-expected-arms.txt delete mode 100755 tools/verify-greenfield-case-coverage.sh delete mode 100755 tools/verify-greenfield-case-coverage.test.sh create mode 100755 tools/verify-greenfield-execution-coverage.sh create mode 100755 tools/verify-greenfield-execution-coverage.test.sh diff --git a/.woodpecker/greenfield-install.yml b/.woodpecker/greenfield-install.yml index aaac6d7c..a648cba1 100644 --- a/.woodpecker/greenfield-install.yml +++ b/.woodpecker/greenfield-install.yml @@ -11,8 +11,12 @@ steps: commands: - | coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" - bash tools/verify-greenfield-case-coverage.sh init \ - tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \ + coverage_root=.mosaic-test-work/greenfield-execution-coverage + bash tools/verify-greenfield-execution-coverage.sh init cases \ + tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ + "$coverage_run" + bash tools/verify-greenfield-execution-coverage.sh init arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ "$coverage_run" greenfield-git-present: @@ -31,9 +35,13 @@ steps: bash tools/verify-greenfield-expected-red.sh \ next-git-present /tmp/greenfield-git-present.log "$fixture_status" coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" - bash tools/verify-greenfield-case-coverage.sh mark \ - tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \ + coverage_root=.mosaic-test-work/greenfield-execution-coverage + bash tools/verify-greenfield-execution-coverage.sh mark cases \ + tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ "$coverage_run" next-git-present + bash tools/verify-greenfield-execution-coverage.sh mark arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ + "$coverage_run" greenfield-git-present greenfield-main-git-present: image: node:22-bookworm-slim @@ -51,9 +59,13 @@ steps: bash tools/verify-greenfield-expected-red.sh \ main-git-present /tmp/greenfield-main-git-present.log "$fixture_status" coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" - bash tools/verify-greenfield-case-coverage.sh mark \ - tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \ + coverage_root=.mosaic-test-work/greenfield-execution-coverage + bash tools/verify-greenfield-execution-coverage.sh mark cases \ + tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ "$coverage_run" main-git-present + bash tools/verify-greenfield-execution-coverage.sh mark arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ + "$coverage_run" greenfield-main-git-present greenfield-remote-installer-contract: image: node:22-bookworm-slim @@ -76,6 +88,11 @@ steps: cat /tmp/greenfield-remote.log bash tools/verify-greenfield-expected-red.sh \ next-git-present /tmp/greenfield-remote.log "$fixture_status" + coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" + coverage_root=.mosaic-test-work/greenfield-execution-coverage + bash tools/verify-greenfield-execution-coverage.sh mark arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ + "$coverage_run" greenfield-remote-installer-contract greenfield-git-absent: image: node:22-bookworm-slim @@ -93,9 +110,13 @@ steps: bash tools/verify-greenfield-expected-red.sh \ next-git-absent /tmp/greenfield-git-absent.log "$fixture_status" coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" - bash tools/verify-greenfield-case-coverage.sh mark \ - tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \ + coverage_root=.mosaic-test-work/greenfield-execution-coverage + bash tools/verify-greenfield-execution-coverage.sh mark cases \ + tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ "$coverage_run" next-git-absent + bash tools/verify-greenfield-execution-coverage.sh mark arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ + "$coverage_run" greenfield-git-absent greenfield-case-denominator: image: node:22-bookworm-slim @@ -110,6 +131,17 @@ steps: commands: - | coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}" - bash tools/verify-greenfield-case-coverage.sh check \ - tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \ + coverage_root=.mosaic-test-work/greenfield-execution-coverage + set +e + bash tools/verify-greenfield-execution-coverage.sh check cases \ + tools/fixtures/greenfield-expected-red.tsv "$coverage_root" \ "$coverage_run" + cases_status=$? + bash tools/verify-greenfield-execution-coverage.sh check arms \ + tools/fixtures/greenfield-expected-arms.txt "$coverage_root" \ + "$coverage_run" + arms_status=$? + set -e + if [ "$cases_status" -ne 0 ] || [ "$arms_status" -ne 0 ]; then + exit 1 + fi diff --git a/docs/PRD.md b/docs/PRD.md index b8e525a1..b330aadb 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1393,14 +1393,14 @@ A from-zero install can report success while leaving the target host unusable be 12. The expected-RED comparator SHALL validate the complete manifest before selecting a case: exact case population, one exit and P0–P9 disposition per case, pinned require/forbid classes, and no malformed, duplicate, or unknown rows. 13. The published installer contract SHALL reject failed fetches, HTTP-success empty bodies, and digest mismatch, then execute the exact digest-verified body. The remote CI arm SHALL enumerate every payload-acquisition path and report a bound/found denominator. It SHALL bind both the installer body and the downstream stack framework/source it consumes to the same immutable CI commit, while retaining `--next` as the lane selector. Source resolution/acquisition SHALL fail closed, and the arm SHALL verify the realised source commit and archive digest after installation. Any out-of-scope unpinned sibling SHALL be named and counted rather than silently included in a broader pinning claim. `ASSUMPTION:` the configured repository's authenticated exact-commit endpoint is trusted to map that commit ID to the returned archive bytes; independent signed provenance/authenticity against repository or TLS trust-root compromise remains excluded by canonical greenfield-install PRD v2 §3. 14. Phase diagnostics SHALL be redacted before terminal or durable-log output. A seeded positive-control canary SHALL remain absent from observed argv, output, command logs, npm configuration, generated files, and shell history. -15. The CI fixture SHALL publish pipeline-level execution coverage for the unconditional case set derived from the expected-RED manifest. Per-case success markers SHALL be run-scoped and written only after that case's verifier passes. A final step SHALL depend on the complete fixture matrix, run after prior success or failure, emit `cases_defined=N cases_executed=M`, and fail unless the expected and executed case-name sets are exactly equal; missing, unexpected, stale, or newly added unexecuted cases SHALL fail closed. +15. The CI fixture SHALL publish pipeline-level execution coverage for both the unconditional case set derived from the expected-RED manifest and the explicitly declared required pipeline-arm set, including the immutable remote-installer contract. Per-case and per-arm success markers SHALL be run-scoped, stored beneath an existing checkout-archive exclusion, and written only after that arm's verifier passes. A final step SHALL depend on the complete fixture matrix, run after prior success or failure, emit both `cases_defined=N cases_executed=M` and `arms_defined=N arms_executed=M`, and fail unless both expected/executed name sets are exactly equal; missing, unexpected, stale, newly added unexecuted, or checkout-contaminating state SHALL fail closed. ### C1 acceptance criteria 1. The pre-C1 from-zero matrix records both discriminating controls: with `git` absent, the legacy installer still exits zero while P1 fails and skill sync degrades; with `git` present, P1 passes and the observed sync store/runtime links are 101/101. The C1 installer must fail at P1 before mutation when `git` is absent. 2. The discriminating P3 row passes: the binary exists at the expected absolute path and reports exactly the resolved `next` lane version, while P4, P5, and P8 fail. 3. The `--check` mutation negative control proves host fingerprints are byte-identical before and after observation. -4. Woodpecker executes and validates the expected RED fixture plus the immutable remote-installer contract; its pipeline-level case-coverage gate reports exact manifest-derived set equality and a skipped-arm control proves non-execution is red. C1 does not repair P4/P5/P8 or activate #869. +4. Woodpecker executes and validates the expected RED fixture plus the immutable remote-installer contract; its pipeline-level coverage gate reports exact equality for the manifest-derived case set and the declared arm set, while skipped-case and skipped-remote-arm controls prove non-execution is red. C1 does not repair P4/P5/P8 or activate #869. 5. Negative controls prove manifest shrink/duplicates/unknown rows fail, unsafe P0/P3/P4/P5 contexts fail, the P2–P8 fault seam enters real actions rather than synthetic writes, empty/mismatched fetched bodies fail, and a deliberately emitted secret canary is redacted from every persisted/output population. ### Explicit exclusions and dependencies diff --git a/docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt b/docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt new file mode 100644 index 00000000..b14a1bd9 --- /dev/null +++ b/docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt @@ -0,0 +1,81 @@ +#1050 C1 ROUND 5 — ARM COVERAGE + CHECKOUT PURITY + +IDENTITY / SCOPE +- predecessor head: f33bd0da96bee3ce518744d6888d018bebb1c841 +- branch: feat/1050-install-state-machine-red-fixture +- review inputs: be-coder-06 review 110; rev-security-02 review 111 +- in scope: exact required pipeline-arm set including greenfield-remote-installer-contract; success-only arm marks; skipped-remote RED control; coverage-state exclusion from checkout source archives +- unchanged/fenced: tools/fixtures/greenfield-expected-red.tsv, tools/verify-greenfield-expected-red.sh, tools/e2e-install-test.sh, #869, and all per-case dispositions +- status: believed-fixed, pending jarvis validation + +RED-FIRST — FOUR ARMS WERE REPRESENTED BY THREE CASES +The f33bd0d gate represented only the three expected-RED case identities. With all three case names marked and no remote-arm identity available, the production checker returned: + SKIPPED_REMOTE_ARM_CURRENT_GATE_RC=0 + [fixture-suite] cases_defined=3 cases_executed=3 +The fourth required arm, greenfield-remote-installer-contract, reused next-git-present and therefore could skip without changing case-set equality. + +RED-FIRST — COVERAGE STATE CONTAMINATED CHECKOUT ARCHIVES +The f33bd0d workflow created .greenfield-case-state in the shared repository before the checkout-mode fixture arms. An independent reproduction using the exact e2e-install-test.sh tar exclusions created .greenfield-case-state/remote-arm.ran and measured: + OLD_STATE_ARCHIVE_MATCHES=1 +The state instrument was therefore part of the checkout source payload it was intended only to observe. + +IMPLEMENTATION +- Renamed the checker to tools/verify-greenfield-execution-coverage.sh and generalized it over two validated dimensions: cases and arms. +- Added tools/fixtures/greenfield-expected-arms.txt as the explicit required arm set: + greenfield-git-present + greenfield-main-git-present + greenfield-remote-installer-contract + greenfield-git-absent +- Expected cases remain derived from the unchanged expected-RED TSV. Expected arms are derived from the new one-name-per-line arm declaration. Both inputs allow comments/blanks and validate every resulting name. +- Each of the four fixture arms writes its unique arm marker only after its exact per-case verifier succeeds. The three distinct expected-RED cases continue to write case markers after successful verification. +- The final step checks both dimensions even when the first check fails, emits both denominators, and fails unless both exact name sets match. +- Coverage state moved to .mosaic-test-work/greenfield-execution-coverage/${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}/{cases,arms}. The checkout archive already excludes every */.mosaic-test-work subtree. +- The expected-RED manifest, per-case verifier, and detector are untouched. + +MEASURED SKIPPED-REMOTE CONTROL +With all three cases marked and only the three non-remote arms marked: + case gate rc=0 + [fixture-suite] cases_defined=3 cases_executed=3 + arm gate rc=1 + [fixture-suite] arms_defined=4 arms_executed=3 + [fixture-suite] missing_arm=greenfield-remote-installer-contract +This is the discriminating control: the case denominator remains green while the arm denominator catches the exact previously invisible skip. + +OTHER DYNAMIC CONTROLS +- missing case: cases 3/2 => rc1 +- case count inflation (one missing + one unexpected): cases 3/3 => rc1 +- stale case markers after re-init: cases 3/0 => rc1 +- exact case set: cases 3/3 => rc0 +- future manifest case: cases 4/3 => rc1 +- arm count inflation (remote missing + unexpected): arms 4/4 => rc1 +- exact arm set: arms 4/4 => rc0 +- production expected-RED manifest comments/blanks: accepted; exact set 3/3 => rc0 +- archive selector firing control: a non-excluded root marker is present in the archive +- archive purity control: no .mosaic-test-work path is present in the archive + +STATIC / STRUCTURAL CONTROLS +- every case mark follows that arm's successful per-case verifier +- every arm mark, including the remote contract, follows that arm's successful per-case verifier +- all four arms depend on coverage initialization +- final denominator depends on all four arms and is eligible after success or failure +- final denominator runs both checks and aggregates their statuses +- workflow contains the already-excluded .mosaic-test-work state root and no .greenfield-case-state reference + +LOCAL BASELINES +- bash -n + ShellCheck on generalized checker/test: PASS +- pnpm test:installer: PASS, including execution-coverage controls +- pnpm typecheck: PASS, 45/45 +- pnpm lint: PASS, 25/25 +- pnpm format:check: PASS +- woodpecker-cli lint --strict: PASS +- git diff --check: PASS +- /home free before/after: 11G + +INDEPENDENT REVIEW +- Codex code review: APPROVE, confidence 0.93, zero findings. It confirmed exact arm coverage, success-only marking, dual-status aggregation, excluded state, checkout-purity controls, and PRD alignment. +- Codex security review: risk NONE, confidence 0.96, zero findings. CI/path identifiers are constrained, expansions quoted, expected sets treated as data, and checks fail closed. +- Review sandboxes could not execute their own dynamic suite/ShellCheck because their filesystem is read-only; the local measured baselines above provide that evidence. + +PENDING +- commit, queue guard, one push, attribution read-back +- announce the new full SHA; no merge or closure diff --git a/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md b/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md index ab822724..a672a3f0 100644 --- a/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md +++ b/docs/reports/verification/1050-c1-fix-round/documentation-checklist.md @@ -2,7 +2,7 @@ ## Required artifacts -- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 binds the complete in-scope remote stack source chain, and requirement 15 binds pipeline-level exact case-set execution coverage. +- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 binds the complete in-scope remote stack source chain, and requirement 15 binds pipeline-level exact case+arm execution coverage without checkout-state contamination. - [x] User guide: not applicable; no user-facing production installer behavior changed. - [x] Admin guide: not applicable; no operator procedure or deployment behavior changed. - [x] Developer guide: existing `docs/guides/installer-state-machine.md` defines the fail-closed installer model; the C1-specific remote-arm contract and evidence live in the PRD, scratchpad, and scoped verification report. @@ -21,4 +21,5 @@ - [x] Trust boundary states both halves: the digest proves fetched bytes remain identical through execution/verify-after; it does not authenticate authorship against provider/TLS compromise. Independent provenance is the inherited PRD v2 §3 deferral. - [x] Round-3 Codex code review APPROVE (confidence 0.92, zero findings) and security re-review risk NONE (confidence 0.96, zero findings). The initial HIGH trust-root finding remains documented as the canonical signed-provenance deferral, not hidden. - [x] Round-4 review cycle closed: two blocking Codex findings (missing dependency ordering; production manifest comments misparsed) were accepted and fixed. Final code review APPROVE (confidence 0.94, zero findings) and security risk NONE (confidence 0.96, zero findings). +- [x] Round-5 exact arm coverage and archive-purity controls reviewed: Codex code APPROVE (confidence 0.93, zero findings) and security risk NONE (confidence 0.96, zero findings). - [x] Canonical evidence remains in-repo. No external publishing action was requested or performed. diff --git a/docs/scratchpads/1050-install-state-machine-red-fixture.md b/docs/scratchpads/1050-install-state-machine-red-fixture.md index 2581692f..fbe05e01 100644 --- a/docs/scratchpads/1050-install-state-machine-red-fixture.md +++ b/docs/scratchpads/1050-install-state-machine-red-fixture.md @@ -151,3 +151,12 @@ Denominator corrections were sent to and accepted by the lane lead before implem - Requirement 5 adds a pipeline-level instrument above the three per-case invocations. Expected names are derived from the manifest, markers are scoped by pipeline+workflow run, and each marker is written only after that exact per-case verifier succeeds. Every producer depends on initialization; the final step depends on the complete case/contract matrix, runs after success or failure, and requires exact expected/actual set equality rather than count equality. - Measured firing controls: skipped arm `3/2` rc1; count inflation `3/3` rc1 with one missing and one unexpected name; stale re-initialized run `3/0` rc1; exact set `3/3` rc0; future manifest case `4/3` rc1. Codex review found and blocked two independent defects: first the missing `depends_on` graph, then comment lines being parsed as case names. Both were accepted; dependency edges are regression-asserted, blank/comments are excluded, and the focused test now consumes the production manifest directly. Final Codex code re-review APPROVE confidence 0.94 and security re-review risk NONE confidence 0.96, both with zero findings. Woodpecker strict lint, Bash syntax, ShellCheck, focused/full installer tests, typecheck, lint, Prettier, and diff check pass. - Full evidence: `docs/reports/verification/1050-c1-fix-round/10-round4-case-coverage.txt`. + +## Round 5 — arm coverage and checkout purity + +- Reviews 110/111 identified two blockers at `f33bd0da`: the three-case set could not represent the fourth `greenfield-remote-installer-contract` arm, and the root `.greenfield-case-state` directory was included in checkout fixture archives. The expected-RED manifest, per-case verifier, detector, and #869 remain fenced. +- RED controls: with all three cases complete and no remote-arm identity, the old gate returned rc0 at `cases_defined=3 cases_executed=3`; the exact fixture tar selector archived `.greenfield-case-state/remote-arm.ran` once. +- The checker is generalized across exact `cases` and `arms` dimensions. A new explicit four-arm declaration includes the remote contract; all four arms mark only after successful per-case verification. With the remote omitted, cases remain 3/3 rc0 while arms report 4/3 rc1 and name `greenfield-remote-installer-contract` as missing. +- State moved beneath `.mosaic-test-work/greenfield-execution-coverage`, which the existing checkout selector excludes. The regression control proves a non-excluded root marker is archived while no `.mosaic-test-work` path is archived. +- Final review: Codex code APPROVE confidence 0.93 and security risk NONE confidence 0.96, both with zero findings. Full installer tests, typecheck 45/45, lint 25/25, format, Bash syntax, ShellCheck, Woodpecker strict lint, and diff check pass with 11G free before/after. +- Full evidence: `docs/reports/verification/1050-c1-fix-round/11-round5-arm-coverage-and-purity.txt`. diff --git a/package.json b/package.json index cee6cce7..0dc82705 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "typecheck": "pnpm preflight && turbo run typecheck", "test:checkout": "node --test scripts/*.test.mjs", "test": "pnpm test:checkout && turbo run test && pnpm run test:installer", - "test:installer": "bash tools/install-state-machine.test.sh && bash tools/install-next-lane.test.sh && bash tools/verify-greenfield-expected-red.test.sh && bash tools/verify-greenfield-case-coverage.test.sh && bash tools/verified-installer-fetch.test.sh", + "test:installer": "bash tools/install-state-machine.test.sh && bash tools/install-next-lane.test.sh && bash tools/verify-greenfield-expected-red.test.sh && bash tools/verify-greenfield-execution-coverage.test.sh && bash tools/verified-installer-fetch.test.sh", "format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"", "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"", "prepare": "node scripts/install-hooks.mjs" diff --git a/tools/fixtures/greenfield-expected-arms.txt b/tools/fixtures/greenfield-expected-arms.txt new file mode 100644 index 00000000..9b78eb93 --- /dev/null +++ b/tools/fixtures/greenfield-expected-arms.txt @@ -0,0 +1,5 @@ +# Required Woodpecker execution arms for C1 acceptance criterion 4. +greenfield-git-present +greenfield-main-git-present +greenfield-remote-installer-contract +greenfield-git-absent diff --git a/tools/verify-greenfield-case-coverage.sh b/tools/verify-greenfield-case-coverage.sh deleted file mode 100755 index 308c471a..00000000 --- a/tools/verify-greenfield-case-coverage.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - echo "usage: $0 [case]" >&2 -} - -[[ "$#" -ge 4 ]] || { usage; exit 2; } -mode="$1" -manifest="$2" -state_root="$3" -run_id="$4" -case_name="${5:-}" - -[[ -s "$manifest" ]] || { echo "[fixture-suite] manifest missing or empty: $manifest" >&2; exit 2; } -[[ -n "$state_root" && "$state_root" != / ]] \ - || { echo "[fixture-suite] unsafe state root" >&2; exit 2; } -[[ "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ - || { echo "[fixture-suite] invalid run id: $run_id" >&2; exit 2; } - -run_dir="$state_root/$run_id" - -expected_cases() { - awk -F '\t' 'NF && $0 !~ /^[[:space:]]*#/ && !seen[$1]++ { print $1 }' "$manifest" | LC_ALL=C sort -} - -validate_expected_cases() { - local found=0 expected - while IFS= read -r expected; do - [[ -n "$expected" ]] || continue - found=1 - [[ "$expected" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ - || { echo "[fixture-suite] invalid manifest case name: $expected" >&2; return 1; } - done < <(expected_cases) - [[ "$found" -eq 1 ]] \ - || { echo "[fixture-suite] manifest defines no cases" >&2; return 1; } -} - -validate_expected_cases - -case "$mode" in - init) - [[ "$#" -eq 4 ]] || { usage; exit 2; } - mkdir -p "$run_dir" - find "$run_dir" -mindepth 1 -delete - ;; - mark) - [[ "$#" -eq 5 ]] || { usage; exit 2; } - [[ "$case_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ - || { echo "[fixture-suite] invalid case marker name: $case_name" >&2; exit 2; } - [[ -d "$run_dir" ]] \ - || { echo "[fixture-suite] run state was not initialized: $run_id" >&2; exit 1; } - expected_cases | grep -Fxq -- "$case_name" \ - || { echo "[fixture-suite] case is not defined by manifest: $case_name" >&2; exit 1; } - : > "$run_dir/$case_name.ran" - ;; - check) - [[ "$#" -eq 4 ]] || { usage; exit 2; } - mkdir -p "$run_dir" - expected_file="$(mktemp "$run_dir/.expected.XXXXXX")" - actual_file="$(mktemp "$run_dir/.actual.XXXXXX")" - cleanup() { rm -f "$expected_file" "$actual_file"; } - trap cleanup EXIT - - expected_cases > "$expected_file" - find "$run_dir" -mindepth 1 -maxdepth 1 -type f -name '*.ran' -printf '%f\n' \ - | sed 's/\.ran$//' | LC_ALL=C sort -u > "$actual_file" - - cases_defined="$(wc -l < "$expected_file" | tr -d ' ')" - cases_executed="$(wc -l < "$actual_file" | tr -d ' ')" - printf '[fixture-suite] cases_defined=%s cases_executed=%s\n' \ - "$cases_defined" "$cases_executed" - - if ! cmp -s "$expected_file" "$actual_file"; then - while IFS= read -r missing; do - [[ -n "$missing" ]] && printf '[fixture-suite] missing_case=%s\n' "$missing" >&2 - done < <(comm -23 "$expected_file" "$actual_file") - while IFS= read -r unexpected; do - [[ -n "$unexpected" ]] && printf '[fixture-suite] unexpected_case=%s\n' "$unexpected" >&2 - done < <(comm -13 "$expected_file" "$actual_file") - exit 1 - fi - ;; - *) - usage - exit 2 - ;; -esac diff --git a/tools/verify-greenfield-case-coverage.test.sh b/tools/verify-greenfield-case-coverage.test.sh deleted file mode 100755 index b44caa9e..00000000 --- a/tools/verify-greenfield-case-coverage.test.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SUBJECT="$ROOT/tools/verify-greenfield-case-coverage.sh" -WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml" -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT - -manifest="$TMP/expected-red.tsv" -state_root="$TMP/state" -run_id="pipeline-1" -printf '%s\n' \ - '# representative manifest comment' \ - $'# case\tkind\tkey/value' \ - '' \ - $'next-git-present\texit\t1' \ - $'main-git-present\texit\t1' \ - $'next-git-absent\texit\t1' > "$manifest" - -bash "$SUBJECT" init "$manifest" "$state_root" "$run_id" - -# Positive firing control: a skipped expected arm makes the final gate red. -bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" next-git-present -bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" main-git-present -set +e -skipped_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)" -skipped_status=$? -set -e -[[ "$skipped_status" -eq 1 ]] -grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$skipped_output" -grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$skipped_output" - -# Count inflation cannot pass: equal counts with one missing and one unexpected remain red. -: > "$state_root/$run_id/unexpected-case.ran" -set +e -inflated_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)" -inflated_status=$? -set -e -[[ "$inflated_status" -eq 1 ]] -grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$inflated_output" -grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$inflated_output" -grep -qF '[fixture-suite] unexpected_case=unexpected-case' <<<"$inflated_output" - -# Re-initializing the same run clears stale markers instead of certifying a later run. -bash "$SUBJECT" init "$manifest" "$state_root" "$run_id" -set +e -stale_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)" -stale_status=$? -set -e -[[ "$stale_status" -eq 1 ]] -grep -qF '[fixture-suite] cases_defined=3 cases_executed=0' <<<"$stale_output" - -# Exact set equality is the sole green disposition. -for case_name in next-git-present main-git-present next-git-absent; do - bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" "$case_name" -done -complete_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id")" -grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$complete_output" - -# The repository's production manifest, including its comments, initializes and checks. -production_manifest="$ROOT/tools/fixtures/greenfield-expected-red.tsv" -production_state="$TMP/production-state" -bash "$SUBJECT" init "$production_manifest" "$production_state" production-1 -for case_name in next-git-present main-git-present next-git-absent; do - bash "$SUBJECT" mark "$production_manifest" "$production_state" production-1 "$case_name" -done -production_output="$(bash "$SUBJECT" check "$production_manifest" "$production_state" production-1)" -grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output" - -# A future manifest case changes the derived set and cannot silently escape coverage. -printf '%s\n' $'future-case\texit\t1' >> "$manifest" -set +e -future_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)" -future_status=$? -set -e -[[ "$future_status" -eq 1 ]] -grep -qF '[fixture-suite] cases_defined=4 cases_executed=3' <<<"$future_output" -grep -qF '[fixture-suite] missing_case=future-case' <<<"$future_output" - -# The workflow writes each marker only after that case's per-case verifier succeeds, -# and the final check is eligible after either success or failure. -python3 - "$WORKFLOW" <<'PY' -import re -import sys -from pathlib import Path - -text = Path(sys.argv[1]).read_text(encoding="utf-8") -steps = { - match.group(1): match.group(2) - for match in re.finditer(r"(?ms)^ ([A-Za-z0-9_-]+):\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text) -} -case_steps = ( - ("greenfield-git-present", "next-git-present"), - ("greenfield-main-git-present", "main-git-present"), - ("greenfield-git-absent", "next-git-absent"), -) -for step, case in case_steps: - block = steps[step] - verify = block.index("bash tools/verify-greenfield-expected-red.sh") - mark = block.index(f"mark \\\n tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \\\n \"$coverage_run\" {case}") - assert verify < mark, f"{step} marks execution before successful verification" - -for step in ( - "greenfield-git-present", - "greenfield-main-git-present", - "greenfield-remote-installer-contract", - "greenfield-git-absent", -): - assert re.search( - r"(?m)^ depends_on:\n - greenfield-case-denominator-init$", steps[step] - ), f"{step} can race marker initialization" - -final = steps["greenfield-case-denominator"] -assert "status: [success, failure]" in final -assert "bash tools/verify-greenfield-case-coverage.sh check" in final -for dependency in ( - "greenfield-git-present", - "greenfield-main-git-present", - "greenfield-remote-installer-contract", - "greenfield-git-absent", -): - assert f" - {dependency}\n" in final, f"final gate can race {dependency}" -PY - -printf 'greenfield case coverage tests passed\n' diff --git a/tools/verify-greenfield-execution-coverage.sh b/tools/verify-greenfield-execution-coverage.sh new file mode 100755 index 00000000..dccf3980 --- /dev/null +++ b/tools/verify-greenfield-execution-coverage.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 [name]" >&2 +} + +[[ "$#" -ge 5 ]] || { usage; exit 2; } +mode="$1" +dimension="$2" +expected_set="$3" +state_root="$4" +run_id="$5" +name="${6:-}" + +[[ "$dimension" == cases || "$dimension" == arms ]] \ + || { echo "[fixture-suite] invalid coverage dimension: $dimension" >&2; exit 2; } +[[ -s "$expected_set" ]] \ + || { echo "[fixture-suite] expected set missing or empty: $expected_set" >&2; exit 2; } +[[ -n "$state_root" && "$state_root" != / ]] \ + || { echo "[fixture-suite] unsafe state root" >&2; exit 2; } +[[ "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ + || { echo "[fixture-suite] invalid run id: $run_id" >&2; exit 2; } + +singular="${dimension%s}" +run_dir="$state_root/$run_id/$dimension" + +expected_names() { + awk -F '\t' 'NF && $0 !~ /^[[:space:]]*#/ && !seen[$1]++ { sub(/\r$/, "", $1); print $1 }' \ + "$expected_set" | LC_ALL=C sort +} + +validate_expected_names() { + local found=0 expected + while IFS= read -r expected; do + [[ -n "$expected" ]] || continue + found=1 + [[ "$expected" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ + || { echo "[fixture-suite] invalid expected $singular name: $expected" >&2; return 1; } + done < <(expected_names) + [[ "$found" -eq 1 ]] \ + || { echo "[fixture-suite] expected set defines no $dimension" >&2; return 1; } +} + +validate_expected_names + +case "$mode" in + init) + [[ "$#" -eq 5 ]] || { usage; exit 2; } + mkdir -p "$run_dir" + find "$run_dir" -mindepth 1 -delete + ;; + mark) + [[ "$#" -eq 6 ]] || { usage; exit 2; } + [[ "$name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ + || { echo "[fixture-suite] invalid $singular marker name: $name" >&2; exit 2; } + [[ -d "$run_dir" ]] \ + || { echo "[fixture-suite] $dimension state was not initialized: $run_id" >&2; exit 1; } + expected_names | grep -Fxq -- "$name" \ + || { echo "[fixture-suite] $singular is not in the expected set: $name" >&2; exit 1; } + : > "$run_dir/$name.ran" + ;; + check) + [[ "$#" -eq 5 ]] || { usage; exit 2; } + mkdir -p "$run_dir" + expected_file="$(mktemp "$run_dir/.expected.XXXXXX")" + actual_file="$(mktemp "$run_dir/.actual.XXXXXX")" + cleanup() { rm -f "$expected_file" "$actual_file"; } + trap cleanup EXIT + + expected_names > "$expected_file" + find "$run_dir" -mindepth 1 -maxdepth 1 -type f -name '*.ran' -printf '%f\n' \ + | sed 's/\.ran$//' | LC_ALL=C sort -u > "$actual_file" + + defined="$(wc -l < "$expected_file" | tr -d ' ')" + executed="$(wc -l < "$actual_file" | tr -d ' ')" + printf '[fixture-suite] %s_defined=%s %s_executed=%s\n' \ + "$dimension" "$defined" "$dimension" "$executed" + + if ! cmp -s "$expected_file" "$actual_file"; then + while IFS= read -r missing; do + [[ -n "$missing" ]] \ + && printf '[fixture-suite] missing_%s=%s\n' "$singular" "$missing" >&2 + done < <(comm -23 "$expected_file" "$actual_file") + while IFS= read -r unexpected; do + [[ -n "$unexpected" ]] \ + && printf '[fixture-suite] unexpected_%s=%s\n' "$singular" "$unexpected" >&2 + done < <(comm -13 "$expected_file" "$actual_file") + exit 1 + fi + ;; + *) + usage + exit 2 + ;; +esac diff --git a/tools/verify-greenfield-execution-coverage.test.sh b/tools/verify-greenfield-execution-coverage.test.sh new file mode 100755 index 00000000..187ed8fb --- /dev/null +++ b/tools/verify-greenfield-execution-coverage.test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SUBJECT="$ROOT/tools/verify-greenfield-execution-coverage.sh" +WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml" +mkdir -p "$ROOT/.mosaic-test-work" +TMP="$(mktemp -d "$ROOT/.mosaic-test-work/execution-coverage.XXXXXX")" +selector_control="$ROOT/greenfield-coverage-selector-control.ran" +cleanup() { + rm -f "$selector_control" + rm -rf "$TMP" +} +trap cleanup EXIT + +manifest="$TMP/expected-red.tsv" +state_root="$TMP/state" +run_id="pipeline-1" +printf '%s\n' \ + '# representative manifest comment' \ + $'# case\tkind\tkey/value' \ + '' \ + $'next-git-present\texit\t1' \ + $'main-git-present\texit\t1' \ + $'next-git-absent\texit\t1' > "$manifest" + +bash "$SUBJECT" init cases "$manifest" "$state_root" "$run_id" + +# Positive firing control: a skipped expected case makes the final gate red. +bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" next-git-present +bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" main-git-present +set +e +skipped_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)" +skipped_status=$? +set -e +[[ "$skipped_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$skipped_output" +grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$skipped_output" + +# Count inflation cannot pass: equal counts with one missing and one unexpected remain red. +: > "$state_root/$run_id/cases/unexpected-case.ran" +set +e +inflated_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)" +inflated_status=$? +set -e +[[ "$inflated_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$inflated_output" +grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$inflated_output" +grep -qF '[fixture-suite] unexpected_case=unexpected-case' <<<"$inflated_output" + +# Re-initializing the same run clears stale markers instead of certifying a later run. +bash "$SUBJECT" init cases "$manifest" "$state_root" "$run_id" +set +e +stale_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)" +stale_status=$? +set -e +[[ "$stale_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=3 cases_executed=0' <<<"$stale_output" + +# Exact case-set equality is green. +for case_name in next-git-present main-git-present next-git-absent; do + bash "$SUBJECT" mark cases "$manifest" "$state_root" "$run_id" "$case_name" +done +complete_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id")" +grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$complete_output" + +# The production manifest, including comments, initializes and checks. +production_manifest="$ROOT/tools/fixtures/greenfield-expected-red.tsv" +production_arms="$ROOT/tools/fixtures/greenfield-expected-arms.txt" +production_state="$TMP/production-state" +bash "$SUBJECT" init cases "$production_manifest" "$production_state" production-1 +for case_name in next-git-present main-git-present next-git-absent; do + bash "$SUBJECT" mark cases "$production_manifest" "$production_state" production-1 "$case_name" +done +production_output="$(bash "$SUBJECT" check cases "$production_manifest" "$production_state" production-1)" +grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output" + +# A future manifest case changes the derived set and cannot silently escape coverage. +printf '%s\n' $'future-case\texit\t1' >> "$manifest" +set +e +future_output="$(bash "$SUBJECT" check cases "$manifest" "$state_root" "$run_id" 2>&1)" +future_status=$? +set -e +[[ "$future_status" -eq 1 ]] +grep -qF '[fixture-suite] cases_defined=4 cases_executed=3' <<<"$future_output" +grep -qF '[fixture-suite] missing_case=future-case' <<<"$future_output" + +# A skipped remote arm is red even while the three case identities are complete. +bash "$SUBJECT" init arms "$production_arms" "$production_state" production-1 +for arm_name in greenfield-git-present greenfield-main-git-present greenfield-git-absent; do + bash "$SUBJECT" mark arms "$production_arms" "$production_state" production-1 "$arm_name" +done +set +e +remote_skipped_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1 2>&1)" +remote_skipped_status=$? +set -e +[[ "$remote_skipped_status" -eq 1 ]] +grep -qF '[fixture-suite] arms_defined=4 arms_executed=3' <<<"$remote_skipped_output" +grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$remote_skipped_output" + +# Arm counts cannot be inflated either. +: > "$production_state/production-1/arms/unexpected-arm.ran" +set +e +arm_inflated_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1 2>&1)" +arm_inflated_status=$? +set -e +[[ "$arm_inflated_status" -eq 1 ]] +grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$arm_inflated_output" +grep -qF '[fixture-suite] missing_arm=greenfield-remote-installer-contract' <<<"$arm_inflated_output" +grep -qF '[fixture-suite] unexpected_arm=unexpected-arm' <<<"$arm_inflated_output" + +# Exact arm-set equality is green. +bash "$SUBJECT" init arms "$production_arms" "$production_state" production-1 +for arm_name in \ + greenfield-git-present \ + greenfield-main-git-present \ + greenfield-remote-installer-contract \ + greenfield-git-absent; do + bash "$SUBJECT" mark arms "$production_arms" "$production_state" production-1 "$arm_name" +done +arm_complete_output="$(bash "$SUBJECT" check arms "$production_arms" "$production_state" production-1)" +grep -qF '[fixture-suite] arms_defined=4 arms_executed=4' <<<"$arm_complete_output" + +# The exact checkout-archive selector excludes coverage state under .mosaic-test-work. +: > "$selector_control" +archive="$TMP/checkout.tar.gz" +archive_list="$TMP/checkout.list" +repo_parent="$(dirname "$ROOT")" +repo_name="$(basename "$ROOT")" +tar -C "$repo_parent" \ + --exclude='*/.git' --exclude='*/node_modules' --exclude='*/dist' \ + --exclude='*/coverage' --exclude='*/.turbo' --exclude='*/.mosaic-test-work' \ + --exclude='*/.env' --exclude='*/.env.*' \ + -czf "$archive" "$repo_name" +tar -tzf "$archive" > "$archive_list" +grep -qF "$repo_name/greenfield-coverage-selector-control.ran" "$archive_list" +if grep -qF "$repo_name/.mosaic-test-work/" "$archive_list"; then + echo 'coverage state leaked into checkout archive' >&2 + exit 1 +fi + +# Workflow contract: success-only case+arm markers, excluded state root, and complete ordering. +python3 - "$WORKFLOW" <<'PY' +import re +import sys +from pathlib import Path + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +steps = { + match.group(1): match.group(2) + for match in re.finditer(r"(?ms)^ ([A-Za-z0-9_-]+):\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text) +} +case_steps = ( + ("greenfield-git-present", "next-git-present"), + ("greenfield-main-git-present", "main-git-present"), + ("greenfield-git-absent", "next-git-absent"), +) +for step, case in case_steps: + block = steps[step] + verify = block.index("bash tools/verify-greenfield-expected-red.sh") + mark = block.index(f'"$coverage_run" {case}') + assert verify < mark, f"{step} marks its case before successful verification" + +arm_steps = ( + "greenfield-git-present", + "greenfield-main-git-present", + "greenfield-remote-installer-contract", + "greenfield-git-absent", +) +for step in arm_steps: + block = steps[step] + verify = block.index("bash tools/verify-greenfield-expected-red.sh") + mark = block.index(f'"$coverage_run" {step}') + assert verify < mark, f"{step} marks its arm before successful verification" + assert re.search( + r"(?m)^ depends_on:\n - greenfield-case-denominator-init$", block + ), f"{step} can race marker initialization" + +assert ".greenfield-case-state" not in text +assert ".mosaic-test-work/greenfield-execution-coverage" in text +final = steps["greenfield-case-denominator"] +assert "status: [success, failure]" in final +assert "check cases" in final and "check arms" in final +assert "cases_status" in final and "arms_status" in final +for dependency in arm_steps: + assert f" - {dependency}\n" in final, f"final gate can race {dependency}" +PY + +printf 'greenfield execution coverage tests passed\n'