ci(installer): enforce greenfield case coverage
This commit is contained in:
@@ -6,8 +6,19 @@ when:
|
|||||||
branch: [next, main]
|
branch: [next, main]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
|
greenfield-case-denominator-init:
|
||||||
|
image: node:22-bookworm-slim
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}"
|
||||||
|
bash tools/verify-greenfield-case-coverage.sh init \
|
||||||
|
tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \
|
||||||
|
"$coverage_run"
|
||||||
|
|
||||||
greenfield-git-present:
|
greenfield-git-present:
|
||||||
image: node:22-bookworm-slim
|
image: node:22-bookworm-slim
|
||||||
|
depends_on:
|
||||||
|
- greenfield-case-denominator-init
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set +e
|
set +e
|
||||||
@@ -19,9 +30,15 @@ steps:
|
|||||||
cat /tmp/greenfield-git-present.log
|
cat /tmp/greenfield-git-present.log
|
||||||
bash tools/verify-greenfield-expected-red.sh \
|
bash tools/verify-greenfield-expected-red.sh \
|
||||||
next-git-present /tmp/greenfield-git-present.log "$fixture_status"
|
next-git-present /tmp/greenfield-git-present.log "$fixture_status"
|
||||||
|
coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}"
|
||||||
|
bash tools/verify-greenfield-case-coverage.sh mark \
|
||||||
|
tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \
|
||||||
|
"$coverage_run" next-git-present
|
||||||
|
|
||||||
greenfield-main-git-present:
|
greenfield-main-git-present:
|
||||||
image: node:22-bookworm-slim
|
image: node:22-bookworm-slim
|
||||||
|
depends_on:
|
||||||
|
- greenfield-case-denominator-init
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set +e
|
set +e
|
||||||
@@ -33,9 +50,15 @@ steps:
|
|||||||
cat /tmp/greenfield-main-git-present.log
|
cat /tmp/greenfield-main-git-present.log
|
||||||
bash tools/verify-greenfield-expected-red.sh \
|
bash tools/verify-greenfield-expected-red.sh \
|
||||||
main-git-present /tmp/greenfield-main-git-present.log "$fixture_status"
|
main-git-present /tmp/greenfield-main-git-present.log "$fixture_status"
|
||||||
|
coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}"
|
||||||
|
bash tools/verify-greenfield-case-coverage.sh mark \
|
||||||
|
tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \
|
||||||
|
"$coverage_run" main-git-present
|
||||||
|
|
||||||
greenfield-remote-installer-contract:
|
greenfield-remote-installer-contract:
|
||||||
image: node:22-bookworm-slim
|
image: node:22-bookworm-slim
|
||||||
|
depends_on:
|
||||||
|
- greenfield-case-denominator-init
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
expected="$(awk 'NF {print $1; exit}' tools/install.sh.sha256)"
|
expected="$(awk 'NF {print $1; exit}' tools/install.sh.sha256)"
|
||||||
@@ -56,6 +79,8 @@ steps:
|
|||||||
|
|
||||||
greenfield-git-absent:
|
greenfield-git-absent:
|
||||||
image: node:22-bookworm-slim
|
image: node:22-bookworm-slim
|
||||||
|
depends_on:
|
||||||
|
- greenfield-case-denominator-init
|
||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set +e
|
set +e
|
||||||
@@ -67,3 +92,24 @@ steps:
|
|||||||
cat /tmp/greenfield-git-absent.log
|
cat /tmp/greenfield-git-absent.log
|
||||||
bash tools/verify-greenfield-expected-red.sh \
|
bash tools/verify-greenfield-expected-red.sh \
|
||||||
next-git-absent /tmp/greenfield-git-absent.log "$fixture_status"
|
next-git-absent /tmp/greenfield-git-absent.log "$fixture_status"
|
||||||
|
coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}"
|
||||||
|
bash tools/verify-greenfield-case-coverage.sh mark \
|
||||||
|
tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \
|
||||||
|
"$coverage_run" next-git-absent
|
||||||
|
|
||||||
|
greenfield-case-denominator:
|
||||||
|
image: node:22-bookworm-slim
|
||||||
|
# Publish exact execution coverage after the full matrix, even if a case failed.
|
||||||
|
depends_on:
|
||||||
|
- greenfield-git-present
|
||||||
|
- greenfield-main-git-present
|
||||||
|
- greenfield-remote-installer-contract
|
||||||
|
- greenfield-git-absent
|
||||||
|
when:
|
||||||
|
- status: [success, failure]
|
||||||
|
commands:
|
||||||
|
- |
|
||||||
|
coverage_run="${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}"
|
||||||
|
bash tools/verify-greenfield-case-coverage.sh check \
|
||||||
|
tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \
|
||||||
|
"$coverage_run"
|
||||||
|
|||||||
+2
-1
@@ -1393,13 +1393,14 @@ A from-zero install can report success while leaving the target host unusable be
|
|||||||
12. The expected-RED comparator SHALL validate the complete manifest before selecting a case: exact case population, one exit and P0–P9 disposition per case, pinned require/forbid classes, and no malformed, duplicate, or unknown rows.
|
12. The expected-RED comparator SHALL validate the complete manifest before selecting a case: exact case population, one exit and P0–P9 disposition per case, pinned require/forbid classes, and no malformed, duplicate, or unknown rows.
|
||||||
13. The published installer contract SHALL reject failed fetches, HTTP-success empty bodies, and digest mismatch, then execute the exact digest-verified body. The remote CI arm SHALL enumerate every payload-acquisition path and report a bound/found denominator. It SHALL bind both the installer body and the downstream stack framework/source it consumes to the same immutable CI commit, while retaining `--next` as the lane selector. Source resolution/acquisition SHALL fail closed, and the arm SHALL verify the realised source commit and archive digest after installation. Any out-of-scope unpinned sibling SHALL be named and counted rather than silently included in a broader pinning claim. `ASSUMPTION:` the configured repository's authenticated exact-commit endpoint is trusted to map that commit ID to the returned archive bytes; independent signed provenance/authenticity against repository or TLS trust-root compromise remains excluded by canonical greenfield-install PRD v2 §3.
|
13. The published installer contract SHALL reject failed fetches, HTTP-success empty bodies, and digest mismatch, then execute the exact digest-verified body. The remote CI arm SHALL enumerate every payload-acquisition path and report a bound/found denominator. It SHALL bind both the installer body and the downstream stack framework/source it consumes to the same immutable CI commit, while retaining `--next` as the lane selector. Source resolution/acquisition SHALL fail closed, and the arm SHALL verify the realised source commit and archive digest after installation. Any out-of-scope unpinned sibling SHALL be named and counted rather than silently included in a broader pinning claim. `ASSUMPTION:` the configured repository's authenticated exact-commit endpoint is trusted to map that commit ID to the returned archive bytes; independent signed provenance/authenticity against repository or TLS trust-root compromise remains excluded by canonical greenfield-install PRD v2 §3.
|
||||||
14. Phase diagnostics SHALL be redacted before terminal or durable-log output. A seeded positive-control canary SHALL remain absent from observed argv, output, command logs, npm configuration, generated files, and shell history.
|
14. Phase diagnostics SHALL be redacted before terminal or durable-log output. A seeded positive-control canary SHALL remain absent from observed argv, output, command logs, npm configuration, generated files, and shell history.
|
||||||
|
15. The CI fixture SHALL publish pipeline-level execution coverage for the unconditional case set derived from the expected-RED manifest. Per-case success markers SHALL be run-scoped and written only after that case's verifier passes. A final step SHALL depend on the complete fixture matrix, run after prior success or failure, emit `cases_defined=N cases_executed=M`, and fail unless the expected and executed case-name sets are exactly equal; missing, unexpected, stale, or newly added unexecuted cases SHALL fail closed.
|
||||||
|
|
||||||
### C1 acceptance criteria
|
### C1 acceptance criteria
|
||||||
|
|
||||||
1. The pre-C1 from-zero matrix records both discriminating controls: with `git` absent, the legacy installer still exits zero while P1 fails and skill sync degrades; with `git` present, P1 passes and the observed sync store/runtime links are 101/101. The C1 installer must fail at P1 before mutation when `git` is absent.
|
1. The pre-C1 from-zero matrix records both discriminating controls: with `git` absent, the legacy installer still exits zero while P1 fails and skill sync degrades; with `git` present, P1 passes and the observed sync store/runtime links are 101/101. The C1 installer must fail at P1 before mutation when `git` is absent.
|
||||||
2. The discriminating P3 row passes: the binary exists at the expected absolute path and reports exactly the resolved `next` lane version, while P4, P5, and P8 fail.
|
2. The discriminating P3 row passes: the binary exists at the expected absolute path and reports exactly the resolved `next` lane version, while P4, P5, and P8 fail.
|
||||||
3. The `--check` mutation negative control proves host fingerprints are byte-identical before and after observation.
|
3. The `--check` mutation negative control proves host fingerprints are byte-identical before and after observation.
|
||||||
4. Woodpecker executes and validates the expected RED fixture plus the immutable remote-installer contract; C1 does not repair P4/P5/P8 or activate #869.
|
4. Woodpecker executes and validates the expected RED fixture plus the immutable remote-installer contract; its pipeline-level case-coverage gate reports exact manifest-derived set equality and a skipped-arm control proves non-execution is red. C1 does not repair P4/P5/P8 or activate #869.
|
||||||
5. Negative controls prove manifest shrink/duplicates/unknown rows fail, unsafe P0/P3/P4/P5 contexts fail, the P2–P8 fault seam enters real actions rather than synthetic writes, empty/mismatched fetched bodies fail, and a deliberately emitted secret canary is redacted from every persisted/output population.
|
5. Negative controls prove manifest shrink/duplicates/unknown rows fail, unsafe P0/P3/P4/P5 contexts fail, the P2–P8 fault seam enters real actions rather than synthetic writes, empty/mismatched fetched bodies fail, and a deliberately emitted secret canary is redacted from every persisted/output population.
|
||||||
|
|
||||||
### Explicit exclusions and dependencies
|
### Explicit exclusions and dependencies
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#1050 C1 ROUND 4 — PIPELINE CASE-COVERAGE DENOMINATOR
|
||||||
|
|
||||||
|
IDENTITY / SCOPE
|
||||||
|
- predecessor head: 0e2eef1c1261444b83c089df6047a2a06592d5c6
|
||||||
|
- branch: feat/1050-install-state-machine-red-fixture
|
||||||
|
- only open requirement: pipeline-level expected-case/executed-case set equality
|
||||||
|
- unchanged/fenced: tools/fixtures/greenfield-expected-red.tsv, tools/verify-greenfield-expected-red.sh, #869, P6 detector/producer, absolute-P3 CLI linking, and per-case fixture dispositions
|
||||||
|
- status language: believed-fixed, pending jarvis validation
|
||||||
|
|
||||||
|
STALE-RULING WITHDRAWAL HONOURED
|
||||||
|
Requirements 1-3 were withdrawn after pipeline 2242 at 0e2eef1c reached terminal green and showed P6 failure propagation, absolute-P3 CLI use, and an executed git-absent arm. A temporary local P6-consumer edit made while the stale ruling was in flight was restored before this change set; tools/e2e-install-test.sh has no final-tree delta. Requirement 5 alone remains in this round.
|
||||||
|
|
||||||
|
RED-FIRST
|
||||||
|
Before the change, .woodpecker/greenfield-install.yml had no pipeline-level initialized marker store, no per-case success marker, and no final exact-set gate. A static assertion for a final run-on-failure case-coverage step failed. Pipeline 2242's green result proved that all three cases ran once, but no gate required that population on a later run.
|
||||||
|
|
||||||
|
IMPLEMENTATION
|
||||||
|
- tools/verify-greenfield-case-coverage.sh is a new pipeline-level instrument; the existing per-case verifier is unchanged.
|
||||||
|
- Expected case names are derived from the first TSV field of tools/fixtures/greenfield-expected-red.tsv and sorted uniquely. There is no literal expected count.
|
||||||
|
- Marker storage is scoped by ${CI_PIPELINE_NUMBER}-${CI_WORKFLOW_NUMBER}. The init action validates the manifest and empties only that run directory, so stale markers cannot satisfy a retry or later run.
|
||||||
|
- Every case/contract step explicitly depends on initialization. Each unconditional case step calls mark only after its exact tools/verify-greenfield-expected-red.sh invocation succeeds.
|
||||||
|
- The final greenfield-case-denominator step explicitly depends on all four case/contract steps, has status eligibility [success, failure], emits cases_defined=N cases_executed=M, and compares sorted expected and actual case-name sets. Missing or unexpected names fail even when the two counts are equal.
|
||||||
|
- A newly defined manifest case changes the expected set automatically and fails until a corresponding successful case step marks it.
|
||||||
|
|
||||||
|
MEASURED CONTROLS
|
||||||
|
1. Skipped expected arm:
|
||||||
|
rc=1
|
||||||
|
[fixture-suite] cases_defined=3 cases_executed=2
|
||||||
|
[fixture-suite] missing_case=next-git-absent
|
||||||
|
2. Count-inflation control (one expected missing, one unexpected added):
|
||||||
|
rc=1
|
||||||
|
[fixture-suite] cases_defined=3 cases_executed=3
|
||||||
|
[fixture-suite] missing_case=next-git-absent
|
||||||
|
[fixture-suite] unexpected_case=unexpected-case
|
||||||
|
3. Stale-marker initialization control (same run re-initialized after markers existed):
|
||||||
|
rc=1
|
||||||
|
[fixture-suite] cases_defined=3 cases_executed=0
|
||||||
|
missing_case rows emitted for all three expected names
|
||||||
|
4. Exact-set positive control:
|
||||||
|
rc=0
|
||||||
|
[fixture-suite] cases_defined=3 cases_executed=3
|
||||||
|
5. Future manifest case control:
|
||||||
|
rc=1
|
||||||
|
[fixture-suite] cases_defined=4 cases_executed=3
|
||||||
|
[fixture-suite] missing_case=future-case
|
||||||
|
|
||||||
|
WORKFLOW / FOCUSED VALIDATION
|
||||||
|
- bash -n new subject and test: PASS
|
||||||
|
- shellcheck new subject and test: PASS
|
||||||
|
- bash tools/verify-greenfield-case-coverage.test.sh: PASS
|
||||||
|
- test statically proves each mark follows its exact per-case verifier, every producer depends on initialization, the final gate depends on the complete matrix, and final status eligibility includes success+failure
|
||||||
|
- woodpecker-cli lint --strict .woodpecker/greenfield-install.yml: PASS
|
||||||
|
- Prettier check for workflow/package/PRD: PASS
|
||||||
|
- git diff --check: PASS
|
||||||
|
|
||||||
|
CURRENT-HEAD FIXTURE MEASUREMENTS TAKEN BEFORE THE STALE RULING WAS WITHDRAWN
|
||||||
|
These measurements were run from 0e2eef1c before requirement 5 changed only pipeline instrumentation. They are retained as measured outcomes, not as justification for modifying requirements 1-3.
|
||||||
|
- next checkout/git-present: fixture rc=1; resolved @next=0.0.50-next.2207; P0/P1/P2/P3/P7 PASS; P4/P5/P6/P8/P9 FAIL; installer_exit=1; done_claims=0; P6 reason was dead enforcement hooks active count=2.
|
||||||
|
- next checkout/git-absent: fixture rc=1; P0/P6/P7 PASS; P1/P2/P3/P4/P5/P8/P9 FAIL; installer_exit=1; done_claims=0.
|
||||||
|
- /home free before the first run: 8.5G; after concurrent authorized fleet reclaim completed: 11G; no be-coder-05 generated payload was deleted because active fixture work made that churn.
|
||||||
|
|
||||||
|
INDEPENDENT REVIEW CYCLE
|
||||||
|
- Initial Codex code review: REQUEST_CHANGES, confidence 0.96; one blocker found that the first draft omitted depends_on ordering and could race initialization/final checking.
|
||||||
|
- Disposition: accepted. Added explicit init -> all case/contract steps -> final dependency graph and a regression assertion for every dependency edge.
|
||||||
|
- Initial Codex security review: risk NONE, confidence 0.94, zero findings. The sandbox could not create its own temp directory; local focused/full tests provide the dynamic evidence.
|
||||||
|
- First post-dependency code re-review: REQUEST_CHANGES, confidence 0.99; the case extractor treated the production manifest's comment lines as case names, while the synthetic test omitted comments.
|
||||||
|
- Disposition: accepted. Case derivation now excludes blank/comment lines; the focused test includes representative comments and executes init/mark/check against the repository's real manifest.
|
||||||
|
- Post-dependency security re-review: risk NONE, confidence 0.96, zero findings.
|
||||||
|
- Final Codex code re-review after the comment fix: APPROVE, confidence 0.94, 8 files, zero findings. It confirmed ordering, run scope, fail-closed set equality, PRD alignment, and negative controls.
|
||||||
|
- Final Codex security re-review: risk NONE, confidence 0.96, 8 files, zero findings.
|
||||||
|
|
||||||
|
FINAL LOCAL BASELINES
|
||||||
|
- pnpm test:installer: PASS, including the new coverage suite
|
||||||
|
- pnpm typecheck: PASS, 45/45
|
||||||
|
- pnpm lint: PASS, 25/25
|
||||||
|
- pnpm format:check: PASS
|
||||||
|
- bash -n + ShellCheck for new shell surfaces: PASS
|
||||||
|
- Woodpecker strict lint: PASS
|
||||||
|
- git diff --check: PASS
|
||||||
|
- /home free before/after local baselines: 11G
|
||||||
|
|
||||||
|
PENDING BEFORE PUSH
|
||||||
|
- commit, queue guard, one push, provider attribution read-back
|
||||||
|
- announce the new full SHA to rev-security-02; do not merge or close
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## Required artifacts
|
## Required artifacts
|
||||||
|
|
||||||
- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 now binds the complete in-scope remote stack source chain and requires a per-path denominator.
|
- [x] `docs/PRD.md` exists; #1050 C1 requirement 13 binds the complete in-scope remote stack source chain, and requirement 15 binds pipeline-level exact case-set execution coverage.
|
||||||
- [x] User guide: not applicable; no user-facing production installer behavior changed.
|
- [x] User guide: not applicable; no user-facing production installer behavior changed.
|
||||||
- [x] Admin guide: not applicable; no operator procedure or deployment behavior changed.
|
- [x] Admin guide: not applicable; no operator procedure or deployment behavior changed.
|
||||||
- [x] Developer guide: existing `docs/guides/installer-state-machine.md` defines the fail-closed installer model; the C1-specific remote-arm contract and evidence live in the PRD, scratchpad, and scoped verification report.
|
- [x] Developer guide: existing `docs/guides/installer-state-machine.md` defines the fail-closed installer model; the C1-specific remote-arm contract and evidence live in the PRD, scratchpad, and scoped verification report.
|
||||||
@@ -20,4 +20,5 @@
|
|||||||
- [x] Verification documentation is in the same logical change set as the shell-test remediation.
|
- [x] Verification documentation is in the same logical change set as the shell-test remediation.
|
||||||
- [x] Trust boundary states both halves: the digest proves fetched bytes remain identical through execution/verify-after; it does not authenticate authorship against provider/TLS compromise. Independent provenance is the inherited PRD v2 §3 deferral.
|
- [x] Trust boundary states both halves: the digest proves fetched bytes remain identical through execution/verify-after; it does not authenticate authorship against provider/TLS compromise. Independent provenance is the inherited PRD v2 §3 deferral.
|
||||||
- [x] Round-3 Codex code review APPROVE (confidence 0.92, zero findings) and security re-review risk NONE (confidence 0.96, zero findings). The initial HIGH trust-root finding remains documented as the canonical signed-provenance deferral, not hidden.
|
- [x] Round-3 Codex code review APPROVE (confidence 0.92, zero findings) and security re-review risk NONE (confidence 0.96, zero findings). The initial HIGH trust-root finding remains documented as the canonical signed-provenance deferral, not hidden.
|
||||||
|
- [x] Round-4 review cycle closed: two blocking Codex findings (missing dependency ordering; production manifest comments misparsed) were accepted and fixed. Final code review APPROVE (confidence 0.94, zero findings) and security risk NONE (confidence 0.96, zero findings).
|
||||||
- [x] Canonical evidence remains in-repo. No external publishing action was requested or performed.
|
- [x] Canonical evidence remains in-repo. No external publishing action was requested or performed.
|
||||||
|
|||||||
@@ -144,3 +144,10 @@ Denominator corrections were sent to and accepted by the lane lead before implem
|
|||||||
- Trust boundary: the exact-commit URL trusts the configured repository provider's authenticated commit-to-archive mapping. The computed digest pins transfer/consumption but does not authenticate against repository/TLS compromise; signed provenance remains the canonical PRD v2 §3 deferral. Initial Codex security review retained this as HIGH/CWE-494; no stronger claim or out-of-scope signing change was made.
|
- Trust boundary: the exact-commit URL trusts the configured repository provider's authenticated commit-to-archive mapping. The computed digest pins transfer/consumption but does not authenticate against repository/TLS compromise; signed provenance remains the canonical PRD v2 §3 deferral. Initial Codex security review retained this as HIGH/CWE-494; no stronger claim or out-of-scope signing change was made.
|
||||||
- Final baselines: Bash syntax, ShellCheck, `pnpm test:installer`, `pnpm typecheck` (45/45), `pnpm lint` (25/25), `pnpm format:check`, and `git diff --check` pass. Codex code review APPROVE confidence 0.92 with zero findings; after explicit trust-boundary documentation, security re-review risk NONE confidence 0.96 with zero findings. The initial HIGH trust-root finding remains recorded as the signed-provenance deferral.
|
- Final baselines: Bash syntax, ShellCheck, `pnpm test:installer`, `pnpm typecheck` (45/45), `pnpm lint` (25/25), `pnpm format:check`, and `git diff --check` pass. Codex code review APPROVE confidence 0.92 with zero findings; after explicit trust-boundary documentation, security re-review risk NONE confidence 0.96 with zero findings. The initial HIGH trust-root finding remains recorded as the signed-provenance deferral.
|
||||||
- Full evidence and named paths: `docs/reports/verification/1050-c1-fix-round/09-round3-source-binding.txt`.
|
- Full evidence and named paths: `docs/reports/verification/1050-c1-fix-round/09-round3-source-binding.txt`.
|
||||||
|
|
||||||
|
## Round 4 — pipeline case-coverage denominator
|
||||||
|
|
||||||
|
- Pipeline 2242 at `0e2eef1c` superseded the lane lead's earlier pipeline-2229 ruling: requirements 1–3 were already satisfied. The temporary local P6-consumer edit started while that ruling was in flight was restored; `tools/e2e-install-test.sh`, the expected-RED manifest, the per-case verifier, and #869 remain unchanged in the final tree.
|
||||||
|
- Requirement 5 adds a pipeline-level instrument above the three per-case invocations. Expected names are derived from the manifest, markers are scoped by pipeline+workflow run, and each marker is written only after that exact per-case verifier succeeds. Every producer depends on initialization; the final step depends on the complete case/contract matrix, runs after success or failure, and requires exact expected/actual set equality rather than count equality.
|
||||||
|
- Measured firing controls: skipped arm `3/2` rc1; count inflation `3/3` rc1 with one missing and one unexpected name; stale re-initialized run `3/0` rc1; exact set `3/3` rc0; future manifest case `4/3` rc1. Codex review found and blocked two independent defects: first the missing `depends_on` graph, then comment lines being parsed as case names. Both were accepted; dependency edges are regression-asserted, blank/comments are excluded, and the focused test now consumes the production manifest directly. Final Codex code re-review APPROVE confidence 0.94 and security re-review risk NONE confidence 0.96, both with zero findings. Woodpecker strict lint, Bash syntax, ShellCheck, focused/full installer tests, typecheck, lint, Prettier, and diff check pass.
|
||||||
|
- Full evidence: `docs/reports/verification/1050-c1-fix-round/10-round4-case-coverage.txt`.
|
||||||
|
|||||||
+1
-1
@@ -11,7 +11,7 @@
|
|||||||
"typecheck": "pnpm preflight && turbo run typecheck",
|
"typecheck": "pnpm preflight && turbo run typecheck",
|
||||||
"test:checkout": "node --test scripts/*.test.mjs",
|
"test:checkout": "node --test scripts/*.test.mjs",
|
||||||
"test": "pnpm test:checkout && turbo run test && pnpm run test:installer",
|
"test": "pnpm test:checkout && turbo run test && pnpm run test:installer",
|
||||||
"test:installer": "bash tools/install-state-machine.test.sh && bash tools/install-next-lane.test.sh && bash tools/verify-greenfield-expected-red.test.sh && bash tools/verified-installer-fetch.test.sh",
|
"test:installer": "bash tools/install-state-machine.test.sh && bash tools/install-next-lane.test.sh && bash tools/verify-greenfield-expected-red.test.sh && bash tools/verify-greenfield-case-coverage.test.sh && bash tools/verified-installer-fetch.test.sh",
|
||||||
"format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"",
|
"format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"",
|
||||||
"format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"",
|
"format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"",
|
||||||
"prepare": "node scripts/install-hooks.mjs"
|
"prepare": "node scripts/install-hooks.mjs"
|
||||||
|
|||||||
Executable
+88
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "usage: $0 <init|mark|check> <manifest> <state-root> <run-id> [case]" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
[[ "$#" -ge 4 ]] || { usage; exit 2; }
|
||||||
|
mode="$1"
|
||||||
|
manifest="$2"
|
||||||
|
state_root="$3"
|
||||||
|
run_id="$4"
|
||||||
|
case_name="${5:-}"
|
||||||
|
|
||||||
|
[[ -s "$manifest" ]] || { echo "[fixture-suite] manifest missing or empty: $manifest" >&2; exit 2; }
|
||||||
|
[[ -n "$state_root" && "$state_root" != / ]] \
|
||||||
|
|| { echo "[fixture-suite] unsafe state root" >&2; exit 2; }
|
||||||
|
[[ "$run_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||||
|
|| { echo "[fixture-suite] invalid run id: $run_id" >&2; exit 2; }
|
||||||
|
|
||||||
|
run_dir="$state_root/$run_id"
|
||||||
|
|
||||||
|
expected_cases() {
|
||||||
|
awk -F '\t' 'NF && $0 !~ /^[[:space:]]*#/ && !seen[$1]++ { print $1 }' "$manifest" | LC_ALL=C sort
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_expected_cases() {
|
||||||
|
local found=0 expected
|
||||||
|
while IFS= read -r expected; do
|
||||||
|
[[ -n "$expected" ]] || continue
|
||||||
|
found=1
|
||||||
|
[[ "$expected" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||||
|
|| { echo "[fixture-suite] invalid manifest case name: $expected" >&2; return 1; }
|
||||||
|
done < <(expected_cases)
|
||||||
|
[[ "$found" -eq 1 ]] \
|
||||||
|
|| { echo "[fixture-suite] manifest defines no cases" >&2; return 1; }
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_expected_cases
|
||||||
|
|
||||||
|
case "$mode" in
|
||||||
|
init)
|
||||||
|
[[ "$#" -eq 4 ]] || { usage; exit 2; }
|
||||||
|
mkdir -p "$run_dir"
|
||||||
|
find "$run_dir" -mindepth 1 -delete
|
||||||
|
;;
|
||||||
|
mark)
|
||||||
|
[[ "$#" -eq 5 ]] || { usage; exit 2; }
|
||||||
|
[[ "$case_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|
||||||
|
|| { echo "[fixture-suite] invalid case marker name: $case_name" >&2; exit 2; }
|
||||||
|
[[ -d "$run_dir" ]] \
|
||||||
|
|| { echo "[fixture-suite] run state was not initialized: $run_id" >&2; exit 1; }
|
||||||
|
expected_cases | grep -Fxq -- "$case_name" \
|
||||||
|
|| { echo "[fixture-suite] case is not defined by manifest: $case_name" >&2; exit 1; }
|
||||||
|
: > "$run_dir/$case_name.ran"
|
||||||
|
;;
|
||||||
|
check)
|
||||||
|
[[ "$#" -eq 4 ]] || { usage; exit 2; }
|
||||||
|
mkdir -p "$run_dir"
|
||||||
|
expected_file="$(mktemp "$run_dir/.expected.XXXXXX")"
|
||||||
|
actual_file="$(mktemp "$run_dir/.actual.XXXXXX")"
|
||||||
|
cleanup() { rm -f "$expected_file" "$actual_file"; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
expected_cases > "$expected_file"
|
||||||
|
find "$run_dir" -mindepth 1 -maxdepth 1 -type f -name '*.ran' -printf '%f\n' \
|
||||||
|
| sed 's/\.ran$//' | LC_ALL=C sort -u > "$actual_file"
|
||||||
|
|
||||||
|
cases_defined="$(wc -l < "$expected_file" | tr -d ' ')"
|
||||||
|
cases_executed="$(wc -l < "$actual_file" | tr -d ' ')"
|
||||||
|
printf '[fixture-suite] cases_defined=%s cases_executed=%s\n' \
|
||||||
|
"$cases_defined" "$cases_executed"
|
||||||
|
|
||||||
|
if ! cmp -s "$expected_file" "$actual_file"; then
|
||||||
|
while IFS= read -r missing; do
|
||||||
|
[[ -n "$missing" ]] && printf '[fixture-suite] missing_case=%s\n' "$missing" >&2
|
||||||
|
done < <(comm -23 "$expected_file" "$actual_file")
|
||||||
|
while IFS= read -r unexpected; do
|
||||||
|
[[ -n "$unexpected" ]] && printf '[fixture-suite] unexpected_case=%s\n' "$unexpected" >&2
|
||||||
|
done < <(comm -13 "$expected_file" "$actual_file")
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage
|
||||||
|
exit 2
|
||||||
|
;;
|
||||||
|
esac
|
||||||
+126
@@ -0,0 +1,126 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
SUBJECT="$ROOT/tools/verify-greenfield-case-coverage.sh"
|
||||||
|
WORKFLOW="$ROOT/.woodpecker/greenfield-install.yml"
|
||||||
|
TMP="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$TMP"' EXIT
|
||||||
|
|
||||||
|
manifest="$TMP/expected-red.tsv"
|
||||||
|
state_root="$TMP/state"
|
||||||
|
run_id="pipeline-1"
|
||||||
|
printf '%s\n' \
|
||||||
|
'# representative manifest comment' \
|
||||||
|
$'# case\tkind\tkey/value' \
|
||||||
|
'' \
|
||||||
|
$'next-git-present\texit\t1' \
|
||||||
|
$'main-git-present\texit\t1' \
|
||||||
|
$'next-git-absent\texit\t1' > "$manifest"
|
||||||
|
|
||||||
|
bash "$SUBJECT" init "$manifest" "$state_root" "$run_id"
|
||||||
|
|
||||||
|
# Positive firing control: a skipped expected arm makes the final gate red.
|
||||||
|
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" next-git-present
|
||||||
|
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" main-git-present
|
||||||
|
set +e
|
||||||
|
skipped_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||||
|
skipped_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$skipped_status" -eq 1 ]]
|
||||||
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=2' <<<"$skipped_output"
|
||||||
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$skipped_output"
|
||||||
|
|
||||||
|
# Count inflation cannot pass: equal counts with one missing and one unexpected remain red.
|
||||||
|
: > "$state_root/$run_id/unexpected-case.ran"
|
||||||
|
set +e
|
||||||
|
inflated_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||||
|
inflated_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$inflated_status" -eq 1 ]]
|
||||||
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$inflated_output"
|
||||||
|
grep -qF '[fixture-suite] missing_case=next-git-absent' <<<"$inflated_output"
|
||||||
|
grep -qF '[fixture-suite] unexpected_case=unexpected-case' <<<"$inflated_output"
|
||||||
|
|
||||||
|
# Re-initializing the same run clears stale markers instead of certifying a later run.
|
||||||
|
bash "$SUBJECT" init "$manifest" "$state_root" "$run_id"
|
||||||
|
set +e
|
||||||
|
stale_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||||
|
stale_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$stale_status" -eq 1 ]]
|
||||||
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=0' <<<"$stale_output"
|
||||||
|
|
||||||
|
# Exact set equality is the sole green disposition.
|
||||||
|
for case_name in next-git-present main-git-present next-git-absent; do
|
||||||
|
bash "$SUBJECT" mark "$manifest" "$state_root" "$run_id" "$case_name"
|
||||||
|
done
|
||||||
|
complete_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id")"
|
||||||
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$complete_output"
|
||||||
|
|
||||||
|
# The repository's production manifest, including its comments, initializes and checks.
|
||||||
|
production_manifest="$ROOT/tools/fixtures/greenfield-expected-red.tsv"
|
||||||
|
production_state="$TMP/production-state"
|
||||||
|
bash "$SUBJECT" init "$production_manifest" "$production_state" production-1
|
||||||
|
for case_name in next-git-present main-git-present next-git-absent; do
|
||||||
|
bash "$SUBJECT" mark "$production_manifest" "$production_state" production-1 "$case_name"
|
||||||
|
done
|
||||||
|
production_output="$(bash "$SUBJECT" check "$production_manifest" "$production_state" production-1)"
|
||||||
|
grep -qF '[fixture-suite] cases_defined=3 cases_executed=3' <<<"$production_output"
|
||||||
|
|
||||||
|
# A future manifest case changes the derived set and cannot silently escape coverage.
|
||||||
|
printf '%s\n' $'future-case\texit\t1' >> "$manifest"
|
||||||
|
set +e
|
||||||
|
future_output="$(bash "$SUBJECT" check "$manifest" "$state_root" "$run_id" 2>&1)"
|
||||||
|
future_status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$future_status" -eq 1 ]]
|
||||||
|
grep -qF '[fixture-suite] cases_defined=4 cases_executed=3' <<<"$future_output"
|
||||||
|
grep -qF '[fixture-suite] missing_case=future-case' <<<"$future_output"
|
||||||
|
|
||||||
|
# The workflow writes each marker only after that case's per-case verifier succeeds,
|
||||||
|
# and the final check is eligible after either success or failure.
|
||||||
|
python3 - "$WORKFLOW" <<'PY'
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||||
|
steps = {
|
||||||
|
match.group(1): match.group(2)
|
||||||
|
for match in re.finditer(r"(?ms)^ ([A-Za-z0-9_-]+):\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text)
|
||||||
|
}
|
||||||
|
case_steps = (
|
||||||
|
("greenfield-git-present", "next-git-present"),
|
||||||
|
("greenfield-main-git-present", "main-git-present"),
|
||||||
|
("greenfield-git-absent", "next-git-absent"),
|
||||||
|
)
|
||||||
|
for step, case in case_steps:
|
||||||
|
block = steps[step]
|
||||||
|
verify = block.index("bash tools/verify-greenfield-expected-red.sh")
|
||||||
|
mark = block.index(f"mark \\\n tools/fixtures/greenfield-expected-red.tsv .greenfield-case-state \\\n \"$coverage_run\" {case}")
|
||||||
|
assert verify < mark, f"{step} marks execution before successful verification"
|
||||||
|
|
||||||
|
for step in (
|
||||||
|
"greenfield-git-present",
|
||||||
|
"greenfield-main-git-present",
|
||||||
|
"greenfield-remote-installer-contract",
|
||||||
|
"greenfield-git-absent",
|
||||||
|
):
|
||||||
|
assert re.search(
|
||||||
|
r"(?m)^ depends_on:\n - greenfield-case-denominator-init$", steps[step]
|
||||||
|
), f"{step} can race marker initialization"
|
||||||
|
|
||||||
|
final = steps["greenfield-case-denominator"]
|
||||||
|
assert "status: [success, failure]" in final
|
||||||
|
assert "bash tools/verify-greenfield-case-coverage.sh check" in final
|
||||||
|
for dependency in (
|
||||||
|
"greenfield-git-present",
|
||||||
|
"greenfield-main-git-present",
|
||||||
|
"greenfield-remote-installer-contract",
|
||||||
|
"greenfield-git-absent",
|
||||||
|
):
|
||||||
|
assert f" - {dependency}\n" in final, f"final gate can race {dependency}"
|
||||||
|
PY
|
||||||
|
|
||||||
|
printf 'greenfield case coverage tests passed\n'
|
||||||
Reference in New Issue
Block a user