From 3931b0e29eb834914f7b17e4db7e221481d436fa Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:13:02 -0500 Subject: [PATCH 01/10] test(ci): inject RM-61 postgres startup failure control --- .woodpecker/ci.yml | 3 ++ .../1000-rm-61-ci-contract-exemption.md | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 docs/scratchpads/1000-rm-61-ci-contract-exemption.md diff --git a/.woodpecker/ci.yml b/.woodpecker/ci.yml index c02f3f1c..806aa616 100644 --- a/.woodpecker/ci.yml +++ b/.woodpecker/ci.yml @@ -139,3 +139,6 @@ services: POSTGRES_USER: mosaic POSTGRES_PASSWORD: mosaic POSTGRES_DB: mosaic + # RM-61 negative control 1/2: force a real PostgreSQL startup failure. + # This commit is intentionally red and will be reverted after its single run. + POSTGRES_INITDB_ARGS: --rm61-invalid-option diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md new file mode 100644 index 00000000..4a1d08f0 --- /dev/null +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -0,0 +1,47 @@ +# RM-61 — CI contract exemption for #1000 teardown artifact + +**Tracking:** RM-61 / issue #1000 +**Branch:** `fix/rm-61-ci-contract-exemption` +**Owner:** `coder-mos1` + +## Objective + +Determine, by red-first provider controls, whether the `ci-postgres` pod-not-found teardown signature discriminates from a real PostgreSQL failure. Only if it discriminates may a named, bounded CI-contract exemption be implemented. The exemption must retire when #1000 is fixed; fixing #1000 is the closure path. + +## Pre-registered kill criterion + +If an injected real `ci-postgres` failure also yields `pods "wp-svc--ci-postgres" not found` as the service's provider-visible failure, the signature does not discriminate. Option B is unsafe; stop exemption implementation and fall to Option A (#1000). + +## Plan + +1. Capture full `-f json` records for the 11 supplied observations and state counts. +2. Run one startup-failure control using the real pgvector/PostgreSQL image with an invalid `initdb` argument. +3. Run one post-readiness crash control using real PostgreSQL, `pg_isready`, and a deliberate postmaster kill while a DB-dependent probe is active. +4. Compare the raw `ci-postgres` service record independently of failures in dependent steps. +5. Investigate runner/time/head clustering only as a hypothesis; never encode incidental correlates or retries into policy. +6. If and only if the controls discriminate, implement and test the exact exemption, document its two-way boundary, and track retirement at #1000. + +## Budget + +No explicit token cap supplied. Working estimate: 20K–30K tokens. Limit provider controls to the two pre-registered runs; no retries or re-roll policy. + +## Initial evidence + +Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplied pipelines: 11 total. Child-step counts: five pipelines with 9 children and six with 10 children. Seven contain the `ci-postgres` pod-not-found failure (#2170, #2175, #2180, #2181, #2182, #2187, #2188); four do not (#2158, #2167, #2184, #2186). Every observed workflow reports `agent_id=44`, so the available JSON does not separate clean and artifact runs by runner. This refutes runner identity as a discriminator in the sampled record. + +## Progress + +- [x] Requirements and kill criterion recorded before control implementation. +- [x] Historical full-JSON records captured. +- [ ] Startup-failure control observed terminal. +- [ ] Post-readiness crash control observed terminal. +- [ ] Discrimination verdict recorded. +- [ ] Conditional exemption implementation (only if verdict permits). + +## Tests / evidence + +Pending. + +## Risks + +The Woodpecker Kubernetes backend may garbage-collect both genuinely failed and successfully used service pods before reconciliation. If so, the provider-visible service signature cannot safely support Option B. -- 2.54.0 From 9455cd6a2650b2b7e70f746c07933d96e5cb3d20 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:20:30 -0500 Subject: [PATCH 02/10] test(ci): inject RM-61 post-readiness crash control --- .woodpecker/ci.yml | 32 +++++++++++++++---- .../1000-rm-61-ci-contract-exemption.md | 13 ++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.woodpecker/ci.yml b/.woodpecker/ci.yml index 806aa616..41fa8076 100644 --- a/.woodpecker/ci.yml +++ b/.woodpecker/ci.yml @@ -125,10 +125,19 @@ steps: echo "ci-postgres did not become ready" >&2 exit 1 fi - # Run migrations (DATABASE_URL is set in environment above) - - pnpm --filter @mosaicstack/db run db:migrate - # Run all tests - - pnpm test + # RM-61 negative control 2/2: arm a real post-readiness database crash, + # then continuously query until the service loss is observed. + - psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'CREATE TABLE rm61_crash_armed (armed boolean NOT NULL)' + - | + for i in $(seq 1 60); do + if ! psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'SELECT 1'; then + echo "RM-61 control observed the armed PostgreSQL crash" >&2 + exit 61 + fi + sleep 1 + done + echo "RM-61 control did not observe the armed PostgreSQL crash" >&2 + exit 62 depends_on: - typecheck @@ -139,6 +148,15 @@ services: POSTGRES_USER: mosaic POSTGRES_PASSWORD: mosaic POSTGRES_DB: mosaic - # RM-61 negative control 1/2: force a real PostgreSQL startup failure. - # This commit is intentionally red and will be reverted after its single run. - POSTGRES_INITDB_ARGS: --rm61-invalid-option + entrypoint: + - /bin/sh + - -c + commands: + - | + docker-entrypoint.sh postgres & + postgres_pid=$$! + until pg_isready -h 127.0.0.1 -p 5432 -U mosaic; do sleep 1; done + until [ "$$(psql -h 127.0.0.1 -U mosaic -d mosaic -Atqc "SELECT to_regclass('public.rm61_crash_armed')")" = "rm61_crash_armed" ]; do sleep 1; done + echo "RM-61 control killing ready PostgreSQL postmaster $$postgres_pid" >&2 + kill -KILL "$$postgres_pid" + wait "$$postgres_pid" diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 4a1d08f0..ce2d656b 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -33,14 +33,23 @@ Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplie - [x] Requirements and kill criterion recorded before control implementation. - [x] Historical full-JSON records captured. -- [ ] Startup-failure control observed terminal. +- [x] Startup-failure control observed terminal. - [ ] Post-readiness crash control observed terminal. - [ ] Discrimination verdict recorded. - [ ] Conditional exemption implementation (only if verdict permits). ## Tests / evidence -Pending. +### Control 1 — real startup failure + +- Commit: `3931b0e29eb834914f7b17e4db7e221481d436fa` +- Pipeline: #2189, exact commit match. +- Full JSON child scan: 9 total — 7 success, 2 failure, 0 skipped/pending/running. +- `ci-postgres`: `state=failure`, `exit_code=1`, `error=null`, with a five-second execution window. +- `test`: `state=failure`, `exit_code=1` after the readiness budget expired. +- Pipeline/workflow: terminal `failure`. + +This control is red and its service record differs from #1000 (`exit_code=0` plus pod-not-found). It proves the startup-failure direction only. It does not settle the dangerous post-readiness crash/garbage-collection path. ## Risks -- 2.54.0 From 25ac59715a94dd1b52ef42577472eb44ecc4b446 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:28:30 -0500 Subject: [PATCH 03/10] test(ci): correct RM-61 crash control entrypoint --- .woodpecker/ci.yml | 2 +- docs/scratchpads/1000-rm-61-ci-contract-exemption.md | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.woodpecker/ci.yml b/.woodpecker/ci.yml index 41fa8076..c11e9052 100644 --- a/.woodpecker/ci.yml +++ b/.woodpecker/ci.yml @@ -148,10 +148,10 @@ services: POSTGRES_USER: mosaic POSTGRES_PASSWORD: mosaic POSTGRES_DB: mosaic + PGPASSWORD: mosaic entrypoint: - /bin/sh - -c - commands: - | docker-entrypoint.sh postgres & postgres_pid=$$! diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index ce2d656b..4955df24 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -51,6 +51,15 @@ Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplie This control is red and its service record differs from #1000 (`exit_code=0` plus pod-not-found). It proves the startup-failure direction only. It does not settle the dangerous post-readiness crash/garbage-collection path. +### Control 2 setup attempt — invalid, excluded from evidence + +- Commit: `9455cd6a2650b2b7e70f746c07933d96e5cb3d20` +- Pipeline: #2190, exact commit match. +- Full JSON child scan: 9 total — 7 success, 2 failure, 0 skipped/pending/running. +- Service log: `/bin/sh: 0: -c requires an argument`. +- Root cause: Woodpecker service `commands` did not become the third `sh -c` argument. PostgreSQL never started, so this run is **not** the post-readiness crash control and provides no discrimination evidence. +- Focused remediation: place the script directly in the third `entrypoint` element and supply `PGPASSWORD` for the marker query. This is a control-fixture correction, not a retry of #1000 and not evidence for either verdict. + ## Risks The Woodpecker Kubernetes backend may garbage-collect both genuinely failed and successfully used service pods before reconciliation. If so, the provider-visible service signature cannot safely support Option B. -- 2.54.0 From ef9d23ab62b84df19d92d114cb44b005a3c2f2eb Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:35:29 -0500 Subject: [PATCH 04/10] test(ci): register RM-61 terminal-green contract cases --- .woodpecker/ci.yml | 29 +------ .../1000-rm-61-ci-contract-exemption.md | 27 ++++++- .../test-terminal-green-contract.sh | 77 +++++++++++++++++++ 3 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh diff --git a/.woodpecker/ci.yml b/.woodpecker/ci.yml index c11e9052..c02f3f1c 100644 --- a/.woodpecker/ci.yml +++ b/.woodpecker/ci.yml @@ -125,19 +125,10 @@ steps: echo "ci-postgres did not become ready" >&2 exit 1 fi - # RM-61 negative control 2/2: arm a real post-readiness database crash, - # then continuously query until the service loss is observed. - - psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'CREATE TABLE rm61_crash_armed (armed boolean NOT NULL)' - - | - for i in $(seq 1 60); do - if ! psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -c 'SELECT 1'; then - echo "RM-61 control observed the armed PostgreSQL crash" >&2 - exit 61 - fi - sleep 1 - done - echo "RM-61 control did not observe the armed PostgreSQL crash" >&2 - exit 62 + # Run migrations (DATABASE_URL is set in environment above) + - pnpm --filter @mosaicstack/db run db:migrate + # Run all tests + - pnpm test depends_on: - typecheck @@ -148,15 +139,3 @@ services: POSTGRES_USER: mosaic POSTGRES_PASSWORD: mosaic POSTGRES_DB: mosaic - PGPASSWORD: mosaic - entrypoint: - - /bin/sh - - -c - - | - docker-entrypoint.sh postgres & - postgres_pid=$$! - until pg_isready -h 127.0.0.1 -p 5432 -U mosaic; do sleep 1; done - until [ "$$(psql -h 127.0.0.1 -U mosaic -d mosaic -Atqc "SELECT to_regclass('public.rm61_crash_armed')")" = "rm61_crash_armed" ]; do sleep 1; done - echo "RM-61 control killing ready PostgreSQL postmaster $$postgres_pid" >&2 - kill -KILL "$$postgres_pid" - wait "$$postgres_pid" diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 4955df24..2818566a 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -34,9 +34,9 @@ Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplie - [x] Requirements and kill criterion recorded before control implementation. - [x] Historical full-JSON records captured. - [x] Startup-failure control observed terminal. -- [ ] Post-readiness crash control observed terminal. -- [ ] Discrimination verdict recorded. -- [ ] Conditional exemption implementation (only if verdict permits). +- [x] Post-readiness crash control observed terminal. +- [x] Discrimination verdict recorded: Option B may proceed. +- [ ] Conditional exemption implementation. ## Tests / evidence @@ -51,6 +51,27 @@ Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplie This control is red and its service record differs from #1000 (`exit_code=0` plus pod-not-found). It proves the startup-failure direction only. It does not settle the dangerous post-readiness crash/garbage-collection path. +### Control 2 — real post-readiness crash + +- Commit: `25ac59715a94dd1b52ef42577472eb44ecc4b446` +- Pipeline: #2191, exact commit match. +- Full JSON child scan: 9 total — 7 success, 2 failure, 0 skipped/pending/running. +- Service log proves PostgreSQL reached `database system is ready to accept connections`, the test created the arm table, and the service then killed postmaster PID 7. +- Test log proves a successful `SELECT 1` followed by `Connection refused`; it exited the pre-registered control code 61. +- `ci-postgres`: `state=failure`, `exit_code=137`, `error=null`, with a 203-second execution window. +- `test`: `state=failure`, `exit_code=61`. +- Pipeline/workflow: terminal `failure`. + +This is the dangerous post-readiness crash path. Its service record is not pod-not-found and therefore differs from #1000 independently of the dependent test failure. + +### Discrimination verdict + +Both real failures are provider-visible as process exits (`exit_code=1` startup; `exit_code=137` crash) with no pod-not-found error. The seven observed #1000 artifacts are provider reconciliation misses (`exit_code=0` plus the exact pod-not-found error). The declared kill criterion did not fire, so Option B may proceed with a matcher requiring the full conjunction. This evidence does **not** prove every future Kubernetes failure is distinguishable; it proves these two concrete real-failure classes remain blocking and bounds the exemption to the observed reconciliation shape. + +### Unit red-first checkpoint + +The nine-case contract harness was written before the verifier. First execution exited 1 because `verify-terminal-green.py` did not exist; no exemption implementation was live. Cases pre-register ordinary green, the exact artifact, both provider controls, near-miss signatures, an independent failure, and a skipped step. + ### Control 2 setup attempt — invalid, excluded from evidence - Commit: `9455cd6a2650b2b7e70f746c07933d96e5cb3d20` diff --git a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh new file mode 100644 index 00000000..3fcdd806 --- /dev/null +++ b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Red-first contract harness for RM-61 / #1000. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERIFIER="$SCRIPT_DIR/verify-terminal-green.py" +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +write_fixture() { + local file="$1" pipeline_status="$2" postgres_state="$3" postgres_exit="$4" postgres_error="$5" test_state="$6" + python3 - "$file" "$pipeline_status" "$postgres_state" "$postgres_exit" "$postgres_error" "$test_state" <<'PY' +import json, sys +path, pipeline_status, pg_state, pg_exit, pg_error, test_state = sys.argv[1:] +steps = [ + {"name": "clone", "type": "clone", "state": "success", "exit_code": 0, "error": None}, + {"name": "ci-postgres", "type": "service", "state": pg_state, "exit_code": int(pg_exit), "error": pg_error or None}, + {"name": "test", "type": "commands", "state": test_state, "exit_code": 0 if test_state == "success" else 1, "error": None}, +] +json.dump({ + "number": 9999, + "status": pipeline_status, + "commit": "a" * 40, + "workflows": [{"name": "ci", "state": pipeline_status, "children": steps}], +}, open(path, "w")) +PY +} + +expect_exit() { + local expected="$1" label="$2" file="$3" + shift 3 + set +e + output=$(python3 "$VERIFIER" "$file" "$@" 2>&1) + actual=$? + set -e + if [[ "$actual" -ne "$expected" ]]; then + printf 'FAIL %s: expected exit %s, got %s\n%s\n' "$label" "$expected" "$actual" "$output" >&2 + exit 1 + fi + printf 'PASS %s\n' "$label" + printf '%s' "$output" +} + +# Ordinary terminal green. +write_fixture "$TMP/green.json" success success 0 '' success +out=$(expect_exit 0 green "$TMP/green.json") +grep -q '"total_steps": 3' <<<"$out" +grep -q '"exempted_steps": 0' <<<"$out" + +# Exact, named #1000 teardown artifact: the only permitted non-success child. +artifact='pods "wp-svc-01kyxzjhdf6w81swsnbfzh85z9-ci-postgres" not found' +write_fixture "$TMP/artifact.json" success failure 0 "$artifact" success +out=$(expect_exit 0 exact-artifact "$TMP/artifact.json") +grep -q '"exemption_id": "WP-K8S-1000-CI-POSTGRES-TEARDOWN"' <<<"$out" +grep -q '"exempted_steps": 1' <<<"$out" + +# Negative controls: both real PostgreSQL failures must remain red. +write_fixture "$TMP/startup.json" failure failure 1 '' failure +expect_exit 1 startup-failure "$TMP/startup.json" >/dev/null +write_fixture "$TMP/crash.json" failure failure 137 '' failure +expect_exit 1 post-readiness-crash "$TMP/crash.json" >/dev/null + +# The exemption is signature-scoped, not step-scoped. +write_fixture "$TMP/wrong-error.json" success failure 0 'connection refused' success +expect_exit 1 other-postgres-error "$TMP/wrong-error.json" >/dev/null +write_fixture "$TMP/wrong-pod.json" success failure 0 'pods "other-ci-postgres" not found' success +expect_exit 1 wrong-pod-signature "$TMP/wrong-pod.json" >/dev/null +write_fixture "$TMP/nonzero-artifact.json" success failure 137 "$artifact" success +expect_exit 1 nonzero-with-artifact-text "$TMP/nonzero-artifact.json" >/dev/null + +# Exact artifact cannot mask any independent failure or non-success pipeline. +write_fixture "$TMP/artifact-plus-failure.json" failure failure 0 "$artifact" failure +expect_exit 1 artifact-plus-real-failure "$TMP/artifact-plus-failure.json" >/dev/null +write_fixture "$TMP/skipped.json" success success 0 '' skipped +expect_exit 1 skipped-step "$TMP/skipped.json" >/dev/null + +printf 'terminal-green contract harness: PASS (9 cases)\n' -- 2.54.0 From 1a6fc996b4a5263f36cc91fd08036486652bb504 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:38:32 -0500 Subject: [PATCH 05/10] feat(ci): bound terminal-green exemption to issue 1000 signature --- .../1000-rm-61-ci-contract-exemption.md | 24 ++- .../framework/fleet/roles/merge-gate.md | 7 +- .../framework/guides/CI-CD-PIPELINES.md | 28 +++ .../framework/tools/woodpecker/README.md | 17 +- .../tools/woodpecker/verify-terminal-green.py | 197 ++++++++++++++++++ packages/mosaic/package.json | 2 +- 6 files changed, 265 insertions(+), 10 deletions(-) create mode 100644 packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 2818566a..bef73dd3 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -36,7 +36,7 @@ Historical JSON saved locally under `.evidence/rm-61/` (not for commit). Supplie - [x] Startup-failure control observed terminal. - [x] Post-readiness crash control observed terminal. - [x] Discrimination verdict recorded: Option B may proceed. -- [ ] Conditional exemption implementation. +- [x] Conditional exemption implementation. ## Tests / evidence @@ -81,6 +81,26 @@ The nine-case contract harness was written before the verifier. First execution - Root cause: Woodpecker service `commands` did not become the third `sh -c` argument. PostgreSQL never started, so this run is **not** the post-readiness crash control and provides no discrimination evidence. - Focused remediation: place the script directly in the third `entrypoint` element and supply `PGPASSWORD` for the marker query. This is a control-fixture correction, not a retry of #1000 and not evidence for either verdict. +## Implementation evidence + +- `verify-terminal-green.py` consumes only the full JSON/API record; it performs no fetch, retry, or trigger. +- Exact #2188 record: exit 0, 10 children, 9 success + 1 named exemption. +- Historical set: #2158/#2167/#2184/#2186 pass with no exemption; #2170/#2175/#2182/#2187/#2188 pass with one named exemption; #2180/#2181 remain red because independent failures exist. +- Provider controls: #2189 and #2191 both exit 1 under the verifier; neither is exempted. +- Unit harness: 9/9 cases pass after the red-first checkpoint. +- Test-membership guard: PASS, population 45; 26 enumerated, 19 signed exclusions; all 39 surface paths present. +- Python compile: PASS. +- Changed-file Prettier check: PASS after formatting the Woodpecker README. + +## Documentation checklist + +- [x] CI contract documented in the canonical framework CI/CD guide. +- [x] Operator command documented in the Woodpecker tool README. +- [x] Merge-gate baseline points to the deterministic verifier and named retirement. +- [x] Tracking and retirement cite issue #1000. +- [x] Both positive and negative guarantee boundaries are stated. +- [x] No API/auth/schema/user-facing navigation change; OpenAPI, user guide, and sitemap are not applicable. + ## Risks -The Woodpecker Kubernetes backend may garbage-collect both genuinely failed and successfully used service pods before reconciliation. If so, the provider-visible service signature cannot safely support Option B. +The controls establish discrimination for deterministic startup failure and an armed post-readiness postmaster crash on the current Woodpecker Kubernetes provider. They cannot prove that every future Kubernetes failure mode will preserve a non-zero exit before reconciliation. The exact matcher minimizes that residual risk, and issue #1000 remains the mandatory provider-seam closure and retirement trigger. diff --git a/packages/mosaic/framework/fleet/roles/merge-gate.md b/packages/mosaic/framework/fleet/roles/merge-gate.md index 7227084a..77b0759b 100644 --- a/packages/mosaic/framework/fleet/roles/merge-gate.md +++ b/packages/mosaic/framework/fleet/roles/merge-gate.md @@ -13,7 +13,12 @@ It is a **gate** role: the one and only merge path. 2. **Use the wrapped scripts as the ONLY merge path** — the merge-gate merges **exclusively** by calling **`pr-merge.sh`** (the merge action, which carries the authoritative forbidden-path guard) and **`pr-ci-wait.sh`** (to wait for green - CI before merging). These two scripts are the _only_ sanctioned merge path. + CI before merging). Before issuing a verdict, scan the full JSON/API child-step + record (including `clone`) with **`verify-terminal-green.py`** and record its + exact step count, anomalies, and named exemptions. The verifier's sole interim + exemption is `WP-K8S-1000-CI-POSTGRES-TEARDOWN`; it is signature-scoped, tracked + by #1000, and retires when #1000 is fixed. These scripts are the _only_ + sanctioned merge path. 3. **Never call the raw API** — the merge-gate **does NOT** call `tea`, the raw Gitea/forge HTTP API, or any other merge mechanism directly. Only `pr-merge.sh` and `pr-ci-wait.sh`. diff --git a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md index 3766b14c..91802c13 100644 --- a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md +++ b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md @@ -868,6 +868,34 @@ steps: 7. **Test on a short-lived non-main branch first** — open a PR and verify quality gates before merging to `main` 8. **Verify images appear** in Gitea Packages tab after successful pipeline +## Terminal-Green Full-Step Contract + +A successful pipeline summary is not sufficient: verification MUST consume the full JSON/API child-step record, including `clone`. + +```bash +~/.config/mosaic/tools/woodpecker/pipeline-status.sh \ + -r mosaicstack/stack -n -f json \ + | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py - +``` + +The verifier reports the total step count, state counts, anomalies, and any applied exemption. Exit `0` means the record satisfies the contract; exit `1` means at least one pipeline, workflow, or child-step state blocks terminal-green; exit `2` means the JSON input could not be verified. + +### Named interim exemption: `WP-K8S-1000-CI-POSTGRES-TEARDOWN` + +Only this exact conjunction is exempted: + +- pipeline and workflow state are `success`; +- exactly one non-success child exists; +- its name is `ci-postgres` and type is `service`; +- its state is `failure`, exit code is `0`; and +- its error exactly matches `pods "wp-svc--ci-postgres" not found`. + +Every near miss remains blocking, including non-zero service exits, startup failures, post-readiness crashes, connection errors, image-pull errors, skipped steps, another failed child, malformed pod names, duplicate matches, or a non-success pipeline/workflow. + +**Boundary in both directions:** this exemption recognizes the observed Woodpecker Kubernetes reconciliation miss after an otherwise-successful run. It does not prove that every future PostgreSQL or Kubernetes failure is distinguishable. It does prove, through provider controls, that a deterministic startup failure (`exit_code=1`) and an armed post-readiness postmaster crash (`exit_code=137`, dependent probe `Connection refused`) do not match and remain red. + +**Tracking and retirement:** [mosaicstack/stack#1000](https://git.mosaicstack.dev/mosaicstack/stack/issues/1000) owns the provider-seam fix. This exemption MUST be removed when #1000 is fixed. It is not authority to retry or re-trigger a pipeline, and no per-PR re-roll is part of the contract. + ## Post-Merge CI Monitoring (Hard Rule) For source-code delivery, completion is not allowed at "PR opened" stage. diff --git a/packages/mosaic/framework/tools/woodpecker/README.md b/packages/mosaic/framework/tools/woodpecker/README.md index 7e7a614a..80a4813a 100644 --- a/packages/mosaic/framework/tools/woodpecker/README.md +++ b/packages/mosaic/framework/tools/woodpecker/README.md @@ -26,12 +26,13 @@ A Woodpecker API token is required. To configure: ## Scripts -| Script | Purpose | -| --------------------- | -------------------------------------------- | -| `pipeline-list.sh` | List recent pipelines for a repo | -| `pipeline-status.sh` | Get status of a specific or latest pipeline | -| `pipeline-trigger.sh` | Trigger a new pipeline build | -| `ci-wait.sh` | Block until pipeline(s) reach terminal state | +| Script | Purpose | +| -------------------------- | -------------------------------------------------------------- | +| `pipeline-list.sh` | List recent pipelines for a repo | +| `pipeline-status.sh` | Get status of a specific or latest pipeline | +| `pipeline-trigger.sh` | Trigger a new pipeline build | +| `ci-wait.sh` | Block until pipeline(s) reach terminal state | +| `verify-terminal-green.py` | Verify every JSON/API child step under the bounded CI contract | ## Common Options @@ -59,4 +60,8 @@ A Woodpecker API token is required. To configure: # Block until one or more pipelines finish (event-driven CI wait) ~/.config/mosaic/tools/woodpecker/ci-wait.sh -r usc/uconnect -n 3917 -n 3918 + +# Verify the full JSON child-step record; do not use the text summary for this gate +~/.config/mosaic/tools/woodpecker/pipeline-status.sh -r mosaicstack/stack -n 2188 -f json \ + | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py - ``` diff --git a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py new file mode 100644 index 00000000..f5677aff --- /dev/null +++ b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Verify Mosaic's full-step Woodpecker terminal-green contract. + +RM-61 permits one named, signature-scoped exception for issue #1000. The +exception retires when #1000 is fixed; all other non-success states block. +This program consumes the JSON/API record emitted by pipeline-status.sh -f json. +It does not fetch, retry, or re-trigger pipelines. +""" + +from __future__ import annotations + +import json +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +EXEMPTION_ID = "WP-K8S-1000-CI-POSTGRES-TEARDOWN" +EXEMPTION_ISSUE = "https://git.mosaicstack.dev/mosaicstack/stack/issues/1000" +POD_NOT_FOUND = re.compile( + r'^pods "wp-svc-[0-9a-hjkmnp-tv-z]{26}-ci-postgres" not found$' +) + + +def fail_usage(message: str) -> int: + print(f"terminal-green contract input error: {message}", file=sys.stderr) + return 2 + + +def load_record(argument: str | None) -> dict[str, Any]: + if argument in (None, "-"): + value = json.load(sys.stdin) + else: + with Path(argument).open(encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError("pipeline record must be a JSON object") + return value + + +def is_issue_1000_artifact(step: dict[str, Any]) -> bool: + error = step.get("error") + return ( + step.get("name") == "ci-postgres" + and step.get("type") == "service" + and step.get("state") == "failure" + and step.get("exit_code") == 0 + and isinstance(error, str) + and POD_NOT_FOUND.fullmatch(error) is not None + ) + + +def verify(record: dict[str, Any]) -> tuple[int, dict[str, Any]]: + anomalies: list[dict[str, Any]] = [] + candidates: list[dict[str, Any]] = [] + steps: list[dict[str, Any]] = [] + + pipeline_status = record.get("status") + if pipeline_status != "success": + anomalies.append( + { + "scope": "pipeline", + "name": str(record.get("number", "unknown")), + "state": pipeline_status, + "reason": "pipeline status is not success", + } + ) + + workflows = record.get("workflows") + if not isinstance(workflows, list) or not workflows: + anomalies.append( + { + "scope": "pipeline", + "name": str(record.get("number", "unknown")), + "state": pipeline_status, + "reason": "workflows are missing or empty", + } + ) + workflows = [] + + for workflow_index, workflow in enumerate(workflows): + if not isinstance(workflow, dict): + anomalies.append( + { + "scope": "workflow", + "name": str(workflow_index), + "state": None, + "reason": "workflow is not an object", + } + ) + continue + workflow_name = str(workflow.get("name", workflow_index)) + if workflow.get("state") != "success": + anomalies.append( + { + "scope": "workflow", + "name": workflow_name, + "state": workflow.get("state"), + "reason": "workflow state is not success", + } + ) + children = workflow.get("children") + if not isinstance(children, list) or not children: + anomalies.append( + { + "scope": "workflow", + "name": workflow_name, + "state": workflow.get("state"), + "reason": "child-step list is missing or empty", + } + ) + continue + for child_index, child in enumerate(children): + if not isinstance(child, dict): + anomalies.append( + { + "scope": "step", + "name": f"{workflow_name}[{child_index}]", + "state": None, + "reason": "step is not an object", + } + ) + continue + steps.append(child) + if child.get("state") == "success": + continue + if is_issue_1000_artifact(child): + candidates.append(child) + continue + anomalies.append( + { + "scope": "step", + "name": child.get("name"), + "type": child.get("type"), + "state": child.get("state"), + "exit_code": child.get("exit_code"), + "error": child.get("error"), + "reason": "non-success step does not match the #1000 teardown signature", + } + ) + + if len(candidates) > 1: + anomalies.append( + { + "scope": "exemption", + "name": EXEMPTION_ID, + "state": "invalid", + "reason": "the #1000 exemption may apply to exactly one step", + } + ) + + exemption_applies = len(candidates) == 1 and not anomalies + state_counts = Counter(str(step.get("state", "missing")) for step in steps) + result: dict[str, Any] = { + "schema_version": "mosaic-terminal-green/v1", + "verdict": "terminal-green" if not anomalies else "not-terminal-green", + "pipeline_number": record.get("number"), + "commit": record.get("commit"), + "pipeline_status": pipeline_status, + "total_steps": len(steps), + "state_counts": dict(sorted(state_counts.items())), + "exempted_steps": 1 if exemption_applies else 0, + "anomalies": anomalies, + } + if exemption_applies: + candidate = candidates[0] + result["exemptions"] = [ + { + "exemption_id": EXEMPTION_ID, + "step": candidate.get("name"), + "signature": candidate.get("error"), + "tracking_issue": EXEMPTION_ISSUE, + "retires_when": "issue #1000 is fixed", + } + ] + else: + result["exemptions"] = [] + + return (0 if not anomalies else 1), result + + +def main() -> int: + if len(sys.argv) > 2 or (len(sys.argv) == 2 and sys.argv[1] in {"-h", "--help"}): + print(f"usage: {Path(sys.argv[0]).name} [pipeline.json|-]", file=sys.stderr) + return 0 if len(sys.argv) == 2 else 2 + try: + record = load_record(sys.argv[1] if len(sys.argv) == 2 else None) + except (OSError, ValueError, json.JSONDecodeError) as error: + return fail_usage(str(error)) + code, result = verify(record) + print(json.dumps(result, indent=2, sort_keys=True)) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/mosaic/package.json b/packages/mosaic/package.json index 56898aa8..401d0015 100644 --- a/packages/mosaic/package.json +++ b/packages/mosaic/package.json @@ -25,7 +25,7 @@ "lint": "eslint src", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests && pnpm run test:framework-shell", - "test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh" + "test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/woodpecker/test-terminal-green-contract.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh" }, "dependencies": { "@mosaicstack/brain": "workspace:*", -- 2.54.0 From 878bf3d58923b7c55aa50a4523fa9db43cbc3f35 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:40:55 -0500 Subject: [PATCH 06/10] docs(ci): record RM-61 control and contract evidence --- docs/scratchpads/1000-rm-61-ci-contract-exemption.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index bef73dd3..0e0a96d7 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -1,7 +1,8 @@ # RM-61 — CI contract exemption for #1000 teardown artifact -**Tracking:** RM-61 / issue #1000 -**Branch:** `fix/rm-61-ci-contract-exemption` +**Tracking:** RM-61 / issue #1000 + +**Branch:** `fix/rm-61-ci-contract-exemption` **Owner:** `coder-mos1` ## Objective @@ -90,7 +91,10 @@ The nine-case contract harness was written before the verifier. First execution - Unit harness: 9/9 cases pass after the red-first checkpoint. - Test-membership guard: PASS, population 45; 26 enumerated, 19 signed exclusions; all 39 surface paths present. - Python compile: PASS. -- Changed-file Prettier check: PASS after formatting the Woodpecker README. +- `pnpm typecheck`: PASS, 45/45 tasks. +- `pnpm lint`: PASS, 25/25 tasks. +- `pnpm format:check`: PASS after moving local evidence outside the repository tree. +- `test:framework-shell`: RM-61 and all preceding suites passed, then the pre-existing wake assertion aborted with exit 97 because this host's Bash 5.2.15 reports `BASH_LINENO [3 5]` where that suite requires `[3 4]`. RM-61 does not modify the wake suite; the command is not fully runnable on this host as written and no substitute result is claimed. ## Documentation checklist -- 2.54.0 From e7b29219e11efd0a19395156ac0b154bec0c3a73 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:41:27 -0500 Subject: [PATCH 07/10] fix(ci): install terminal-green verifier as executable --- .../framework/tools/woodpecker/test-terminal-green-contract.sh | 0 .../mosaic/framework/tools/woodpecker/verify-terminal-green.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh mode change 100644 => 100755 packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py diff --git a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh old mode 100644 new mode 100755 diff --git a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py old mode 100644 new mode 100755 -- 2.54.0 From 033b2ffb46674b2c0bcc5197273c109b461f62d9 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 08:53:36 -0500 Subject: [PATCH 08/10] fix(ci): bind terminal-green evidence to PR head --- .../1000-rm-61-ci-contract-exemption.md | 9 +++- .../framework/fleet/roles/merge-gate.md | 6 ++- .../framework/guides/CI-CD-PIPELINES.md | 8 +++- .../framework/tools/woodpecker/README.md | 3 +- .../test-terminal-green-contract.sh | 31 ++++++++++--- .../tools/woodpecker/verify-terminal-green.py | 44 ++++++++++++++++--- 6 files changed, 82 insertions(+), 19 deletions(-) diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 0e0a96d7..4825f082 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -88,7 +88,7 @@ The nine-case contract harness was written before the verifier. First execution - Exact #2188 record: exit 0, 10 children, 9 success + 1 named exemption. - Historical set: #2158/#2167/#2184/#2186 pass with no exemption; #2170/#2175/#2182/#2187/#2188 pass with one named exemption; #2180/#2181 remain red because independent failures exist. - Provider controls: #2189 and #2191 both exit 1 under the verifier; neither is exempted. -- Unit harness: 9/9 cases pass after the red-first checkpoint. +- Unit harness: initial 9/9 cases passed after the red-first checkpoint; review remediation expands this to 12 cases with expected-head match/missing/mismatch coverage. - Test-membership guard: PASS, population 45; 26 enumerated, 19 signed exclusions; all 39 surface paths present. - Python compile: PASS. - `pnpm typecheck`: PASS, 45/45 tasks. @@ -96,6 +96,13 @@ The nine-case contract harness was written before the verifier. First execution - `pnpm format:check`: PASS after moving local evidence outside the repository tree. - `test:framework-shell`: RM-61 and all preceding suites passed, then the pre-existing wake assertion aborted with exit 97 because this host's Bash 5.2.15 reports `BASH_LINENO [3 5]` where that suite requires `[3 4]`. RM-61 does not modify the wake suite; the command is not fully runnable on this host as written and no substitute result is claimed. +## Independent review + +- Review 67 / comment 20403 at exact head `e7b29219e11efd0a19395156ac0b154bec0c3a73`: **REQUEST CHANGES**. +- Blocker: the verifier echoed the pipeline commit but did not bind it to the current PR head; mutating only #2188's commit still returned terminal-green. +- Remediation: require `--expect-commit `, add a pipeline anomaly on missing/mismatched record commits, emit expected and observed values, wire both CI documentation and the merge-gate baseline to pass provider PR head, and add match/missing/mismatch tests. +- This binding is not prohibited head-based clustering policy: it proves the evidence belongs to the commit under verdict. Runner/node/time/head correlation remains excluded from the teardown signature itself. + ## Documentation checklist - [x] CI contract documented in the canonical framework CI/CD guide. diff --git a/packages/mosaic/framework/fleet/roles/merge-gate.md b/packages/mosaic/framework/fleet/roles/merge-gate.md index 77b0759b..3f6f67b9 100644 --- a/packages/mosaic/framework/fleet/roles/merge-gate.md +++ b/packages/mosaic/framework/fleet/roles/merge-gate.md @@ -14,8 +14,10 @@ It is a **gate** role: the one and only merge path. **exclusively** by calling **`pr-merge.sh`** (the merge action, which carries the authoritative forbidden-path guard) and **`pr-ci-wait.sh`** (to wait for green CI before merging). Before issuing a verdict, scan the full JSON/API child-step - record (including `clone`) with **`verify-terminal-green.py`** and record its - exact step count, anomalies, and named exemptions. The verifier's sole interim + record (including `clone`) with **`verify-terminal-green.py --expect-commit +`** and record the equal expected/observed full-40 + commits, exact step count, anomalies, and named exemptions. Missing or mismatched + commit binding is a hard refusal. The verifier's sole interim exemption is `WP-K8S-1000-CI-POSTGRES-TEARDOWN`; it is signature-scoped, tracked by #1000, and retires when #1000 is fixed. These scripts are the _only_ sanctioned merge path. diff --git a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md index 91802c13..035954c9 100644 --- a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md +++ b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md @@ -873,12 +873,16 @@ steps: A successful pipeline summary is not sufficient: verification MUST consume the full JSON/API child-step record, including `clone`. ```bash +PR_HEAD= ~/.config/mosaic/tools/woodpecker/pipeline-status.sh \ -r mosaicstack/stack -n -f json \ - | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py - + | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py \ + --expect-commit "$PR_HEAD" - ``` -The verifier reports the total step count, state counts, anomalies, and any applied exemption. Exit `0` means the record satisfies the contract; exit `1` means at least one pipeline, workflow, or child-step state blocks terminal-green; exit `2` means the JSON input could not be verified. +`PR_HEAD` MUST come from the current provider PR metadata and MUST be the full 40-hex head, not a local branch guess. The verifier fails if the argument is missing, malformed, absent from the pipeline record, or differs from that record. + +The verifier reports the expected and observed commits, total step count, state counts, anomalies, and any applied exemption. Exit `0` means the record satisfies the contract; exit `1` means the commit binding or at least one pipeline, workflow, or child-step state blocks terminal-green; exit `2` means the invocation or JSON input could not be verified. ### Named interim exemption: `WP-K8S-1000-CI-POSTGRES-TEARDOWN` diff --git a/packages/mosaic/framework/tools/woodpecker/README.md b/packages/mosaic/framework/tools/woodpecker/README.md index 80a4813a..ea163121 100644 --- a/packages/mosaic/framework/tools/woodpecker/README.md +++ b/packages/mosaic/framework/tools/woodpecker/README.md @@ -62,6 +62,7 @@ A Woodpecker API token is required. To configure: ~/.config/mosaic/tools/woodpecker/ci-wait.sh -r usc/uconnect -n 3917 -n 3918 # Verify the full JSON child-step record; do not use the text summary for this gate +PR_HEAD= ~/.config/mosaic/tools/woodpecker/pipeline-status.sh -r mosaicstack/stack -n 2188 -f json \ - | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py - + | ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py --expect-commit "$PR_HEAD" - ``` diff --git a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh index 3fcdd806..81e96880 100755 --- a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh +++ b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" VERIFIER="$SCRIPT_DIR/verify-terminal-green.py" +EXPECTED_COMMIT=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT @@ -27,14 +28,13 @@ PY } expect_exit() { - local expected="$1" label="$2" file="$3" - shift 3 + local expected_exit="$1" label="$2" file="$3" expected_commit="${4:-$EXPECTED_COMMIT}" set +e - output=$(python3 "$VERIFIER" "$file" "$@" 2>&1) + output=$(python3 "$VERIFIER" --expect-commit "$expected_commit" "$file" 2>&1) actual=$? set -e - if [[ "$actual" -ne "$expected" ]]; then - printf 'FAIL %s: expected exit %s, got %s\n%s\n' "$label" "$expected" "$actual" "$output" >&2 + if [[ "$actual" -ne "$expected_exit" ]]; then + printf 'FAIL %s: expected exit %s, got %s\n%s\n' "$label" "$expected_exit" "$actual" "$output" >&2 exit 1 fi printf 'PASS %s\n' "$label" @@ -74,4 +74,23 @@ expect_exit 1 artifact-plus-real-failure "$TMP/artifact-plus-failure.json" >/dev write_fixture "$TMP/skipped.json" success success 0 '' skipped expect_exit 1 skipped-step "$TMP/skipped.json" >/dev/null -printf 'terminal-green contract harness: PASS (9 cases)\n' +# The scanned pipeline must be bound to an explicit, full PR-head commit. +set +e +missing_output=$(python3 "$VERIFIER" "$TMP/artifact.json" 2>&1) +missing_rc=$? +set -e +if [[ "$missing_rc" -ne 2 ]] || ! grep -q -- '--expect-commit' <<<"$missing_output"; then + printf 'FAIL missing-expected-commit: expected usage exit 2\n%s\n' "$missing_output" >&2 + exit 1 +fi +expect_exit 1 mismatched-expected-commit "$TMP/artifact.json" bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb >/dev/null + +python3 - "$TMP/artifact.json" "$TMP/missing-record-commit.json" <<'PY' +import json, sys +record = json.load(open(sys.argv[1])) +record.pop("commit") +json.dump(record, open(sys.argv[2], "w")) +PY +expect_exit 1 missing-record-commit "$TMP/missing-record-commit.json" >/dev/null + +printf 'terminal-green contract harness: PASS (12 cases)\n' diff --git a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py index f5677aff..a16b133c 100755 --- a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py +++ b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py @@ -9,6 +9,7 @@ It does not fetch, retry, or re-trigger pipelines. from __future__ import annotations +import argparse import json import re import sys @@ -51,12 +52,24 @@ def is_issue_1000_artifact(step: dict[str, Any]) -> bool: ) -def verify(record: dict[str, Any]) -> tuple[int, dict[str, Any]]: +def verify(record: dict[str, Any], expected_commit: str) -> tuple[int, dict[str, Any]]: anomalies: list[dict[str, Any]] = [] candidates: list[dict[str, Any]] = [] steps: list[dict[str, Any]] = [] pipeline_status = record.get("status") + actual_commit = record.get("commit") + if actual_commit != expected_commit: + anomalies.append( + { + "scope": "pipeline", + "name": str(record.get("number", "unknown")), + "state": pipeline_status, + "reason": "pipeline commit does not equal the expected PR head", + "expected_commit": expected_commit, + "actual_commit": actual_commit, + } + ) if pipeline_status != "success": anomalies.append( { @@ -156,7 +169,8 @@ def verify(record: dict[str, Any]) -> tuple[int, dict[str, Any]]: "schema_version": "mosaic-terminal-green/v1", "verdict": "terminal-green" if not anomalies else "not-terminal-green", "pipeline_number": record.get("number"), - "commit": record.get("commit"), + "commit": actual_commit, + "expected_commit": expected_commit, "pipeline_status": pipeline_status, "total_steps": len(steps), "state_counts": dict(sorted(state_counts.items())), @@ -180,15 +194,31 @@ def verify(record: dict[str, Any]) -> tuple[int, dict[str, Any]]: return (0 if not anomalies else 1), result +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="verify the full Woodpecker terminal-green child-step contract" + ) + parser.add_argument( + "--expect-commit", + required=True, + metavar="FULL_SHA", + help="full 40-hex PR-head commit that the pipeline record must match", + ) + parser.add_argument("record", nargs="?", default="-", help="pipeline JSON file or -") + arguments = parser.parse_args() + if re.fullmatch(r"[0-9a-fA-F]{40}", arguments.expect_commit) is None: + parser.error("--expect-commit must be a full 40-hex commit") + arguments.expect_commit = arguments.expect_commit.lower() + return arguments + + def main() -> int: - if len(sys.argv) > 2 or (len(sys.argv) == 2 and sys.argv[1] in {"-h", "--help"}): - print(f"usage: {Path(sys.argv[0]).name} [pipeline.json|-]", file=sys.stderr) - return 0 if len(sys.argv) == 2 else 2 + arguments = parse_arguments() try: - record = load_record(sys.argv[1] if len(sys.argv) == 2 else None) + record = load_record(arguments.record) except (OSError, ValueError, json.JSONDecodeError) as error: return fail_usage(str(error)) - code, result = verify(record) + code, result = verify(record, arguments.expect_commit) print(json.dumps(result, indent=2, sort_keys=True)) return code -- 2.54.0 From 6e7a336debb049c78ba4b22774e10f407886b4c0 Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 09:18:21 -0500 Subject: [PATCH 09/10] fix(ci): reject non-integer teardown exit codes --- .../1000-rm-61-ci-contract-exemption.md | 1 + .../mosaic/framework/guides/CI-CD-PIPELINES.md | 2 +- .../woodpecker/test-terminal-green-contract.sh | 15 ++++++++++++++- .../tools/woodpecker/verify-terminal-green.py | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 4825f082..0f5e1bbf 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -102,6 +102,7 @@ The nine-case contract harness was written before the verifier. First execution - Blocker: the verifier echoed the pipeline commit but did not bind it to the current PR head; mutating only #2188's commit still returned terminal-green. - Remediation: require `--expect-commit `, add a pipeline anomaly on missing/mismatched record commits, emit expected and observed values, wire both CI documentation and the merge-gate baseline to pass provider PR head, and add match/missing/mismatch tests. - This binding is not prohibited head-based clustering policy: it proves the evidence belongs to the commit under verdict. Runner/node/time/head correlation remains excluded from the teardown signature itself. +- Review 69 later approved the commit-binding remediation at exact head `033b2ffb46674b2c0bcc5197273c109b461f62d9`; pipeline #2193 was 9/9 success. Before merge-gate, an independent adjudicator found that Python treats JSON `false == 0`, allowing a non-integer exit value to match. The prior gate-ready state was withdrawn. A four-case red-first control (`false`, `0.0`, `"0"`, `null`) reproduced the over-match before implementation; remediation requires the decoded type to be exactly `int`. ## Documentation checklist diff --git a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md index 035954c9..cd84c947 100644 --- a/packages/mosaic/framework/guides/CI-CD-PIPELINES.md +++ b/packages/mosaic/framework/guides/CI-CD-PIPELINES.md @@ -891,7 +891,7 @@ Only this exact conjunction is exempted: - pipeline and workflow state are `success`; - exactly one non-success child exists; - its name is `ci-postgres` and type is `service`; -- its state is `failure`, exit code is `0`; and +- its state is `failure`, exit code is the JSON integer `0` (not boolean, float, string, or null); and - its error exactly matches `pods "wp-svc--ci-postgres" not found`. Every near miss remains blocking, including non-zero service exits, startup failures, post-readiness crashes, connection errors, image-pull errors, skipped steps, another failed child, malformed pod names, duplicate matches, or a non-success pipeline/workflow. diff --git a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh index 81e96880..7bd65783 100755 --- a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh +++ b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh @@ -68,6 +68,19 @@ expect_exit 1 wrong-pod-signature "$TMP/wrong-pod.json" >/dev/null write_fixture "$TMP/nonzero-artifact.json" success failure 137 "$artifact" success expect_exit 1 nonzero-with-artifact-text "$TMP/nonzero-artifact.json" >/dev/null +# JSON booleans and non-integer zero look equal to 0 in Python but are not exit codes. +python3 - "$TMP/artifact.json" "$TMP" <<'PY' +import json, os, sys +record = json.load(open(sys.argv[1])) +for label, value in (("false", False), ("float", 0.0), ("string", "0"), ("null", None)): + changed = json.loads(json.dumps(record)) + changed["workflows"][0]["children"][1]["exit_code"] = value + json.dump(changed, open(os.path.join(sys.argv[2], f"exit-{label}.json"), "w")) +PY +for label in false float string null; do + expect_exit 1 "non-integer-exit-$label" "$TMP/exit-$label.json" >/dev/null +done + # Exact artifact cannot mask any independent failure or non-success pipeline. write_fixture "$TMP/artifact-plus-failure.json" failure failure 0 "$artifact" failure expect_exit 1 artifact-plus-real-failure "$TMP/artifact-plus-failure.json" >/dev/null @@ -93,4 +106,4 @@ json.dump(record, open(sys.argv[2], "w")) PY expect_exit 1 missing-record-commit "$TMP/missing-record-commit.json" >/dev/null -printf 'terminal-green contract harness: PASS (12 cases)\n' +printf 'terminal-green contract harness: PASS (16 cases)\n' diff --git a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py index a16b133c..662d4c38 100755 --- a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py +++ b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py @@ -46,6 +46,7 @@ def is_issue_1000_artifact(step: dict[str, Any]) -> bool: step.get("name") == "ci-postgres" and step.get("type") == "service" and step.get("state") == "failure" + and type(step.get("exit_code")) is int and step.get("exit_code") == 0 and isinstance(error, str) and POD_NOT_FOUND.fullmatch(error) is not None -- 2.54.0 From 57cae04f2b5d7072840ce515de2fbdb672bde82f Mon Sep 17 00:00:00 2001 From: coder-mos1 Date: Sat, 1 Aug 2026 09:25:41 -0500 Subject: [PATCH 10/10] test(ci): explicitly reject boolean exit codes --- docs/scratchpads/1000-rm-61-ci-contract-exemption.md | 2 +- .../tools/woodpecker/test-terminal-green-contract.sh | 6 +++--- .../framework/tools/woodpecker/verify-terminal-green.py | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md index 0f5e1bbf..16e1a7ab 100644 --- a/docs/scratchpads/1000-rm-61-ci-contract-exemption.md +++ b/docs/scratchpads/1000-rm-61-ci-contract-exemption.md @@ -102,7 +102,7 @@ The nine-case contract harness was written before the verifier. First execution - Blocker: the verifier echoed the pipeline commit but did not bind it to the current PR head; mutating only #2188's commit still returned terminal-green. - Remediation: require `--expect-commit `, add a pipeline anomaly on missing/mismatched record commits, emit expected and observed values, wire both CI documentation and the merge-gate baseline to pass provider PR head, and add match/missing/mismatch tests. - This binding is not prohibited head-based clustering policy: it proves the evidence belongs to the commit under verdict. Runner/node/time/head correlation remains excluded from the teardown signature itself. -- Review 69 later approved the commit-binding remediation at exact head `033b2ffb46674b2c0bcc5197273c109b461f62d9`; pipeline #2193 was 9/9 success. Before merge-gate, an independent adjudicator found that Python treats JSON `false == 0`, allowing a non-integer exit value to match. The prior gate-ready state was withdrawn. A four-case red-first control (`false`, `0.0`, `"0"`, `null`) reproduced the over-match before implementation; remediation requires the decoded type to be exactly `int`. +- Review 69 later approved the commit-binding remediation at exact head `033b2ffb46674b2c0bcc5197273c109b461f62d9`; pipeline #2193 was 9/9 success. Before merge-gate, an independent adjudicator found that Python treats JSON `false == 0`, allowing a non-integer exit value to match. The prior gate-ready state was withdrawn. The type-strict set distinguishes genuine red-first controls (`false`, `0.0`, which wrongly exempted) from regression guards (`true`, `"0"`, `null`, which already blocked). Remediation requires the decoded type to be exactly `int` and excludes `bool` explicitly. ## Documentation checklist diff --git a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh index 7bd65783..86e04aa8 100755 --- a/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh +++ b/packages/mosaic/framework/tools/woodpecker/test-terminal-green-contract.sh @@ -72,12 +72,12 @@ expect_exit 1 nonzero-with-artifact-text "$TMP/nonzero-artifact.json" >/dev/null python3 - "$TMP/artifact.json" "$TMP" <<'PY' import json, os, sys record = json.load(open(sys.argv[1])) -for label, value in (("false", False), ("float", 0.0), ("string", "0"), ("null", None)): +for label, value in (("false", False), ("true", True), ("float", 0.0), ("string", "0"), ("null", None)): changed = json.loads(json.dumps(record)) changed["workflows"][0]["children"][1]["exit_code"] = value json.dump(changed, open(os.path.join(sys.argv[2], f"exit-{label}.json"), "w")) PY -for label in false float string null; do +for label in false true float string null; do expect_exit 1 "non-integer-exit-$label" "$TMP/exit-$label.json" >/dev/null done @@ -106,4 +106,4 @@ json.dump(record, open(sys.argv[2], "w")) PY expect_exit 1 missing-record-commit "$TMP/missing-record-commit.json" >/dev/null -printf 'terminal-green contract harness: PASS (16 cases)\n' +printf 'terminal-green contract harness: PASS (17 cases)\n' diff --git a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py index 662d4c38..18b45104 100755 --- a/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py +++ b/packages/mosaic/framework/tools/woodpecker/verify-terminal-green.py @@ -42,12 +42,14 @@ def load_record(argument: str | None) -> dict[str, Any]: def is_issue_1000_artifact(step: dict[str, Any]) -> bool: error = step.get("error") + exit_code = step.get("exit_code") return ( step.get("name") == "ci-postgres" and step.get("type") == "service" and step.get("state") == "failure" - and type(step.get("exit_code")) is int - and step.get("exit_code") == 0 + and type(exit_code) is int + and not isinstance(exit_code, bool) + and exit_code == 0 and isinstance(error, str) and POD_NOT_FOUND.fullmatch(error) is not None ) -- 2.54.0