RM-61: prove ci-postgres teardown discrimination (#1033)
Co-authored-by: coder-mos1 <[email protected]>
This commit was merged in pull request #1033.
This commit is contained in:
@@ -26,12 +26,13 @@ A Woodpecker API token is required. To configure:
|
||||
|
||||
## Scripts
|
||||
|
||||
| Script | Purpose |
|
||||
| --------------------- | -------------------------------------------- |
|
||||
| `pipeline-list.sh` | List recent pipelines for a repo |
|
||||
| `pipeline-status.sh` | Get status of a specific or latest pipeline |
|
||||
| `pipeline-trigger.sh` | Trigger a new pipeline build |
|
||||
| `ci-wait.sh` | Block until pipeline(s) reach terminal state |
|
||||
| Script | Purpose |
|
||||
| -------------------------- | -------------------------------------------------------------- |
|
||||
| `pipeline-list.sh` | List recent pipelines for a repo |
|
||||
| `pipeline-status.sh` | Get status of a specific or latest pipeline |
|
||||
| `pipeline-trigger.sh` | Trigger a new pipeline build |
|
||||
| `ci-wait.sh` | Block until pipeline(s) reach terminal state |
|
||||
| `verify-terminal-green.py` | Verify every JSON/API child step under the bounded CI contract |
|
||||
|
||||
## Common Options
|
||||
|
||||
@@ -59,4 +60,9 @@ A Woodpecker API token is required. To configure:
|
||||
|
||||
# Block until one or more pipelines finish (event-driven CI wait)
|
||||
~/.config/mosaic/tools/woodpecker/ci-wait.sh -r usc/uconnect -n 3917 -n 3918
|
||||
|
||||
# Verify the full JSON child-step record; do not use the text summary for this gate
|
||||
PR_HEAD=<full-40-hex-provider-head>
|
||||
~/.config/mosaic/tools/woodpecker/pipeline-status.sh -r mosaicstack/stack -n 2188 -f json \
|
||||
| ~/.config/mosaic/tools/woodpecker/verify-terminal-green.py --expect-commit "$PR_HEAD" -
|
||||
```
|
||||
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env bash
|
||||
# Red-first contract harness for RM-61 / #1000.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VERIFIER="$SCRIPT_DIR/verify-terminal-green.py"
|
||||
EXPECTED_COMMIT=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
write_fixture() {
|
||||
local file="$1" pipeline_status="$2" postgres_state="$3" postgres_exit="$4" postgres_error="$5" test_state="$6"
|
||||
python3 - "$file" "$pipeline_status" "$postgres_state" "$postgres_exit" "$postgres_error" "$test_state" <<'PY'
|
||||
import json, sys
|
||||
path, pipeline_status, pg_state, pg_exit, pg_error, test_state = sys.argv[1:]
|
||||
steps = [
|
||||
{"name": "clone", "type": "clone", "state": "success", "exit_code": 0, "error": None},
|
||||
{"name": "ci-postgres", "type": "service", "state": pg_state, "exit_code": int(pg_exit), "error": pg_error or None},
|
||||
{"name": "test", "type": "commands", "state": test_state, "exit_code": 0 if test_state == "success" else 1, "error": None},
|
||||
]
|
||||
json.dump({
|
||||
"number": 9999,
|
||||
"status": pipeline_status,
|
||||
"commit": "a" * 40,
|
||||
"workflows": [{"name": "ci", "state": pipeline_status, "children": steps}],
|
||||
}, open(path, "w"))
|
||||
PY
|
||||
}
|
||||
|
||||
expect_exit() {
|
||||
local expected_exit="$1" label="$2" file="$3" expected_commit="${4:-$EXPECTED_COMMIT}"
|
||||
set +e
|
||||
output=$(python3 "$VERIFIER" --expect-commit "$expected_commit" "$file" 2>&1)
|
||||
actual=$?
|
||||
set -e
|
||||
if [[ "$actual" -ne "$expected_exit" ]]; then
|
||||
printf 'FAIL %s: expected exit %s, got %s\n%s\n' "$label" "$expected_exit" "$actual" "$output" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'PASS %s\n' "$label"
|
||||
printf '%s' "$output"
|
||||
}
|
||||
|
||||
# Ordinary terminal green.
|
||||
write_fixture "$TMP/green.json" success success 0 '' success
|
||||
out=$(expect_exit 0 green "$TMP/green.json")
|
||||
grep -q '"total_steps": 3' <<<"$out"
|
||||
grep -q '"exempted_steps": 0' <<<"$out"
|
||||
|
||||
# Exact, named #1000 teardown artifact: the only permitted non-success child.
|
||||
artifact='pods "wp-svc-01kyxzjhdf6w81swsnbfzh85z9-ci-postgres" not found'
|
||||
write_fixture "$TMP/artifact.json" success failure 0 "$artifact" success
|
||||
out=$(expect_exit 0 exact-artifact "$TMP/artifact.json")
|
||||
grep -q '"exemption_id": "WP-K8S-1000-CI-POSTGRES-TEARDOWN"' <<<"$out"
|
||||
grep -q '"exempted_steps": 1' <<<"$out"
|
||||
|
||||
# Negative controls: both real PostgreSQL failures must remain red.
|
||||
write_fixture "$TMP/startup.json" failure failure 1 '' failure
|
||||
expect_exit 1 startup-failure "$TMP/startup.json" >/dev/null
|
||||
write_fixture "$TMP/crash.json" failure failure 137 '' failure
|
||||
expect_exit 1 post-readiness-crash "$TMP/crash.json" >/dev/null
|
||||
|
||||
# The exemption is signature-scoped, not step-scoped.
|
||||
write_fixture "$TMP/wrong-error.json" success failure 0 'connection refused' success
|
||||
expect_exit 1 other-postgres-error "$TMP/wrong-error.json" >/dev/null
|
||||
write_fixture "$TMP/wrong-pod.json" success failure 0 'pods "other-ci-postgres" not found' success
|
||||
expect_exit 1 wrong-pod-signature "$TMP/wrong-pod.json" >/dev/null
|
||||
write_fixture "$TMP/nonzero-artifact.json" success failure 137 "$artifact" success
|
||||
expect_exit 1 nonzero-with-artifact-text "$TMP/nonzero-artifact.json" >/dev/null
|
||||
|
||||
# JSON booleans and non-integer zero look equal to 0 in Python but are not exit codes.
|
||||
python3 - "$TMP/artifact.json" "$TMP" <<'PY'
|
||||
import json, os, sys
|
||||
record = json.load(open(sys.argv[1]))
|
||||
for label, value in (("false", False), ("true", True), ("float", 0.0), ("string", "0"), ("null", None)):
|
||||
changed = json.loads(json.dumps(record))
|
||||
changed["workflows"][0]["children"][1]["exit_code"] = value
|
||||
json.dump(changed, open(os.path.join(sys.argv[2], f"exit-{label}.json"), "w"))
|
||||
PY
|
||||
for label in false true float string null; do
|
||||
expect_exit 1 "non-integer-exit-$label" "$TMP/exit-$label.json" >/dev/null
|
||||
done
|
||||
|
||||
# Exact artifact cannot mask any independent failure or non-success pipeline.
|
||||
write_fixture "$TMP/artifact-plus-failure.json" failure failure 0 "$artifact" failure
|
||||
expect_exit 1 artifact-plus-real-failure "$TMP/artifact-plus-failure.json" >/dev/null
|
||||
write_fixture "$TMP/skipped.json" success success 0 '' skipped
|
||||
expect_exit 1 skipped-step "$TMP/skipped.json" >/dev/null
|
||||
|
||||
# The scanned pipeline must be bound to an explicit, full PR-head commit.
|
||||
set +e
|
||||
missing_output=$(python3 "$VERIFIER" "$TMP/artifact.json" 2>&1)
|
||||
missing_rc=$?
|
||||
set -e
|
||||
if [[ "$missing_rc" -ne 2 ]] || ! grep -q -- '--expect-commit' <<<"$missing_output"; then
|
||||
printf 'FAIL missing-expected-commit: expected usage exit 2\n%s\n' "$missing_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
expect_exit 1 mismatched-expected-commit "$TMP/artifact.json" bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb >/dev/null
|
||||
|
||||
python3 - "$TMP/artifact.json" "$TMP/missing-record-commit.json" <<'PY'
|
||||
import json, sys
|
||||
record = json.load(open(sys.argv[1]))
|
||||
record.pop("commit")
|
||||
json.dump(record, open(sys.argv[2], "w"))
|
||||
PY
|
||||
expect_exit 1 missing-record-commit "$TMP/missing-record-commit.json" >/dev/null
|
||||
|
||||
printf 'terminal-green contract harness: PASS (17 cases)\n'
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify Mosaic's full-step Woodpecker terminal-green contract.
|
||||
|
||||
RM-61 permits one named, signature-scoped exception for issue #1000. The
|
||||
exception retires when #1000 is fixed; all other non-success states block.
|
||||
This program consumes the JSON/API record emitted by pipeline-status.sh -f json.
|
||||
It does not fetch, retry, or re-trigger pipelines.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
EXEMPTION_ID = "WP-K8S-1000-CI-POSTGRES-TEARDOWN"
|
||||
EXEMPTION_ISSUE = "https://git.mosaicstack.dev/mosaicstack/stack/issues/1000"
|
||||
POD_NOT_FOUND = re.compile(
|
||||
r'^pods "wp-svc-[0-9a-hjkmnp-tv-z]{26}-ci-postgres" not found$'
|
||||
)
|
||||
|
||||
|
||||
def fail_usage(message: str) -> int:
|
||||
print(f"terminal-green contract input error: {message}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def load_record(argument: str | None) -> dict[str, Any]:
|
||||
if argument in (None, "-"):
|
||||
value = json.load(sys.stdin)
|
||||
else:
|
||||
with Path(argument).open(encoding="utf-8") as handle:
|
||||
value = json.load(handle)
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("pipeline record must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def is_issue_1000_artifact(step: dict[str, Any]) -> bool:
|
||||
error = step.get("error")
|
||||
exit_code = step.get("exit_code")
|
||||
return (
|
||||
step.get("name") == "ci-postgres"
|
||||
and step.get("type") == "service"
|
||||
and step.get("state") == "failure"
|
||||
and type(exit_code) is int
|
||||
and not isinstance(exit_code, bool)
|
||||
and exit_code == 0
|
||||
and isinstance(error, str)
|
||||
and POD_NOT_FOUND.fullmatch(error) is not None
|
||||
)
|
||||
|
||||
|
||||
def verify(record: dict[str, Any], expected_commit: str) -> tuple[int, dict[str, Any]]:
|
||||
anomalies: list[dict[str, Any]] = []
|
||||
candidates: list[dict[str, Any]] = []
|
||||
steps: list[dict[str, Any]] = []
|
||||
|
||||
pipeline_status = record.get("status")
|
||||
actual_commit = record.get("commit")
|
||||
if actual_commit != expected_commit:
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "pipeline",
|
||||
"name": str(record.get("number", "unknown")),
|
||||
"state": pipeline_status,
|
||||
"reason": "pipeline commit does not equal the expected PR head",
|
||||
"expected_commit": expected_commit,
|
||||
"actual_commit": actual_commit,
|
||||
}
|
||||
)
|
||||
if pipeline_status != "success":
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "pipeline",
|
||||
"name": str(record.get("number", "unknown")),
|
||||
"state": pipeline_status,
|
||||
"reason": "pipeline status is not success",
|
||||
}
|
||||
)
|
||||
|
||||
workflows = record.get("workflows")
|
||||
if not isinstance(workflows, list) or not workflows:
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "pipeline",
|
||||
"name": str(record.get("number", "unknown")),
|
||||
"state": pipeline_status,
|
||||
"reason": "workflows are missing or empty",
|
||||
}
|
||||
)
|
||||
workflows = []
|
||||
|
||||
for workflow_index, workflow in enumerate(workflows):
|
||||
if not isinstance(workflow, dict):
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "workflow",
|
||||
"name": str(workflow_index),
|
||||
"state": None,
|
||||
"reason": "workflow is not an object",
|
||||
}
|
||||
)
|
||||
continue
|
||||
workflow_name = str(workflow.get("name", workflow_index))
|
||||
if workflow.get("state") != "success":
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "workflow",
|
||||
"name": workflow_name,
|
||||
"state": workflow.get("state"),
|
||||
"reason": "workflow state is not success",
|
||||
}
|
||||
)
|
||||
children = workflow.get("children")
|
||||
if not isinstance(children, list) or not children:
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "workflow",
|
||||
"name": workflow_name,
|
||||
"state": workflow.get("state"),
|
||||
"reason": "child-step list is missing or empty",
|
||||
}
|
||||
)
|
||||
continue
|
||||
for child_index, child in enumerate(children):
|
||||
if not isinstance(child, dict):
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "step",
|
||||
"name": f"{workflow_name}[{child_index}]",
|
||||
"state": None,
|
||||
"reason": "step is not an object",
|
||||
}
|
||||
)
|
||||
continue
|
||||
steps.append(child)
|
||||
if child.get("state") == "success":
|
||||
continue
|
||||
if is_issue_1000_artifact(child):
|
||||
candidates.append(child)
|
||||
continue
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "step",
|
||||
"name": child.get("name"),
|
||||
"type": child.get("type"),
|
||||
"state": child.get("state"),
|
||||
"exit_code": child.get("exit_code"),
|
||||
"error": child.get("error"),
|
||||
"reason": "non-success step does not match the #1000 teardown signature",
|
||||
}
|
||||
)
|
||||
|
||||
if len(candidates) > 1:
|
||||
anomalies.append(
|
||||
{
|
||||
"scope": "exemption",
|
||||
"name": EXEMPTION_ID,
|
||||
"state": "invalid",
|
||||
"reason": "the #1000 exemption may apply to exactly one step",
|
||||
}
|
||||
)
|
||||
|
||||
exemption_applies = len(candidates) == 1 and not anomalies
|
||||
state_counts = Counter(str(step.get("state", "missing")) for step in steps)
|
||||
result: dict[str, Any] = {
|
||||
"schema_version": "mosaic-terminal-green/v1",
|
||||
"verdict": "terminal-green" if not anomalies else "not-terminal-green",
|
||||
"pipeline_number": record.get("number"),
|
||||
"commit": actual_commit,
|
||||
"expected_commit": expected_commit,
|
||||
"pipeline_status": pipeline_status,
|
||||
"total_steps": len(steps),
|
||||
"state_counts": dict(sorted(state_counts.items())),
|
||||
"exempted_steps": 1 if exemption_applies else 0,
|
||||
"anomalies": anomalies,
|
||||
}
|
||||
if exemption_applies:
|
||||
candidate = candidates[0]
|
||||
result["exemptions"] = [
|
||||
{
|
||||
"exemption_id": EXEMPTION_ID,
|
||||
"step": candidate.get("name"),
|
||||
"signature": candidate.get("error"),
|
||||
"tracking_issue": EXEMPTION_ISSUE,
|
||||
"retires_when": "issue #1000 is fixed",
|
||||
}
|
||||
]
|
||||
else:
|
||||
result["exemptions"] = []
|
||||
|
||||
return (0 if not anomalies else 1), result
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="verify the full Woodpecker terminal-green child-step contract"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expect-commit",
|
||||
required=True,
|
||||
metavar="FULL_SHA",
|
||||
help="full 40-hex PR-head commit that the pipeline record must match",
|
||||
)
|
||||
parser.add_argument("record", nargs="?", default="-", help="pipeline JSON file or -")
|
||||
arguments = parser.parse_args()
|
||||
if re.fullmatch(r"[0-9a-fA-F]{40}", arguments.expect_commit) is None:
|
||||
parser.error("--expect-commit must be a full 40-hex commit")
|
||||
arguments.expect_commit = arguments.expect_commit.lower()
|
||||
return arguments
|
||||
|
||||
|
||||
def main() -> int:
|
||||
arguments = parse_arguments()
|
||||
try:
|
||||
record = load_record(arguments.record)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
return fail_usage(str(error))
|
||||
code, result = verify(record, arguments.expect_commit)
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user