fix(rm-03): make CI queue guard fail on asserted non-readiness (#1032)

Co-authored-by: coder-mos1 <[email protected]>

packages/mosaic/framework/tools/git/ci-queue-wait.sh

@@ -7,7 +7,9 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "$SCRIPT_DIR/detect-platform.sh"
 
-BRANCH="main"
+BRANCH=""
+TARGET_REPO=""
+HEAD_SHA=""
 TIMEOUT_SEC=900
 INTERVAL_SEC=15
 PURPOSE="merge"
@@ -15,10 +17,12 @@ REQUIRE_STATUS=0
 
 usage() {
     cat <<EOF
-Usage: $(basename "$0") [-B branch] [-t timeout_sec] [-i interval_sec] [--purpose push|merge] [--require-status]
+Usage: $(basename "$0") [-B branch] [-R owner/repo] [--sha full-40] [-t timeout_sec] [-i interval_sec] [--purpose push|merge] [--require-status]
 
 Options:
-  -B, --branch BRANCH      Branch head to inspect (default: main)
+  -B, --branch BRANCH      Branch head to inspect (default: current branch)
+  -R, --repo OWNER/REPO    Repository containing the branch (default: origin repo)
+      --sha FULL_SHA       Inspect this exact 40-character commit instead of resolving the branch
   -t, --timeout SECONDS    Max wait time in seconds (default: 900)
   -i, --interval SECONDS   Poll interval in seconds (default: 15)
       --purpose VALUE      Log context: push|merge (default: merge)
@@ -27,63 +31,65 @@ Options:
 
 Examples:
   $(basename "$0")
-  $(basename "$0") --purpose push -B main -t 600 -i 10
+  $(basename "$0") --purpose push -t 600 -i 10
 EOF
 }
 
 # get_remote_host and get_gitea_token are provided by detect-platform.sh
 
 get_state_from_status_json() {
-    python3 - <<'PY'
+    # Python source comes from -c so the provider payload remains on stdin.
+    # Never move the payload to argv: commit-status responses can exceed ARG_MAX.
+    python3 -c '
 import json
 import sys
 
 try:
     payload = json.load(sys.stdin)
+    if not isinstance(payload, dict):
+        raise ValueError("status payload is not an object")
 except Exception:
-    print("unknown")
+    print("malformed")
     raise SystemExit(0)
 
-statuses = payload.get("statuses") or []
-state = (payload.get("state") or "").lower()
+raw_statuses = payload.get("statuses", [])
+raw_state = payload.get("state", "")
+if not isinstance(raw_statuses, list) or not isinstance(raw_state, str):
+    print("malformed")
+    raise SystemExit(0)
+statuses = raw_statuses
+state = raw_state.lower()
 
 pending_values = {"pending", "queued", "running", "waiting"}
 failure_values = {"failure", "error", "failed"}
 success_values = {"success"}
 
-if state in pending_values:
-    print("pending")
-    raise SystemExit(0)
-if state in failure_values:
-    print("terminal-failure")
-    raise SystemExit(0)
-if state in success_values:
-    print("terminal-success")
-    raise SystemExit(0)
-
 values = []
 for item in statuses:
     if not isinstance(item, dict):
-        continue
-    value = (item.get("status") or item.get("state") or "").lower()
-    if value:
-        values.append(value)
+        print("malformed")
+        raise SystemExit(0)
+    raw_value = item.get("status") or item.get("state")
+    if not isinstance(raw_value, str) or not raw_value:
+        print("malformed")
+        raise SystemExit(0)
+    values.append(raw_value.lower())
 
-if not values and not state:
-    print("no-status")
-elif any(v in pending_values for v in values):
+if any(value in pending_values for value in values) or state in pending_values:
     print("pending")
-elif any(v in failure_values for v in values):
+elif any(value in failure_values for value in values) or state in failure_values:
     print("terminal-failure")
-elif values and all(v in success_values for v in values):
+elif values and all(value in success_values for value in values) and state in {"", "success"}:
     print("terminal-success")
+elif not values:
+    print("no-status")
 else:
     print("unknown")
-PY
+'
 }
 
 print_pending_contexts() {
-    python3 - <<'PY'
+    python3 -c '
 import json
 import sys
 
@@ -104,17 +110,61 @@ for item in statuses:
     if not isinstance(item, dict):
         continue
     name = item.get("context") or item.get("name") or "unknown-context"
-    value = (item.get("status") or item.get("state") or "unknown").lower()
+    value = str(item.get("status") or item.get("state") or "unknown").lower()
     target = item.get("target_url") or item.get("url") or ""
     if value in pending_values:
         found = True
-        if target:
-            print(f"[ci-queue-wait] pending: {name}={value} ({target})")
-        else:
-            print(f"[ci-queue-wait] pending: {name}={value}")
+        suffix = f" ({target})" if target else ""
+        print(f"[ci-queue-wait] pending: {name}={value}{suffix}")
 if not found:
     print("[ci-queue-wait] no pending contexts")
+'
+}
+
+record_cannot_assert() {
+    local reason="$1"
+    local audit_log="${MOSAIC_CI_QUEUE_AUDIT_LOG:-${XDG_STATE_HOME:-${HOME:-}/.local/state}/mosaic/audit/ci-queue-wait.jsonl}"
+
+    if [[ -z "$audit_log" ]] || ! mkdir -p "$(dirname "$audit_log")"; then
+        echo "Error: CANNOT_ASSERT and audit directory is unavailable; refusing degraded pass." >&2
+        return 70
+    fi
+
+    if ! python3 - "$audit_log" "$reason" "${PLATFORM:-unknown}" "$PURPOSE" "${BRANCH:-unknown}" "${OWNER:-unknown}/${REPO:-unknown}" <<'PY'
+import datetime
+import json
+import os
+import sys
+
+path, reason, platform, purpose, branch, repo = sys.argv[1:]
+record = {
+    "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+    "outcome": "CANNOT_ASSERT",
+    "reason": reason,
+    "platform": platform,
+    "purpose": purpose,
+    "disposition": "hold" if purpose == "merge" else "degraded-pass",
+    "branch": branch,
+    "repo": repo,
+}
+fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
+try:
+    os.write(fd, (json.dumps(record, separators=(",", ":")) + "\n").encode())
+finally:
+    os.close(fd)
 PY
+    then
+        echo "Error: CANNOT_ASSERT and audit write failed at ${audit_log}; refusing degraded pass." >&2
+        return 70
+    fi
+
+    if [[ "$PURPOSE" == "merge" ]]; then
+        echo "[ci-queue-wait] CANNOT_ASSERT reason=${reason} purpose=merge branch=${BRANCH:-unknown}; audited=${audit_log}; HOLD (exit 75). Retry after provider recovery; no manual reset is required." >&2
+        return 75
+    fi
+
+    echo "[ci-queue-wait] CANNOT_ASSERT reason=${reason} purpose=push branch=${BRANCH:-unknown}; audited=${audit_log}; push may proceed in degraded mode." >&2
+    return 0
 }
 
 github_get_branch_head_sha() {
@@ -128,7 +178,87 @@ github_get_commit_status_json() {
     local owner="$1"
     local repo="$2"
     local sha="$3"
-    gh api "repos/${owner}/${repo}/commits/${sha}/status"
+    local work_root status_file checks_file
+    work_root="${AGENT_WORK_ROOT:-${HOME:-}/.cache/mosaic/ci-queue-wait}"
+    mkdir -p "$work_root" || return 1
+    status_file=$(mktemp "$work_root/github-status.XXXXXX") || return 1
+    checks_file=$(mktemp "$work_root/github-checks.XXXXXX") || {
+        rm -f "$status_file"
+        return 1
+    }
+
+    if ! gh api --paginate --slurp "repos/${owner}/${repo}/commits/${sha}/statuses?per_page=100" > "$status_file" ||
+       ! gh api --paginate --slurp "repos/${owner}/${repo}/commits/${sha}/check-runs?per_page=100&filter=latest" > "$checks_file"; then
+        rm -f "$status_file" "$checks_file"
+        return 1
+    fi
+
+    python3 - "$status_file" "$checks_file" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as handle:
+    status_pages = json.load(handle)
+with open(sys.argv[2], encoding="utf-8") as handle:
+    check_pages = json.load(handle)
+
+if not isinstance(status_pages, list) or not isinstance(check_pages, list):
+    raise SystemExit(1)
+
+# The statuses endpoint is newest-first and can contain retries for one context.
+# Keep only the newest entry per context after flattening every page.
+combined = []
+seen_contexts = set()
+for page in status_pages:
+    if not isinstance(page, list):
+        raise SystemExit(1)
+    for status in page:
+        if not isinstance(status, dict):
+            raise SystemExit(1)
+        context = status.get("context")
+        if not isinstance(context, str) or not context or context in seen_contexts:
+            continue
+        seen_contexts.add(context)
+        combined.append(status)
+
+check_runs = []
+reported_total = 0
+for page in check_pages:
+    if not isinstance(page, dict):
+        raise SystemExit(1)
+    page_runs = page.get("check_runs") or []
+    total_count = page.get("total_count")
+    if not isinstance(page_runs, list) or not isinstance(total_count, int):
+        raise SystemExit(1)
+    reported_total = max(reported_total, total_count)
+    check_runs.extend(page_runs)
+if len(check_runs) < reported_total:
+    raise SystemExit(1)
+
+for run in check_runs:
+    if not isinstance(run, dict):
+        raise SystemExit(1)
+    status = run.get("status")
+    conclusion = run.get("conclusion")
+    if status != "completed":
+        value = "pending"
+    elif conclusion == "success":
+        value = "success"
+    elif conclusion in {"failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"}:
+        value = "failure"
+    else:
+        value = "unknown"
+    combined.append({
+        "context": run.get("name") or "github-check",
+        "status": value,
+        "target_url": run.get("html_url") or run.get("details_url") or "",
+    })
+
+json.dump({"state": "", "statuses": combined}, sys.stdout)
+PY
+    local status=$?
+    rm -f "$status_file" "$checks_file"
+    return "$status"
 }
 
 gitea_get_branch_head_sha() {
@@ -174,6 +304,14 @@ while [[ $# -gt 0 ]]; do
             BRANCH="$2"
             shift 2
             ;;
+        -R|--repo)
+            TARGET_REPO="$2"
+            shift 2
+            ;;
+        --sha)
+            HEAD_SHA="$2"
+            shift 2
+            ;;
         -t|--timeout)
             TIMEOUT_SEC="$2"
             shift 2
@@ -206,45 +344,89 @@ if ! [[ "$TIMEOUT_SEC" =~ ^[0-9]+$ ]] || ! [[ "$INTERVAL_SEC" =~ ^[0-9]+$ ]]; th
     echo "Error: timeout and interval must be integer seconds." >&2
     exit 1
 fi
+if [[ -n "$HEAD_SHA" && ! "$HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
+    echo "Error: --sha must be a full 40-character hexadecimal commit SHA." >&2
+    exit 1
+fi
+if [[ -n "$TARGET_REPO" && ! "$TARGET_REPO" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then
+    echo "Error: --repo must be OWNER/REPO." >&2
+    exit 1
+fi
 
-OWNER=$(get_repo_owner)
-REPO=$(get_repo_name)
-detect_platform > /dev/null
+if [[ "$PURPOSE" != "push" && "$PURPOSE" != "merge" ]]; then
+    echo "Error: --purpose must be push or merge." >&2
+    exit 1
+fi
+
+OWNER="unknown"
+REPO="unknown"
+PLATFORM="unknown"
+if ! OWNER=$(get_repo_owner) || [[ -z "$OWNER" ]]; then
+    record_cannot_assert "repository-owner-unresolvable"
+    exit $?
+fi
+if ! REPO=$(get_repo_name) || [[ -z "$REPO" ]]; then
+    record_cannot_assert "repository-name-unresolvable"
+    exit $?
+fi
+if ! detect_platform > /dev/null; then
+    PLATFORM="${PLATFORM:-unknown}"
+    record_cannot_assert "unsupported-platform"
+    exit $?
+fi
 PLATFORM="${PLATFORM:-unknown}"
 
+if [[ -n "$TARGET_REPO" ]]; then
+    OWNER="${TARGET_REPO%%/*}"
+    REPO="${TARGET_REPO##*/}"
+fi
+
+if [[ -z "$BRANCH" ]]; then
+    if ! BRANCH=$(git symbolic-ref --quiet --short HEAD) || [[ -z "$BRANCH" ]]; then
+        record_cannot_assert "current-branch-unresolvable"
+        exit $?
+    fi
+fi
+
 if [[ "$PLATFORM" == "github" ]]; then
     if ! command -v gh >/dev/null 2>&1; then
-        echo "Error: gh CLI is required for GitHub CI queue guard." >&2
-        exit 1
+        record_cannot_assert "github-cli-unavailable"
+        exit $?
     fi
-    HEAD_SHA=$(github_get_branch_head_sha "$OWNER" "$REPO" "$BRANCH")
     if [[ -z "$HEAD_SHA" ]]; then
-        echo "Error: Could not resolve ${BRANCH} head SHA." >&2
-        exit 1
+        if ! HEAD_SHA=$(github_get_branch_head_sha "$OWNER" "$REPO" "$BRANCH") || [[ -z "$HEAD_SHA" ]]; then
+            record_cannot_assert "branch-head-unavailable"
+            exit $?
+        fi
     fi
     echo "[ci-queue-wait] platform=github purpose=${PURPOSE} branch=${BRANCH} sha=${HEAD_SHA}"
 elif [[ "$PLATFORM" == "gitea" ]]; then
-    HOST=$(get_remote_host) || {
-        echo "Error: Could not determine remote host." >&2
-        exit 1
-    }
-    TOKEN=$(get_gitea_token "$HOST") || {
-        echo "Error: Gitea token not found. Set GITEA_TOKEN or configure ~/.git-credentials." >&2
-        exit 1
-    }
-    HEAD_SHA=$(gitea_get_branch_head_sha "$HOST" "$OWNER/$REPO" "$BRANCH" "$TOKEN")
-    if [[ "$HEAD_SHA" == "__BRANCH_ABSENT__" ]]; then
-        echo "[ci-queue-wait] branch ${BRANCH} not yet on remote — no in-flight pipeline; queue clear."
-        exit 0
+    if ! HOST=$(get_remote_host) || [[ -z "$HOST" ]]; then
+        record_cannot_assert "remote-host-unresolvable"
+        exit $?
+    fi
+    if ! TOKEN=$(get_gitea_token "$HOST") || [[ -z "$TOKEN" ]]; then
+        record_cannot_assert "credential-unresolvable"
+        exit $?
     fi
     if [[ -z "$HEAD_SHA" ]]; then
-        echo "Error: Could not resolve ${BRANCH} head SHA." >&2
-        exit 1
+        if ! HEAD_SHA=$(gitea_get_branch_head_sha "$HOST" "$OWNER/$REPO" "$BRANCH" "$TOKEN"); then
+            record_cannot_assert "branch-head-unavailable"
+            exit $?
+        fi
+        if [[ "$HEAD_SHA" == "__BRANCH_ABSENT__" ]]; then
+            echo "[ci-queue-wait] branch ${BRANCH} not yet on remote — no in-flight pipeline; queue clear."
+            exit 0
+        fi
+        if [[ -z "$HEAD_SHA" ]]; then
+            record_cannot_assert "branch-head-unavailable"
+            exit $?
+        fi
     fi
     echo "[ci-queue-wait] platform=gitea purpose=${PURPOSE} branch=${BRANCH} sha=${HEAD_SHA}"
 else
-    echo "Error: Unsupported platform '${PLATFORM}'." >&2
-    exit 1
+    record_cannot_assert "unsupported-platform"
+    exit $?
 fi
 
 START_TS=$(date +%s)
@@ -253,14 +435,20 @@ DEADLINE_TS=$((START_TS + TIMEOUT_SEC))
 while true; do
     NOW_TS=$(date +%s)
     if (( NOW_TS > DEADLINE_TS )); then
-        echo "Error: Timed out waiting for CI queue to clear on ${BRANCH} after ${TIMEOUT_SEC}s." >&2
+        echo "Error: ASSERTED_NOT_READY state=pending; timed out waiting for CI queue to clear on ${BRANCH} after ${TIMEOUT_SEC}s." >&2
         exit 124
     fi
 
     if [[ "$PLATFORM" == "github" ]]; then
-        STATUS_JSON=$(github_get_commit_status_json "$OWNER" "$REPO" "$HEAD_SHA")
+        if ! STATUS_JSON=$(github_get_commit_status_json "$OWNER" "$REPO" "$HEAD_SHA"); then
+            record_cannot_assert "status-provider-unreachable"
+            exit $?
+        fi
     else
-        STATUS_JSON=$(gitea_get_commit_status_json "$HOST" "$OWNER/$REPO" "$HEAD_SHA" "$TOKEN")
+        if ! STATUS_JSON=$(gitea_get_commit_status_json "$HOST" "$OWNER/$REPO" "$HEAD_SHA" "$TOKEN"); then
+            record_cannot_assert "status-provider-unreachable"
+            exit $?
+        fi
     fi
 
     STATE=$(printf '%s' "$STATUS_JSON" | get_state_from_status_json)
@@ -271,21 +459,24 @@ while true; do
             printf '%s' "$STATUS_JSON" | print_pending_contexts
             sleep "$INTERVAL_SEC"
             ;;
+        terminal-success)
+            exit 0
+            ;;
         no-status)
             if [[ "$REQUIRE_STATUS" -eq 1 ]]; then
-                echo "Error: No CI status contexts found for ${BRANCH} while --require-status is set." >&2
-                exit 1
+                echo "Error: ASSERTED_NOT_READY state=no-status; --require-status was set for ${BRANCH}." >&2
+            else
+                echo "Error: ASSERTED_NOT_READY state=no-status purpose=${PURPOSE} branch=${BRANCH}." >&2
             fi
-            echo "[ci-queue-wait] no status contexts present; proceeding."
-            exit 0
+            exit 3
             ;;
-        terminal-success|terminal-failure|unknown)
-            # Queue guard only blocks on pending/running/queued states.
-            exit 0
+        terminal-failure|malformed|unknown)
+            echo "Error: ASSERTED_NOT_READY state=${STATE} purpose=${PURPOSE} branch=${BRANCH}." >&2
+            exit 3
             ;;
         *)
-            echo "[ci-queue-wait] unrecognized state '${STATE}', proceeding conservatively."
-            exit 0
+            echo "Error: ASSERTED_NOT_READY unrecognized-state=${STATE} purpose=${PURPOSE} branch=${BRANCH}." >&2
+            exit 3
             ;;
     esac
 done