CHECKPOINT ONLY — not a deliverable, not reviewed, not complete.
Committed by mos-remediation as TRANSPORT after the authoring seat (coder-mos1) became
unable to act: its context window was exceeded at 102.6%/372k AND its overflow-recovery
path failed ("Turn prefix summarization failed: servers overloaded"). The seat could not
process instructions or commit its own work, leaving 28 modified files durable only on
local disk.
Authorship preserved as coder-mos1 — this is transport, not authorship. The work is the
seat's; committing it is the only way to make it survive the rotation that must follow.
Scope note, unresolved: this diff spans the queue-guard tools (9 files under
framework/tools/git) AND 5 framework guides plus 11 agent templates. RM-03 was scoped to
ci-queue-wait.sh and its tests. Whether the doc/template edits are consequential to the
tri-state change or drive-by was queried and never answered — the seat was bricked before
it could reply. That question stays open and must be resolved before any of this is
reviewed; the wider edits may need splitting out.
Direct instance of D-31: commit-then-rotate works, rotate-without-commit loses everything.
483 lines
15 KiB
Bash
Executable File
483 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ci-queue-wait.sh - Wait until project CI queue is clear (no running/queued pipeline on branch head)
|
|
# Usage: ci-queue-wait.sh [-B branch] [-t timeout_sec] [-i interval_sec] [--purpose push|merge] [--require-status]
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/detect-platform.sh"
|
|
|
|
BRANCH=""
|
|
TARGET_REPO=""
|
|
HEAD_SHA=""
|
|
TIMEOUT_SEC=900
|
|
INTERVAL_SEC=15
|
|
PURPOSE="merge"
|
|
REQUIRE_STATUS=0
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") [-B branch] [-R owner/repo] [--sha full-40] [-t timeout_sec] [-i interval_sec] [--purpose push|merge] [--require-status]
|
|
|
|
Options:
|
|
-B, --branch BRANCH Branch head to inspect (default: current branch)
|
|
-R, --repo OWNER/REPO Repository containing the branch (default: origin repo)
|
|
--sha FULL_SHA Inspect this exact 40-character commit instead of resolving the branch
|
|
-t, --timeout SECONDS Max wait time in seconds (default: 900)
|
|
-i, --interval SECONDS Poll interval in seconds (default: 15)
|
|
--purpose VALUE Log context: push|merge (default: merge)
|
|
--require-status Fail if no CI status contexts are present
|
|
-h, --help Show this help
|
|
|
|
Examples:
|
|
$(basename "$0")
|
|
$(basename "$0") --purpose push -t 600 -i 10
|
|
EOF
|
|
}
|
|
|
|
# get_remote_host and get_gitea_token are provided by detect-platform.sh
|
|
|
|
get_state_from_status_json() {
|
|
# Python source comes from -c so the provider payload remains on stdin.
|
|
# Never move the payload to argv: commit-status responses can exceed ARG_MAX.
|
|
python3 -c '
|
|
import json
|
|
import sys
|
|
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("status payload is not an object")
|
|
except Exception:
|
|
print("malformed")
|
|
raise SystemExit(0)
|
|
|
|
raw_statuses = payload.get("statuses", [])
|
|
raw_state = payload.get("state", "")
|
|
if not isinstance(raw_statuses, list) or not isinstance(raw_state, str):
|
|
print("malformed")
|
|
raise SystemExit(0)
|
|
statuses = raw_statuses
|
|
state = raw_state.lower()
|
|
|
|
pending_values = {"pending", "queued", "running", "waiting"}
|
|
failure_values = {"failure", "error", "failed"}
|
|
success_values = {"success"}
|
|
|
|
values = []
|
|
for item in statuses:
|
|
if not isinstance(item, dict):
|
|
print("malformed")
|
|
raise SystemExit(0)
|
|
raw_value = item.get("status") or item.get("state")
|
|
if not isinstance(raw_value, str) or not raw_value:
|
|
print("malformed")
|
|
raise SystemExit(0)
|
|
values.append(raw_value.lower())
|
|
|
|
if any(value in pending_values for value in values) or state in pending_values:
|
|
print("pending")
|
|
elif any(value in failure_values for value in values) or state in failure_values:
|
|
print("terminal-failure")
|
|
elif values and all(value in success_values for value in values) and state in {"", "success"}:
|
|
print("terminal-success")
|
|
elif not values:
|
|
print("no-status")
|
|
else:
|
|
print("unknown")
|
|
'
|
|
}
|
|
|
|
print_pending_contexts() {
|
|
python3 -c '
|
|
import json
|
|
import sys
|
|
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
except Exception:
|
|
print("[ci-queue-wait] unable to decode status payload")
|
|
raise SystemExit(0)
|
|
|
|
statuses = payload.get("statuses") or []
|
|
if not statuses:
|
|
print("[ci-queue-wait] no status contexts reported")
|
|
raise SystemExit(0)
|
|
|
|
pending_values = {"pending", "queued", "running", "waiting"}
|
|
found = False
|
|
for item in statuses:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
name = item.get("context") or item.get("name") or "unknown-context"
|
|
value = str(item.get("status") or item.get("state") or "unknown").lower()
|
|
target = item.get("target_url") or item.get("url") or ""
|
|
if value in pending_values:
|
|
found = True
|
|
suffix = f" ({target})" if target else ""
|
|
print(f"[ci-queue-wait] pending: {name}={value}{suffix}")
|
|
if not found:
|
|
print("[ci-queue-wait] no pending contexts")
|
|
'
|
|
}
|
|
|
|
record_cannot_assert() {
|
|
local reason="$1"
|
|
local audit_log="${MOSAIC_CI_QUEUE_AUDIT_LOG:-${XDG_STATE_HOME:-${HOME:-}/.local/state}/mosaic/audit/ci-queue-wait.jsonl}"
|
|
|
|
if [[ -z "$audit_log" ]] || ! mkdir -p "$(dirname "$audit_log")"; then
|
|
echo "Error: CANNOT_ASSERT and audit directory is unavailable; refusing degraded pass." >&2
|
|
return 70
|
|
fi
|
|
|
|
if ! python3 - "$audit_log" "$reason" "${PLATFORM:-unknown}" "$PURPOSE" "${BRANCH:-unknown}" "${OWNER:-unknown}/${REPO:-unknown}" <<'PY'
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
path, reason, platform, purpose, branch, repo = sys.argv[1:]
|
|
record = {
|
|
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"outcome": "CANNOT_ASSERT",
|
|
"reason": reason,
|
|
"platform": platform,
|
|
"purpose": purpose,
|
|
"disposition": "hold" if purpose == "merge" else "degraded-pass",
|
|
"branch": branch,
|
|
"repo": repo,
|
|
}
|
|
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
|
|
try:
|
|
os.write(fd, (json.dumps(record, separators=(",", ":")) + "\n").encode())
|
|
finally:
|
|
os.close(fd)
|
|
PY
|
|
then
|
|
echo "Error: CANNOT_ASSERT and audit write failed at ${audit_log}; refusing degraded pass." >&2
|
|
return 70
|
|
fi
|
|
|
|
if [[ "$PURPOSE" == "merge" ]]; then
|
|
echo "[ci-queue-wait] CANNOT_ASSERT reason=${reason} purpose=merge branch=${BRANCH:-unknown}; audited=${audit_log}; HOLD (exit 75). Retry after provider recovery; no manual reset is required." >&2
|
|
return 75
|
|
fi
|
|
|
|
echo "[ci-queue-wait] CANNOT_ASSERT reason=${reason} purpose=push branch=${BRANCH:-unknown}; audited=${audit_log}; push may proceed in degraded mode." >&2
|
|
return 0
|
|
}
|
|
|
|
github_get_branch_head_sha() {
|
|
local owner="$1"
|
|
local repo="$2"
|
|
local branch="$3"
|
|
gh api "repos/${owner}/${repo}/branches/${branch}" --jq '.commit.sha'
|
|
}
|
|
|
|
github_get_commit_status_json() {
|
|
local owner="$1"
|
|
local repo="$2"
|
|
local sha="$3"
|
|
local work_root status_file checks_file
|
|
work_root="${AGENT_WORK_ROOT:-${HOME:-}/.cache/mosaic/ci-queue-wait}"
|
|
mkdir -p "$work_root" || return 1
|
|
status_file=$(mktemp "$work_root/github-status.XXXXXX") || return 1
|
|
checks_file=$(mktemp "$work_root/github-checks.XXXXXX") || {
|
|
rm -f "$status_file"
|
|
return 1
|
|
}
|
|
|
|
if ! gh api --paginate --slurp "repos/${owner}/${repo}/commits/${sha}/statuses?per_page=100" > "$status_file" ||
|
|
! gh api --paginate --slurp "repos/${owner}/${repo}/commits/${sha}/check-runs?per_page=100&filter=latest" > "$checks_file"; then
|
|
rm -f "$status_file" "$checks_file"
|
|
return 1
|
|
fi
|
|
|
|
python3 - "$status_file" "$checks_file" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
status_pages = json.load(handle)
|
|
with open(sys.argv[2], encoding="utf-8") as handle:
|
|
check_pages = json.load(handle)
|
|
|
|
if not isinstance(status_pages, list) or not isinstance(check_pages, list):
|
|
raise SystemExit(1)
|
|
|
|
# The statuses endpoint is newest-first and can contain retries for one context.
|
|
# Keep only the newest entry per context after flattening every page.
|
|
combined = []
|
|
seen_contexts = set()
|
|
for page in status_pages:
|
|
if not isinstance(page, list):
|
|
raise SystemExit(1)
|
|
for status in page:
|
|
if not isinstance(status, dict):
|
|
raise SystemExit(1)
|
|
context = status.get("context")
|
|
if not isinstance(context, str) or not context or context in seen_contexts:
|
|
continue
|
|
seen_contexts.add(context)
|
|
combined.append(status)
|
|
|
|
check_runs = []
|
|
reported_total = 0
|
|
for page in check_pages:
|
|
if not isinstance(page, dict):
|
|
raise SystemExit(1)
|
|
page_runs = page.get("check_runs") or []
|
|
total_count = page.get("total_count")
|
|
if not isinstance(page_runs, list) or not isinstance(total_count, int):
|
|
raise SystemExit(1)
|
|
reported_total = max(reported_total, total_count)
|
|
check_runs.extend(page_runs)
|
|
if len(check_runs) < reported_total:
|
|
raise SystemExit(1)
|
|
|
|
for run in check_runs:
|
|
if not isinstance(run, dict):
|
|
raise SystemExit(1)
|
|
status = run.get("status")
|
|
conclusion = run.get("conclusion")
|
|
if status != "completed":
|
|
value = "pending"
|
|
elif conclusion == "success":
|
|
value = "success"
|
|
elif conclusion in {"failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"}:
|
|
value = "failure"
|
|
else:
|
|
value = "unknown"
|
|
combined.append({
|
|
"context": run.get("name") or "github-check",
|
|
"status": value,
|
|
"target_url": run.get("html_url") or run.get("details_url") or "",
|
|
})
|
|
|
|
json.dump({"state": "", "statuses": combined}, sys.stdout)
|
|
PY
|
|
local status=$?
|
|
rm -f "$status_file" "$checks_file"
|
|
return "$status"
|
|
}
|
|
|
|
gitea_get_branch_head_sha() {
|
|
local host="$1"
|
|
local repo="$2"
|
|
local branch="$3"
|
|
local token="$4"
|
|
local url="https://${host}/api/v1/repos/${repo}/branches/${branch}"
|
|
# Capture HTTP status so an absent branch (404) is distinguished from an API
|
|
# error. A not-yet-pushed feature branch has no in-flight pipeline, so the
|
|
# pre-push queue guard must treat 404 as "queue clear", not crash.
|
|
local resp code body
|
|
resp=$(curl -sS -H "User-Agent: curl/8" -H "Authorization: token ${token}" -w $'\n%{http_code}' "$url")
|
|
code="${resp##*$'\n'}"
|
|
body="${resp%$'\n'*}"
|
|
if [[ "$code" == "404" ]]; then
|
|
echo "__BRANCH_ABSENT__"
|
|
return 0
|
|
fi
|
|
if [[ "$code" != "200" ]]; then
|
|
return 1
|
|
fi
|
|
printf '%s' "$body" | python3 -c '
|
|
import json, sys
|
|
data = json.load(sys.stdin)
|
|
commit = data.get("commit") or {}
|
|
print((commit.get("id") or "").strip())
|
|
'
|
|
}
|
|
|
|
gitea_get_commit_status_json() {
|
|
local host="$1"
|
|
local repo="$2"
|
|
local sha="$3"
|
|
local token="$4"
|
|
local url="https://${host}/api/v1/repos/${repo}/commits/${sha}/status"
|
|
curl -fsSL -H "User-Agent: curl/8" -H "Authorization: token ${token}" "$url"
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-B|--branch)
|
|
BRANCH="$2"
|
|
shift 2
|
|
;;
|
|
-R|--repo)
|
|
TARGET_REPO="$2"
|
|
shift 2
|
|
;;
|
|
--sha)
|
|
HEAD_SHA="$2"
|
|
shift 2
|
|
;;
|
|
-t|--timeout)
|
|
TIMEOUT_SEC="$2"
|
|
shift 2
|
|
;;
|
|
-i|--interval)
|
|
INTERVAL_SEC="$2"
|
|
shift 2
|
|
;;
|
|
--purpose)
|
|
PURPOSE="$2"
|
|
shift 2
|
|
;;
|
|
--require-status)
|
|
REQUIRE_STATUS=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1" >&2
|
|
usage >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if ! [[ "$TIMEOUT_SEC" =~ ^[0-9]+$ ]] || ! [[ "$INTERVAL_SEC" =~ ^[0-9]+$ ]]; then
|
|
echo "Error: timeout and interval must be integer seconds." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "$HEAD_SHA" && ! "$HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
|
echo "Error: --sha must be a full 40-character hexadecimal commit SHA." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "$TARGET_REPO" && ! "$TARGET_REPO" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then
|
|
echo "Error: --repo must be OWNER/REPO." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$PURPOSE" != "push" && "$PURPOSE" != "merge" ]]; then
|
|
echo "Error: --purpose must be push or merge." >&2
|
|
exit 1
|
|
fi
|
|
|
|
OWNER="unknown"
|
|
REPO="unknown"
|
|
PLATFORM="unknown"
|
|
if ! OWNER=$(get_repo_owner) || [[ -z "$OWNER" ]]; then
|
|
record_cannot_assert "repository-owner-unresolvable"
|
|
exit $?
|
|
fi
|
|
if ! REPO=$(get_repo_name) || [[ -z "$REPO" ]]; then
|
|
record_cannot_assert "repository-name-unresolvable"
|
|
exit $?
|
|
fi
|
|
if ! detect_platform > /dev/null; then
|
|
PLATFORM="${PLATFORM:-unknown}"
|
|
record_cannot_assert "unsupported-platform"
|
|
exit $?
|
|
fi
|
|
PLATFORM="${PLATFORM:-unknown}"
|
|
|
|
if [[ -n "$TARGET_REPO" ]]; then
|
|
OWNER="${TARGET_REPO%%/*}"
|
|
REPO="${TARGET_REPO##*/}"
|
|
fi
|
|
|
|
if [[ -z "$BRANCH" ]]; then
|
|
if ! BRANCH=$(git symbolic-ref --quiet --short HEAD) || [[ -z "$BRANCH" ]]; then
|
|
record_cannot_assert "current-branch-unresolvable"
|
|
exit $?
|
|
fi
|
|
fi
|
|
|
|
if [[ "$PLATFORM" == "github" ]]; then
|
|
if ! command -v gh >/dev/null 2>&1; then
|
|
record_cannot_assert "github-cli-unavailable"
|
|
exit $?
|
|
fi
|
|
if [[ -z "$HEAD_SHA" ]]; then
|
|
if ! HEAD_SHA=$(github_get_branch_head_sha "$OWNER" "$REPO" "$BRANCH") || [[ -z "$HEAD_SHA" ]]; then
|
|
record_cannot_assert "branch-head-unavailable"
|
|
exit $?
|
|
fi
|
|
fi
|
|
echo "[ci-queue-wait] platform=github purpose=${PURPOSE} branch=${BRANCH} sha=${HEAD_SHA}"
|
|
elif [[ "$PLATFORM" == "gitea" ]]; then
|
|
if ! HOST=$(get_remote_host) || [[ -z "$HOST" ]]; then
|
|
record_cannot_assert "remote-host-unresolvable"
|
|
exit $?
|
|
fi
|
|
if ! TOKEN=$(get_gitea_token "$HOST") || [[ -z "$TOKEN" ]]; then
|
|
record_cannot_assert "credential-unresolvable"
|
|
exit $?
|
|
fi
|
|
if [[ -z "$HEAD_SHA" ]]; then
|
|
if ! HEAD_SHA=$(gitea_get_branch_head_sha "$HOST" "$OWNER/$REPO" "$BRANCH" "$TOKEN"); then
|
|
record_cannot_assert "branch-head-unavailable"
|
|
exit $?
|
|
fi
|
|
if [[ "$HEAD_SHA" == "__BRANCH_ABSENT__" ]]; then
|
|
echo "[ci-queue-wait] branch ${BRANCH} not yet on remote — no in-flight pipeline; queue clear."
|
|
exit 0
|
|
fi
|
|
if [[ -z "$HEAD_SHA" ]]; then
|
|
record_cannot_assert "branch-head-unavailable"
|
|
exit $?
|
|
fi
|
|
fi
|
|
echo "[ci-queue-wait] platform=gitea purpose=${PURPOSE} branch=${BRANCH} sha=${HEAD_SHA}"
|
|
else
|
|
record_cannot_assert "unsupported-platform"
|
|
exit $?
|
|
fi
|
|
|
|
START_TS=$(date +%s)
|
|
DEADLINE_TS=$((START_TS + TIMEOUT_SEC))
|
|
|
|
while true; do
|
|
NOW_TS=$(date +%s)
|
|
if (( NOW_TS > DEADLINE_TS )); then
|
|
echo "Error: ASSERTED_NOT_READY state=pending; timed out waiting for CI queue to clear on ${BRANCH} after ${TIMEOUT_SEC}s." >&2
|
|
exit 124
|
|
fi
|
|
|
|
if [[ "$PLATFORM" == "github" ]]; then
|
|
if ! STATUS_JSON=$(github_get_commit_status_json "$OWNER" "$REPO" "$HEAD_SHA"); then
|
|
record_cannot_assert "status-provider-unreachable"
|
|
exit $?
|
|
fi
|
|
else
|
|
if ! STATUS_JSON=$(gitea_get_commit_status_json "$HOST" "$OWNER/$REPO" "$HEAD_SHA" "$TOKEN"); then
|
|
record_cannot_assert "status-provider-unreachable"
|
|
exit $?
|
|
fi
|
|
fi
|
|
|
|
STATE=$(printf '%s' "$STATUS_JSON" | get_state_from_status_json)
|
|
echo "[ci-queue-wait] state=${STATE} purpose=${PURPOSE} branch=${BRANCH}"
|
|
|
|
case "$STATE" in
|
|
pending)
|
|
printf '%s' "$STATUS_JSON" | print_pending_contexts
|
|
sleep "$INTERVAL_SEC"
|
|
;;
|
|
terminal-success)
|
|
exit 0
|
|
;;
|
|
no-status)
|
|
if [[ "$REQUIRE_STATUS" -eq 1 ]]; then
|
|
echo "Error: ASSERTED_NOT_READY state=no-status; --require-status was set for ${BRANCH}." >&2
|
|
else
|
|
echo "Error: ASSERTED_NOT_READY state=no-status purpose=${PURPOSE} branch=${BRANCH}." >&2
|
|
fi
|
|
exit 3
|
|
;;
|
|
terminal-failure|malformed|unknown)
|
|
echo "Error: ASSERTED_NOT_READY state=${STATE} purpose=${PURPOSE} branch=${BRANCH}." >&2
|
|
exit 3
|
|
;;
|
|
*)
|
|
echo "Error: ASSERTED_NOT_READY unrecognized-state=${STATE} purpose=${PURPOSE} branch=${BRANCH}." >&2
|
|
exit 3
|
|
;;
|
|
esac
|
|
done
|