365 lines
13 KiB
Bash
365 lines
13 KiB
Bash
#!/usr/bin/env bash
|
|
# Exit-asserting RM-03 regression harness for ci-queue-wait.sh.
|
|
# Every case is a process-level assertion: a classifier-only green cannot satisfy it.
|
|
|
|
set -u
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/ci-queue-wait-tristate}"
|
|
REPO_DIR="$WORK_DIR/repo"
|
|
STUB_DIR="$WORK_DIR/stubs"
|
|
AUDIT_LOG="$WORK_DIR/audit/ci-queue-wait.jsonl"
|
|
STATUS_OBSERVED="$WORK_DIR/status-observed"
|
|
CLOCK_LOG="$WORK_DIR/clock.log"
|
|
WATCHDOG_PYTHON="/usr/bin/python3"
|
|
WATCHDOG_SCRIPT="$WORK_DIR/real-clock-watchdog.py"
|
|
WATCHDOG_TIMEOUT_SEC=5
|
|
WATCHDOG_EXIT=90
|
|
FEATURE_BRANCH="fix/rm-03-fixture"
|
|
|
|
if [[ ! -x "$WATCHDOG_PYTHON" ]]; then
|
|
echo "FAIL setup: required real-clock watchdog runtime is unavailable at $WATCHDOG_PYTHON" >&2
|
|
exit 1
|
|
fi
|
|
|
|
rm -rf "$WORK_DIR"
|
|
mkdir -p "$REPO_DIR" "$STUB_DIR"
|
|
cat > "$WATCHDOG_SCRIPT" <<'PY'
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
|
|
if len(sys.argv) < 3:
|
|
raise SystemExit(2)
|
|
|
|
timeout_seconds = float(sys.argv[1])
|
|
process = subprocess.Popen(sys.argv[2:], start_new_session=True)
|
|
try:
|
|
return_code = process.wait(timeout=timeout_seconds)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
process.wait()
|
|
print(
|
|
f"FAIL HANG watchdog: subject exceeded {timeout_seconds:g}s "
|
|
"before completing its intended path",
|
|
file=sys.stderr,
|
|
)
|
|
raise SystemExit(90)
|
|
|
|
if return_code < 0:
|
|
raise SystemExit(128 - return_code)
|
|
raise SystemExit(return_code)
|
|
PY
|
|
git -C "$REPO_DIR" init -q
|
|
git -C "$REPO_DIR" checkout -q -b "$FEATURE_BRANCH"
|
|
git -C "$REPO_DIR" remote add origin https://git.example.test/acme/widgets.git
|
|
|
|
cat > "$STUB_DIR/curl" <<'SH'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
url=""
|
|
has_write_out=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
-w) has_write_out=1 ;;
|
|
http://*|https://*) url="$arg" ;;
|
|
esac
|
|
done
|
|
printf '%s\n' "$url" >> "${MOSAIC_STUB_URL_LOG:?}"
|
|
|
|
case "$url" in
|
|
*/branches/*)
|
|
if [[ "${MOSAIC_STUB_BRANCH_MODE:-ok}" == "hang-before-provider" ]]; then
|
|
while :; do :; done
|
|
fi
|
|
if [[ "${MOSAIC_STUB_BRANCH_MODE:-ok}" == "unreachable" ]]; then
|
|
exit 7
|
|
fi
|
|
body='{"commit":{"id":"deadbeefcafef00d0123456789abcdef01234567"}}'
|
|
if [[ "$has_write_out" -eq 1 ]]; then
|
|
printf '%s\n200' "$body"
|
|
else
|
|
printf '%s' "$body"
|
|
fi
|
|
;;
|
|
*/status)
|
|
: > "${MOSAIC_STUB_STATUS_OBSERVED:?}"
|
|
case "${MOSAIC_STUB_STATUS_MODE:?}" in
|
|
success) printf '%s' '{"state":"success","statuses":[{"status":"success"}]}' ;;
|
|
pending) printf '%s' '{"state":"pending","statuses":[{"status":"pending","context":"ci/test"}]}' ;;
|
|
failure) printf '%s' '{"state":"failure","statuses":[{"status":"failure"}]}' ;;
|
|
no-status) printf '%s' '{"state":"","statuses":[]}' ;;
|
|
aggregate-success-no-status) printf '%s' '{"state":"success","statuses":[]}' ;;
|
|
malformed) printf '%s' 'not-json' ;;
|
|
malformed-statuses-type) printf '%s' '{"state":"success","statuses":"corrupt"}' ;;
|
|
malformed-status-entry) printf '%s' '{"state":"success","statuses":[null]}' ;;
|
|
large-success)
|
|
python3 -c 'import json; print(json.dumps({"state":"success", "statuses":[{"status":"success"}], "padding":"x" * (160 * 1024)}), end="")'
|
|
;;
|
|
unreachable) exit 7 ;;
|
|
*) echo "unknown status mode" >&2; exit 2 ;;
|
|
esac
|
|
;;
|
|
*) echo "unexpected curl URL: $url" >&2; exit 2 ;;
|
|
esac
|
|
SH
|
|
|
|
cat > "$STUB_DIR/date" <<'SH'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
if [[ "$#" -ne 1 || "$1" != "+%s" ]]; then
|
|
echo "unexpected date invocation: $*" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ -e "${MOSAIC_STUB_STATUS_OBSERVED:?}" ]]; then
|
|
printf 'date-phase=after-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}"
|
|
printf '1002\n'
|
|
else
|
|
printf 'date-phase=before-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}"
|
|
printf '1000\n'
|
|
fi
|
|
SH
|
|
|
|
cat > "$STUB_DIR/sleep" <<'SH'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
printf 'sleep-after-status=%s\n' "$*" >> "${MOSAIC_STUB_CLOCK_LOG:?}"
|
|
SH
|
|
chmod +x "$STUB_DIR/curl" "$STUB_DIR/date" "$STUB_DIR/sleep"
|
|
|
|
run_guard() {
|
|
local status_mode="$1"
|
|
local audit_log="${2:-$AUDIT_LOG}"
|
|
shift 2 || true
|
|
(
|
|
cd "$REPO_DIR" || exit
|
|
export PATH="$STUB_DIR:$PATH"
|
|
export MOSAIC_CREDENTIALS_FILE="$WORK_DIR/no-credentials.json"
|
|
if [[ "$status_mode" == "credential-unresolvable" ]]; then
|
|
export HOME="$WORK_DIR/empty-home"
|
|
mkdir -p "$HOME"
|
|
unset GITEA_TOKEN GITEA_URL MOSAIC_GIT_IDENTITY
|
|
export MOSAIC_STUB_STATUS_MODE=success
|
|
else
|
|
export GITEA_TOKEN=stub-token
|
|
export GITEA_URL=https://git.example.test
|
|
export MOSAIC_STUB_STATUS_MODE="$status_mode"
|
|
fi
|
|
rm -f "$STATUS_OBSERVED" "$CLOCK_LOG"
|
|
export MOSAIC_STUB_URL_LOG="$WORK_DIR/urls.log"
|
|
export MOSAIC_STUB_STATUS_OBSERVED="$STATUS_OBSERVED"
|
|
export MOSAIC_STUB_CLOCK_LOG="$CLOCK_LOG"
|
|
export MOSAIC_CI_QUEUE_AUDIT_LOG="$audit_log"
|
|
# Provider observation is the synchronization event. The one-second
|
|
# timeout is subject semantics under virtual time, never a wall wait.
|
|
# The absolute Python runtime uses an internal monotonic wait and kills
|
|
# the subject's isolated process group. Neither operation can resolve
|
|
# to the virtual date/sleep stubs at the front of PATH.
|
|
local subject_rc
|
|
if "$WATCHDOG_PYTHON" "$WATCHDOG_SCRIPT" "$WATCHDOG_TIMEOUT_SEC" \
|
|
"$SCRIPT_DIR/ci-queue-wait.sh" --purpose "${MOSAIC_TEST_PURPOSE:-push}" -t 1 -i 1 "$@"; then
|
|
subject_rc=0
|
|
else
|
|
subject_rc=$?
|
|
fi
|
|
return "$subject_rc"
|
|
)
|
|
}
|
|
|
|
failures=0
|
|
assert_provider_observed() {
|
|
local name="$1" require_expiration="${2:-0}"
|
|
if [[ ! -e "$STATUS_OBSERVED" ]]; then
|
|
echo "FAIL $name: status provider was not observed" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ ! -s "$CLOCK_LOG" ]] || ! grep -q '^date-phase=before-status$' "$CLOCK_LOG"; then
|
|
echo "FAIL $name: virtual clock interception did not run before provider observation" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$require_expiration" -eq 1 ]]; then
|
|
if ! grep -q '^sleep-after-status=' "$CLOCK_LOG" || ! grep -q '^date-phase=after-status$' "$CLOCK_LOG"; then
|
|
echo "FAIL $name: pending path did not expire after provider observation" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
fi
|
|
}
|
|
|
|
run_assertion() {
|
|
local name="$1" expected_rc="$2" status_mode="$3" required_text="$4"
|
|
local output rc
|
|
shift 4
|
|
set +e
|
|
output=$(run_guard "$status_mode" "$AUDIT_LOG" "$@" 2>&1)
|
|
rc=$?
|
|
set -e
|
|
|
|
case "$expected_rc" in
|
|
zero)
|
|
if [[ "$rc" -ne 0 ]]; then
|
|
echo "FAIL $name: expected rc=0, got rc=$rc" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
;;
|
|
nonzero)
|
|
if [[ "$rc" -eq 0 ]]; then
|
|
echo "FAIL $name: expected rc!=0, got rc=0" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
;;
|
|
not126)
|
|
if [[ "$rc" -eq 126 ]]; then
|
|
echo "FAIL $name: payload transport hit ARG_MAX (rc=126)" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
;;
|
|
esac
|
|
if [[ "$output" != *"$required_text"* ]]; then
|
|
echo "FAIL $name: output missing '$required_text' (rc=$rc)" >&2
|
|
printf '%s\n' "$output" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$status_mode" != "credential-unresolvable" ]]; then
|
|
if [[ "$status_mode" == "pending" ]]; then
|
|
assert_provider_observed "$name" 1
|
|
else
|
|
assert_provider_observed "$name"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
set -e
|
|
: > "$WORK_DIR/urls.log"
|
|
run_assertion success zero success 'state=terminal-success'
|
|
run_assertion pending nonzero pending 'ASSERTED_NOT_READY'
|
|
run_assertion failure nonzero failure 'ASSERTED_NOT_READY'
|
|
run_assertion no-status nonzero no-status 'ASSERTED_NOT_READY'
|
|
run_assertion aggregate-success-no-status nonzero aggregate-success-no-status 'ASSERTED_NOT_READY'
|
|
run_assertion malformed nonzero malformed 'ASSERTED_NOT_READY'
|
|
run_assertion malformed-statuses-type nonzero malformed-statuses-type 'ASSERTED_NOT_READY'
|
|
run_assertion malformed-status-entry nonzero malformed-status-entry 'ASSERTED_NOT_READY'
|
|
run_assertion large-payload not126 large-success 'state=terminal-success'
|
|
run_assertion credential-unresolvable zero credential-unresolvable 'CANNOT_ASSERT'
|
|
run_assertion provider-unreachable zero unreachable 'CANNOT_ASSERT'
|
|
|
|
# Positive liveness control: a subject mutant hangs before the branch lookup
|
|
# can reach the status provider. Only the independent real-clock watchdog may
|
|
# terminate it, and its failure must be distinct from subject timeout rc=124.
|
|
set +e
|
|
watchdog_output=$(MOSAIC_STUB_BRANCH_MODE=hang-before-provider run_guard success "$AUDIT_LOG" 2>&1)
|
|
watchdog_rc=$?
|
|
set -e
|
|
if [[ "$watchdog_rc" -ne "$WATCHDOG_EXIT" ]]; then
|
|
echo "FAIL watchdog-control: expected hang-specific rc=$WATCHDOG_EXIT, got rc=$watchdog_rc" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$watchdog_output" != *"FAIL HANG watchdog:"* ]]; then
|
|
echo "FAIL watchdog-control: expected distinct hang-specific diagnostic" >&2
|
|
printf '%s\n' "$watchdog_output" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ -e "$STATUS_OBSERVED" ]]; then
|
|
echo "FAIL watchdog-control: hanging mutant unexpectedly reached the status provider" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
if [[ ! -s "$AUDIT_LOG" ]] || ! grep -q '"outcome":"CANNOT_ASSERT"' "$AUDIT_LOG"; then
|
|
echo "FAIL provider-unreachable-audit: expected durable CANNOT_ASSERT JSONL record" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
# Merge cannot proceed without exact-head evidence. CANNOT_ASSERT is retryable exit 75,
|
|
# distinct from ASSERTED_NOT_READY (3/124), and still writes its audit record.
|
|
merge_audit_lines_before=$(wc -l < "$AUDIT_LOG")
|
|
set +e
|
|
merge_unreachable_output=$(MOSAIC_TEST_PURPOSE=merge run_guard unreachable "$AUDIT_LOG" 2>&1)
|
|
merge_unreachable_rc=$?
|
|
set -e
|
|
if [[ "$merge_unreachable_rc" -ne 75 ]]; then
|
|
echo "FAIL merge-provider-unreachable: expected rc=75, got rc=$merge_unreachable_rc" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$merge_unreachable_output" != *"CANNOT_ASSERT"* ]]; then
|
|
echo "FAIL merge-provider-unreachable: expected loud CANNOT_ASSERT diagnostic" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
assert_provider_observed merge-provider-unreachable
|
|
merge_audit_lines_after=$(wc -l < "$AUDIT_LOG")
|
|
if [[ "$merge_audit_lines_after" -le "$merge_audit_lines_before" ]]; then
|
|
echo "FAIL merge-provider-unreachable: expected an additional audit record" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
# A feature-branch push with no -B must inspect the checked-out feature branch.
|
|
if ! grep -q "/branches/$FEATURE_BRANCH" "$WORK_DIR/urls.log"; then
|
|
echo "FAIL implicit-branch: provider was not queried for $FEATURE_BRANCH" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
# Merge callers can pin both a fork repository and the exact reviewed head SHA.
|
|
exact_sha=0123456789abcdef0123456789abcdef01234567
|
|
: > "$WORK_DIR/urls.log"
|
|
run_assertion exact-fork-head zero success 'state=terminal-success' \
|
|
-B fix/rm-03-fixture -R contributor/widgets-fork --sha "$exact_sha"
|
|
if ! grep -q "/repos/contributor/widgets-fork/commits/$exact_sha/status" "$WORK_DIR/urls.log"; then
|
|
echo "FAIL exact-fork-head: status URL did not bind fork repository and exact SHA" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if grep -q '/branches/' "$WORK_DIR/urls.log"; then
|
|
echo "FAIL exact-fork-head: explicit SHA must not be re-resolved through a branch" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
# Platform/repository discovery failures use the same audited CANNOT_ASSERT path.
|
|
audit_lines_before=$(wc -l < "$AUDIT_LOG")
|
|
git -C "$REPO_DIR" remote set-url origin https://gitlab.com/acme/widgets.git
|
|
set +e
|
|
unsupported_output=$(run_guard success "$AUDIT_LOG" 2>&1)
|
|
unsupported_rc=$?
|
|
set -e
|
|
git -C "$REPO_DIR" remote set-url origin https://git.example.test/acme/widgets.git
|
|
if [[ "$unsupported_rc" -ne 0 ]]; then
|
|
echo "FAIL unsupported-platform: expected degraded rc=0, got rc=$unsupported_rc" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$unsupported_output" != *"CANNOT_ASSERT"* ]]; then
|
|
echo "FAIL unsupported-platform: expected loud CANNOT_ASSERT diagnostic" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
audit_lines_after=$(wc -l < "$AUDIT_LOG")
|
|
if [[ "$audit_lines_after" -le "$audit_lines_before" ]]; then
|
|
echo "FAIL unsupported-platform: expected an additional audit record" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
|
|
# A degraded pass is forbidden if the audit receipt cannot be written.
|
|
mkdir -p "$WORK_DIR/not-a-directory"
|
|
printf 'file' > "$WORK_DIR/not-a-directory/parent"
|
|
set +e
|
|
audit_failure_output=$(run_guard unreachable "$WORK_DIR/not-a-directory/parent/audit.jsonl" 2>&1)
|
|
audit_failure_rc=$?
|
|
set -e
|
|
if [[ "$audit_failure_rc" -eq 0 ]]; then
|
|
echo "FAIL audit-unavailable: expected rc!=0, got rc=0" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
if [[ "$audit_failure_output" != *"audit"* ]]; then
|
|
echo "FAIL audit-unavailable: expected loud audit failure diagnostic" >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
assert_provider_observed audit-unavailable
|
|
|
|
if [[ "$failures" -ne 0 ]]; then
|
|
echo "ci-queue-wait tri-state regression failed ($failures assertions)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "ci-queue-wait tri-state regression passed (all outcome classes)"
|