From a8137119efdc5922da7cea782fb80f119d040185 Mon Sep 17 00:00:00 2001 From: be-coder-08 Date: Wed, 5 Aug 2026 16:14:19 -0500 Subject: [PATCH] test(ci): make queue guard harness deterministic Refs #1019 --- docs/scratchpads/1019-zero-timeout-harness.md | 49 +++++++++++++++ .../tools/git/test-ci-queue-wait-tristate.sh | 63 ++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 docs/scratchpads/1019-zero-timeout-harness.md diff --git a/docs/scratchpads/1019-zero-timeout-harness.md b/docs/scratchpads/1019-zero-timeout-harness.md new file mode 100644 index 00000000..3ea6342a --- /dev/null +++ b/docs/scratchpads/1019-zero-timeout-harness.md @@ -0,0 +1,49 @@ +# #1019 — Zero-timeout queue-guard harness race + +- **Issue:** #1019 (parent status remains `believed-fixed, pending jarvis validation`; do not close) +- **Branch:** `fix/1019-ci-queue-timeout-harness` +- **Owner:** `be-coder-08` +- **Base:** `origin/main` at `5916aeefd6ed12bcac086c6834c7f6c4ae38e1bc` +- **Charter:** `/home/hermes/agent-work/tl-mosaic/CHARTER-1019-HARNESS-FIX.md` + +## Objective + +Make `test-ci-queue-wait-tristate.sh` deterministic without changing any asserted outcome. Remove the indiscriminate zero-timeout race, require every status-classification case to prove the provider was observed, and prove the harness-controlled virtual clock is active. + +## Scope + +- In scope: `packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh` only, plus this evidence scratchpad. +- Out of scope: guard parsers, D2/D3 behavior, installer/reseed staleness, PR #1060, and issue closure. + +## Acceptance criteria + +1. RED deterministically reproduces deadline pre-emption before the provider call. +2. Every case that intends status classification positively proves provider observation. +3. Pending observes `pending` before deterministic virtual-time expiration. +4. The virtual clock has a positive interception control; a broken-clock mutant makes the suite red. +5. The exact CI-base image passes the final harness repeatedly with zero failures. +6. Baseline gates, independent code/security review, exact-head CI, and coordinator-authorized squash merge pass. + +## Plan + +1. Add deterministic RED instrumentation for the known merge/provider-unreachable pre-emption. +2. Replace global `-t 0` with a nonzero timeout interpreted under an event-driven virtual clock; stub sleep without wall waiting. +3. Add provider-observation and virtual-clock positive controls without changing outcome assertions. +4. Run focused shell checks, repeat in exact CI-base image, baseline gates, and independent reviews. +5. Commit with both identity layers, queue-guard plus direct Woodpecker terminal-state verification, push, self-post PR, verify poster/head/CI, obtain coordinator merge authorization, then squash merge without closing #1019. + +## Budget + +- No explicit token cap supplied. Keep scope to one harness file and one scratchpad; stop/report at the charter's 60% context gate. + +## Evidence + +- RED, deterministic pre-provider expiry: `evidence/1019-harness-fix/red-pre-provider-expiry.log` — rc 1; merge/provider-unreachable got rc 124 instead of 75, omitted CANNOT_ASSERT, did not observe the status provider, and wrote no additional audit record (four named failures). +- GREEN host focused harness: `evidence/1019-harness-fix/green-host.log` — rc 0, all outcome classes passed. +- Load-bearing clock negative control: a temporary same-directory mutant replaced the virtual `date` body with `/bin/date`; `evidence/1019-harness-fix/red-clock-not-intercepted.log` — rc 1 with named `virtual clock interception did not run` failures. The mutant file was removed after the run. +- Exact CI-base repeat: `git.mosaicstack.dev/mosaicstack/stack/ci-base:latest`, repository mounted read-only, harness work under container `/tmp`; `evidence/1019-harness-fix/ci-image-repeat/summary.log` — **100 pass / 0 fail / 100 total**. +- Synchronization design: provider-status observation creates the event marker; virtual time is 1000 before the event and 2000 afterward. Pending alone reaches the stubbed no-op sleep and a post-observation deadline check. `-t 1` is numeric subject semantics under virtual time, not a wall-clock synchronization duration. + +## 60% context hold + +Stopped before baseline/review/commit as required by the charter. Remaining: inspect final diff, shell/static/baseline gates, independent code/security review, remediation if any, identity-bound commit/trailer verification, mandatory queue guard plus direct terminal Woodpecker `mosaic` enumeration, push, self-posted PR/provider poster read-back, exact-head terminal-green CI, coordinator merge authorization, squash merge, main CI verification, and leave #1019 unclosed as `believed-fixed, pending jarvis validation`. diff --git a/packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh b/packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh index 35ea0aee..1eaa076a 100644 --- a/packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh +++ b/packages/mosaic/framework/tools/git/test-ci-queue-wait-tristate.sh @@ -9,6 +9,8 @@ WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/ci-queue-wait-tristate} REPO_DIR="$WORK_DIR/repo" STUB_DIR="$WORK_DIR/stubs" AUDIT_LOG="$WORK_DIR/audit/ci-queue-wait.jsonl" +STATUS_OBSERVED="$WORK_DIR/status-observed" +CLOCK_LOG="$WORK_DIR/clock.log" FEATURE_BRANCH="fix/rm-03-fixture" rm -rf "$WORK_DIR" @@ -44,6 +46,7 @@ case "$url" in fi ;; */status) + : > "${MOSAIC_STUB_STATUS_OBSERVED:?}" case "${MOSAIC_STUB_STATUS_MODE:?}" in success) printf '%s' '{"state":"success","statuses":[{"status":"success"}]}' ;; pending) printf '%s' '{"state":"pending","statuses":[{"status":"pending","context":"ci/test"}]}' ;; @@ -63,7 +66,31 @@ case "$url" in *) echo "unexpected curl URL: $url" >&2; exit 2 ;; esac SH -chmod +x "$STUB_DIR/curl" + +cat > "$STUB_DIR/date" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 1 || "$1" != "+%s" ]]; then + echo "unexpected date invocation: $*" >&2 + exit 2 +fi + +if [[ -e "${MOSAIC_STUB_STATUS_OBSERVED:?}" ]]; then + printf 'date-phase=after-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}" + printf '2000\n' +else + printf 'date-phase=before-status\n' >> "${MOSAIC_STUB_CLOCK_LOG:?}" + printf '1000\n' +fi +SH + +cat > "$STUB_DIR/sleep" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +printf 'sleep-after-status=%s\n' "$*" >> "${MOSAIC_STUB_CLOCK_LOG:?}" +SH +chmod +x "$STUB_DIR/curl" "$STUB_DIR/date" "$STUB_DIR/sleep" run_guard() { local status_mode="$1" @@ -83,13 +110,36 @@ run_guard() { export GITEA_URL=https://git.example.test export MOSAIC_STUB_STATUS_MODE="$status_mode" fi + rm -f "$STATUS_OBSERVED" "$CLOCK_LOG" export MOSAIC_STUB_URL_LOG="$WORK_DIR/urls.log" + export MOSAIC_STUB_STATUS_OBSERVED="$STATUS_OBSERVED" + export MOSAIC_STUB_CLOCK_LOG="$CLOCK_LOG" export MOSAIC_CI_QUEUE_AUDIT_LOG="$audit_log" - "$SCRIPT_DIR/ci-queue-wait.sh" --purpose "${MOSAIC_TEST_PURPOSE:-push}" -t 0 -i 0 "$@" + # Provider observation is the synchronization event. The one-second + # timeout is subject semantics under virtual time, never a wall wait. + "$SCRIPT_DIR/ci-queue-wait.sh" --purpose "${MOSAIC_TEST_PURPOSE:-push}" -t 1 -i 1 "$@" ) } failures=0 +assert_provider_observed() { + local name="$1" require_expiration="${2:-0}" + if [[ ! -e "$STATUS_OBSERVED" ]]; then + echo "FAIL $name: status provider was not observed" >&2 + failures=$((failures + 1)) + fi + if [[ ! -s "$CLOCK_LOG" ]] || ! grep -q '^date-phase=before-status$' "$CLOCK_LOG"; then + echo "FAIL $name: virtual clock interception did not run before provider observation" >&2 + failures=$((failures + 1)) + fi + if [[ "$require_expiration" -eq 1 ]]; then + if ! grep -q '^sleep-after-status=' "$CLOCK_LOG" || ! grep -q '^date-phase=after-status$' "$CLOCK_LOG"; then + echo "FAIL $name: pending path did not expire after provider observation" >&2 + failures=$((failures + 1)) + fi + fi +} + run_assertion() { local name="$1" expected_rc="$2" status_mode="$3" required_text="$4" local output rc @@ -124,6 +174,13 @@ run_assertion() { printf '%s\n' "$output" >&2 failures=$((failures + 1)) fi + if [[ "$status_mode" != "credential-unresolvable" ]]; then + if [[ "$status_mode" == "pending" ]]; then + assert_provider_observed "$name" 1 + else + assert_provider_observed "$name" + fi + fi } set -e @@ -160,6 +217,7 @@ if [[ "$merge_unreachable_output" != *"CANNOT_ASSERT"* ]]; then echo "FAIL merge-provider-unreachable: expected loud CANNOT_ASSERT diagnostic" >&2 failures=$((failures + 1)) fi +assert_provider_observed merge-provider-unreachable merge_audit_lines_after=$(wc -l < "$AUDIT_LOG") if [[ "$merge_audit_lines_after" -le "$merge_audit_lines_before" ]]; then echo "FAIL merge-provider-unreachable: expected an additional audit record" >&2 @@ -223,6 +281,7 @@ if [[ "$audit_failure_output" != *"audit"* ]]; then echo "FAIL audit-unavailable: expected loud audit failure diagnostic" >&2 failures=$((failures + 1)) fi +assert_provider_observed audit-unavailable if [[ "$failures" -ne 0 ]]; then echo "ci-queue-wait tri-state regression failed ($failures assertions)" >&2