From e2ec927b1c931686644eb7462b4d85ededb47859 Mon Sep 17 00:00:00 2001 From: "jason.woltje" Date: Sun, 26 Jul 2026 06:02:02 +0000 Subject: [PATCH] fix(wake): #908 unify observed_seq on a single store-side allocator (dissolve detector-private-counter seam) (#915) Co-authored-by: jason.woltje Co-committed-by: jason.woltje --- .../framework/tools/wake/_wake-common.sh | 45 ++++++ .../mosaic/framework/tools/wake/detector.sh | 64 ++++---- .../mosaic/framework/tools/wake/manifest.txt | 21 ++- .../mosaic/framework/tools/wake/reconcile.sh | 127 ++++++---------- packages/mosaic/framework/tools/wake/store.sh | 117 ++++++++++++--- .../tools/wake/test-wake-detector.sh | 68 ++++++++- .../tools/wake/test-wake-reconcile.sh | 60 +++++--- .../tools/wake/test-wake-store-ack.sh | 142 ++++++++++++++++++ 8 files changed, 487 insertions(+), 157 deletions(-) diff --git a/packages/mosaic/framework/tools/wake/_wake-common.sh b/packages/mosaic/framework/tools/wake/_wake-common.sh index 68e701ad..4fad0df2 100755 --- a/packages/mosaic/framework/tools/wake/_wake-common.sh +++ b/packages/mosaic/framework/tools/wake/_wake-common.sh @@ -65,6 +65,51 @@ _atomic_write() { fi } +# --------------------------------------------------------------------------- +# Enqueue serialization (single store-side observed_seq allocator, #908). +# +# store.sh is the SOLE allocator of observed_seq: it reads the observed_seq +# cursor, computes next = observed_seq + 1, and writes the pending record + +# observed.set + the cursor as ONE transaction. That read-modify-write MUST be +# serialized so two concurrent enqueues cannot read the same cursor and collide +# on a seq. These helpers take an exclusive lock for the duration of the +# transaction. flock (Linux/CI) is the load-bearing mechanism; where flock is +# absent the open still succeeds so a single-threaded enqueue is unaffected (the +# concurrency guarantee then degrades — the concurrency test SKIPs without flock, +# exactly as the detector's single-instance test already does). +# +# The lock uses a FIXED fd (8) within one store.sh process. Each store.sh +# invocation is its own process (the detector/reconciler call it as a +# subprocess), so fd 8 is always free here and never clashes with the detector +# run-loop lock (fd 9, a DIFFERENT process). +# --------------------------------------------------------------------------- +_wake_lock_acquire() { + # _wake_lock_acquire LOCKFILE — open fd 8 on LOCKFILE and take an exclusive + # (blocking) lock. Returns non-zero if the lock cannot be taken. + local lf="$1" dir + dir="$(dirname "$lf")" + [ -d "$dir" ] || mkdir -p "$dir" 2>/dev/null || true + # Open the lock fd. If the file does not yet exist and the dir is writable it + # is created; if the dir is read-only but the file exists, opening it O_WRONLY + # still succeeds (write perm on the file, not the dir). + # NB: a command-less `exec` redirection persists for the WHOLE shell, so we must + # NOT append `2>/dev/null` here (it would permanently silence the caller's + # stderr and swallow every later fail-loud diagnostic). Redirect only fd 8. + exec 8>"$lf" || return 1 + if command -v flock >/dev/null 2>&1; then + flock 8 || return 1 + fi + return 0 +} + +_wake_lock_release() { + # _wake_lock_release — drop the enqueue lock (closing fd 8 releases the flock). + # The `2>/dev/null` is SCOPED to the brace group (suppressing a "bad fd" close + # error) — it must NOT sit on a bare `exec`, where the redirection would persist + # for the whole shell and silence every later fail-loud diagnostic. + { exec 8>&-; } 2>/dev/null || true +} + # _wake_clean_stale_tmp DIR — remove leftover atomic-write temp files (e.g. from # a crash mid-write). Safe: these are never the live store. _wake_clean_stale_tmp() { diff --git a/packages/mosaic/framework/tools/wake/detector.sh b/packages/mosaic/framework/tools/wake/detector.sh index 04344565..493de771 100755 --- a/packages/mosaic/framework/tools/wake/detector.sh +++ b/packages/mosaic/framework/tools/wake/detector.sh @@ -9,11 +9,14 @@ # proven delta-gated poller (gitea-pr-watch: poll -> hash -> deliver only # on delta; 0-wasted) and adds (a) repo-section/anchor-scoped hashing so # human-decision FILE edits are caught, not just API-visible state. -# §1.2 observe -> assign observed_seq -> enqueue into the W2 durable store. -# §2.4 Cursor semantics: `observed_seq` is a detector-local monotonic int -# assigned AT OBSERVATION and is AUTHORITATIVE; source SHAs are -# DESCRIPTORS, not the cursor. A per-watch LAST-OBSERVED HASH is compared -# each poll so a revert (A->B->A across polls) is caught as a delta. +# §1.2 observe -> enqueue into the W2 durable store (the store ALLOCATES the +# observed_seq — single store-side allocator, #908). +# §2.4 Cursor semantics: `observed_seq` is a monotonic int, AUTHORITATIVE, now +# allocated SOLELY by store.sh enqueue (#908 unified the two former +# allocators). The detector no longer keeps a private counter: it enqueues +# and captures the store-assigned seq. Source SHAs are DESCRIPTORS, not the +# cursor. A per-watch LAST-OBSERVED HASH is compared each poll so a revert +# (A->B->A across polls) is caught as a delta. # §4/G2a FAIL-LOUD source semantics: a source/target failure OR a 401 / 403 / # privacy-404 / partial / ambiguous-empty response MUST fail loud, MUST # NOT advance the authoritative cursor (`observed_seq`), and MUST NOT be @@ -100,8 +103,9 @@ Commands: validate Load + validate the watch-list and its schema_version against the manifest; exit 0 if usable, non-zero (loud) if not. - cursors Print the detector-local observed_seq - counter and the store cursors. + cursors Print the store cursors (observed_seq is + the store's single source of truth, #908; + the detector keeps no private counter). Environment: WAKE_WATCH_LIST Path to the operator watch-list JSON (required). @@ -171,19 +175,15 @@ _load_watchlist() { printf '%s' "$json" } -# --- detector-local observed_seq (§2.4, authoritative monotonic int) -------- - -# _next_observed_seq — atomically increment + return the detector-local counter. -# Assigned AT OBSERVATION (only on a real delta). A single global monotonic -# sequence across ALL watches (source SHAs are descriptors, NOT the cursor). -_next_observed_seq() { - mkdir -p "$DET_DIR" - local cur nxt - cur="$(_wake_read_int "$DET_DIR/observed_seq_counter" 0)" - nxt=$((cur + 1)) - printf '%s' "$nxt" | _atomic_write "$DET_DIR/observed_seq_counter" - printf '%s' "$nxt" -} +# --- observed_seq: the store is the SOLE allocator (#908) ------------------- +# +# The detector NO LONGER keeps a private observed_seq counter. That private +# counter was the shared root of three defects (burn-before-enqueue, W5 +# dual-allocator aliasing, and the migration-restart silent-swallow); see the PR +# for #908. observed_seq is now allocated EXCLUSIVELY by store.sh enqueue (which +# reads its own cursor and returns the assigned seq). The detector calls enqueue +# WITHOUT --seq and captures the store-returned seq for logging/locators — it +# never allocates a seq itself, so the seam is gone. # --- per-watch last-observed hash (§2.4 revert/ABA-at-rest detection) -------- @@ -274,10 +274,11 @@ _poll_source() { return 0 fi - # --- DELTA: assign the next observed_seq AT OBSERVATION and enqueue -------- - # (a revert A->B->A lands here because $h != the stored $last). - local seq locators emit_ts - seq="$(_next_observed_seq)" + # --- DELTA: enqueue and let the STORE allocate observed_seq (#908) --------- + # (a revert A->B->A lands here because $h != the stored $last). The store is + # the sole allocator: we call enqueue WITHOUT --seq and capture the seq it + # assigns (for logging/locators/emit). We do NOT allocate a seq ourselves. + local locators emit_ts emit_ts="$(date +%s)" # NB: `def` is a reserved word in jq — the source-definition arg is `$sdef`. locators="$(jq -cn \ @@ -288,16 +289,19 @@ _poll_source() { '{kind:$kind, id:$id, observed_hash:$hash} + ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')" - local args=(enqueue --seq "$seq" --locators "$locators" --emit-ts "$emit_ts") + local args=(enqueue --locators "$locators" --emit-ts "$emit_ts") # Pass class through only when the source declares one; absent => the store # defaults to `actionable` (fail-safe, §2.3). Never guess a coalescible class. [ -n "$class" ] && args+=(--class "$class") - if ! "$STORE_SH" "${args[@]}" >/dev/null; then - echo "detector.sh: store enqueue FAILED for '$kind/$id' seq $seq" >&2 + local seq + if ! seq="$("$STORE_SH" "${args[@]}")"; then + echo "detector.sh: store enqueue FAILED for '$kind/$id' (observed_seq NOT advanced; hash NOT advanced)" >&2 return 1 fi # Only AFTER a durable enqueue do we advance the per-watch last-observed hash. + # (store-allocated seq $seq is available for logging/emit if needed.) + : "$seq" printf '%s' "$h" | _atomic_write "$hf" return 0 } @@ -420,9 +424,9 @@ cmd_run() { } cmd_cursors() { - local counter - counter="$(_wake_read_int "$DET_DIR/observed_seq_counter" 0)" - printf 'detector_observed_seq=%s\n' "$counter" + # observed_seq has a SINGLE source of truth now (#908): the STORE cursor. The + # detector keeps no private counter, so cursors simply reports the store's + # authoritative cursors (observed_seq / consumed_seq / pending_depth). "$STORE_SH" cursors 2>/dev/null || true } diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index 5debfecf..b65d5826 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -24,8 +24,18 @@ # alarm-target/HMAC-key install-validation. Also folds in the two W6 # monitor-integration observations (monitor-side ingested_ts # staleness + beacon HMAC-verify at record). +# 0.6.1 #908 — UNIFY observed_seq on a SINGLE store-side allocator. store.sh +# enqueue is now the sole allocator (reads its own cursor, next=+1 +# under an exclusive lock, prints the seq; commits IFF the durable +# write succeeds). The detector-private observed_seq_counter and its +# --seq hand-off are DELETED; the reconciler enumerates via the same +# store allocator (its dual-allocator fail-closed guard retired). This +# dissolves the three defects rooted in the private-counter seam: +# burn-before-enqueue (arrow 1), W5 co-feed aliasing (arrow 2), and +# the migration-restart silent-swallow (arrow 3, now structurally +# impossible — allocation is always > consumed or fails loud). component=wake -version=0.6.0 +version=0.6.1 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -43,8 +53,9 @@ schema_max=1 # sign.sh A5 — non-circular HMAC signer (independent wake_id, # load_credentials by-name; fills the hmac placeholder). (W3) # detector.sh A1 — per-host single-instance delta-gated detector daemon -# (flock, anchor-scoped hashing, detector-local observed_seq, -# fail-loud source semantics; enqueues deltas to store.sh). (W4) +# (flock, anchor-scoped hashing, fail-loud source semantics; +# enqueues deltas to store.sh and captures the store-allocated +# observed_seq — no private counter, #908). (W4) # fn-oracle.sh A6 — synthetic-canary FN-oracle: injects a KNOWN delta at the # source boundary, drives the pipeline through the detector's # public poll-once, asserts CONSUMED within the per-class SLO @@ -53,7 +64,9 @@ schema_max=1 # reconcile.sh A7 — source-parity reconciler: (i) source-coverage parity # inventory (an omitted source cannot pass the vector # vacuously) + (ii) periodic full reconcile to 0-unaccounted, -# enumerating pre-existing/startup state into the store. (W5) +# enumerating pre-existing/startup state into the store via the +# SINGLE store-side allocator (co-feed is safe; the former +# dual-allocator fail-closed guard retired, #908). (W5) # beacon.sh A8 — off-host DEAD-MAN liveness beacon: a monotonic beacon # EMITTER (emit — the primitive the detector run-loop calls # each cycle), the off-host monitor's RECEIVER + beacon-ABSENCE diff --git a/packages/mosaic/framework/tools/wake/reconcile.sh b/packages/mosaic/framework/tools/wake/reconcile.sh index 0bc1beb3..e13a9a95 100755 --- a/packages/mosaic/framework/tools/wake/reconcile.sh +++ b/packages/mosaic/framework/tools/wake/reconcile.sh @@ -29,23 +29,24 @@ # silently "no state". # # CONTRACT NOTES (flagged, not silently guessed — see PR body): -# * observed_seq DUAL-ALLOCATOR HAZARD (fail-closed; #908 tracks the fix). -# observed_seq has TWO INDEPENDENT allocators today: the detector's PRIVATE -# counter ($STATE_DIR/detector/observed_seq_counter, W4) and — if this -# reconciler enumerated — the STORE cursor (observed_seq+offset). Because the -# detector's next seq is (private_counter+1), NOT (store_observed_seq+1), the -# two are decoupled and SERIALIZING THEM DOES NOT PREVENT COLLISION: after -# the reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2, -# detector private still 0), the detector's next delta allocates private -# 0->1 and enqueues seq1 — ALIASING alpha. A consumer acking CONSUMED 1 -# (trusting §2.4's unique observed_seq) then silently DROPS a distinct -# obligation — the exact G3 swallow this reconciler exists to prevent. -# THEREFORE, until #908 unifies on a single store-side allocator, the -# reconciler FAILS CLOSED: it REFUSES to enumerate (loud, non-zero; the -# obligation is FLAGGED, never silently swallowed) whenever a detector is -# co-feeding this store, and otherwise requires the operator to assert -# reconciler-SOLE-FEEDER mode. OPERATIONAL GUARD: a detector and this -# reconciler MUST NOT co-feed one store until #908 lands. +# * observed_seq DUAL-ALLOCATOR HAZARD — RESOLVED by #908 (single store-side +# allocator). HISTORY: observed_seq once had TWO INDEPENDENT allocators — the +# detector's PRIVATE counter (W4) and the STORE cursor (observed_seq+offset) +# this reconciler used to enumerate from. Because the detector's next seq was +# (private_counter+1), NOT (store_observed_seq+1), the two were decoupled and +# even serializing them did NOT prevent collision: the reconciler enumerating +# alpha->seq1, beta->seq2 while the detector's private counter was still 0 +# meant the detector's next delta re-allocated seq1, ALIASING alpha, and a +# consumer acking CONSUMED 1 silently dropped a distinct obligation. +# #908 DISSOLVES this at the root: store.sh enqueue is now the SOLE allocator +# (reads its own observed_seq cursor, next=+1, under an exclusive lock). This +# reconciler now enumerates by calling store.sh enqueue WITHOUT --seq — the +# SAME single allocator the detector uses — so co-feeding one store is SAFE: +# both paths draw distinct, contiguous seqs from the one cursor. The former +# fail-closed refusal (and the reconciler-SOLE-FEEDER assertion) are therefore +# RETIRED; --allow-enumerate / WAKE_RECONCILE_ALLOW_ENUMERATE remain accepted +# as deprecated no-ops for caller compatibility. The G3 accounting invariants +# (0-UNACCOUNTED, vacuous-pass prevention) are unaffected and still enforced. # * "accounted" is judged against observed_seq/inbox (§4/G3 wording) PLUS the # reconciler's OWN durable reconciled-state ledger — NOT the detector's # hash-file. Judging by the detector baseline would let a silent first-seen @@ -67,13 +68,6 @@ STATE_DIR="$(wake_state_dir)" # Reconciler-local state (its durable reconciled-state ledger) lives in its own # subdir under the store's STATE_DIR so it never collides with store/detector. RECON_DIR="$STATE_DIR/reconciler" -# The detector's PRIVATE state dir (W4). Its presence signals a detector is (or -# has been) allocating observed_seq into THIS store from an INDEPENDENT private -# counter — the co-feeding condition under which reconciler enumeration is -# unsafe (see the dual-allocator hazard note above; #908). Read-only coupling: -# reconcile.sh never writes or imports W4, it only OBSERVES this path to fail -# closed rather than silently alias a seq. -DET_STATE="$STATE_DIR/detector" _need_jq() { command -v jq >/dev/null 2>&1 || { @@ -119,16 +113,14 @@ Commands: ENUMERATED into the durable store (the startup/pre-existing obligation path). Exit 0 IFF 0 unaccounted were FOUND this pass. - DUAL-ALLOCATOR SAFETY (fail-closed; #908): enumeration WRITES observed_seq - from an allocator INDEPENDENT of the detector's private counter, so it - FAILS LOUD (refuses; the obligation is FLAGGED, never swallowed) when it - would risk aliasing a seq: - * if a detector is co-feeding this store (its private state dir exists) - => refuse, ALWAYS (even with the ack below); - * else the operator MUST assert reconciler-SOLE-FEEDER mode via - --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1), promising no - detector co-feeds this store until #908 lands. - --no-enumerate (and `check`) never write, so they are always safe. + SINGLE STORE-SIDE ALLOCATOR (#908, resolved): enumeration allocates + observed_seq by calling store.sh enqueue WITHOUT --seq — the SAME sole + allocator the detector uses (store cursor +1 under an exclusive lock). A + detector co-feeding this store is therefore SAFE: both draw distinct, + contiguous seqs from the one cursor, so no aliasing is possible. The old + fail-closed refusal is retired; --allow-enumerate (and + WAKE_RECONCILE_ALLOW_ENUMERATE=1) are accepted as deprecated no-ops. + --no-enumerate (and `check`) never write, so they remain side-effect-free. check The full §4/G3 gate, side-effect-free: `inventory` AND @@ -141,8 +133,9 @@ Environment: contract as the detector: , def on stdin; exit!=0 or empty => FAIL LOUD). WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh). - WAKE_RECONCILE_ALLOW_ENUMERATE set to 1 to assert reconciler-SOLE-FEEDER mode - (see reconcile's dual-allocator safety note; #908). + WAKE_RECONCILE_ALLOW_ENUMERATE DEPRECATED no-op (#908 made co-feeding safe via + the single store-side allocator); accepted for + backward compatibility, no longer required. EOF } @@ -404,30 +397,14 @@ cmd_reconcile() { "$STORE_SH" init >/dev/null 2>&1 || true mkdir -p "$RECON_DIR" - # --- DUAL-ALLOCATOR SAFETY GATE (fail-closed; #908) ------------------------ - # Decide, ONCE, whether ENUMERATION (the observed_seq WRITE path) is safe this - # pass. It is unsafe — and we REFUSE (flag, never swallow) — whenever a seq we - # allocate could be REISSUED by the detector's INDEPENDENT private counter and - # alias a distinct obligation (see the header hazard note). --no-enumerate and - # `check` never write, so the gate does not apply to them. - local enum_mode="$enumerate" enum_refuse_reason="" - if [ "$enumerate" -eq 1 ]; then - if [ -e "$DET_STATE" ]; then - enum_mode=0 - enum_refuse_reason="a detector is co-feeding this store ($DET_STATE exists); its private observed_seq counter (W4) is INDEPENDENT of the store cursor this reconciler allocates from, so enumerating would risk REISSUING/ALIASING a seq and silently swallowing a distinct obligation. #908 tracks the store-side allocator unification that makes co-feeding safe. OPERATIONAL GUARD: do not run a detector and the reconciler against one store until #908." - elif [ "$allow_enumerate" -ne 1 ]; then - enum_mode=0 - enum_refuse_reason="enumeration writes observed_seq via an allocator INDEPENDENT of the detector's private counter (W4); until #908 unifies them, co-feeding one store aliases seqs even under strictly-sequential calls. Set --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1) to assert this store is reconciler-SOLE-FEEDER (no detector co-feeds it)." - fi - fi - - # Seq allocation base: the store's authoritative observed_seq (§2.4). Used ONLY - # on the sole-feeder path the gate above has proven safe. - local obs_base offset - obs_base="$("$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p')" - case "$obs_base" in '' | *[!0-9]*) obs_base=0 ;; esac - offset=0 - local refuse_announced=0 refused=0 + # --- SINGLE STORE-SIDE ALLOCATOR (#908) ----------------------------------- + # Enumeration allocates observed_seq by calling store.sh enqueue WITHOUT --seq + # (the sole allocator the detector also uses). Co-feeding one store is safe — + # both paths draw distinct, contiguous seqs from the one store cursor under its + # exclusive lock — so the former dual-allocator fail-closed refusal is retired. + # allow_enumerate is now a deprecated no-op, retained only so old callers that + # pass --allow-enumerate / WAKE_RECONCILE_ALLOW_ENUMERATE=1 still work. + : "$allow_enumerate" local pairs kind id failed=0 unaccounted=0 enumerated=0 pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')" @@ -478,34 +455,24 @@ cmd_reconcile() { unaccounted=$((unaccounted + 1)) printf ' %s/%s: UNACCOUNTED (current state not reflected in observed_seq/inbox)\n' "$kind" "$id" >&2 - if [ "$enumerate" -eq 1 ] && [ "$enum_mode" -eq 0 ]; then - # REFUSE to enumerate (fail-closed): the obligation is loudly FLAGGED, NOT - # silently swallowed. Announce the dual-allocator reason once. - if [ "$refuse_announced" -eq 0 ]; then - echo "reconcile.sh: FAIL LOUD (dual-allocator hazard, #908) — $enum_refuse_reason" >&2 - refuse_announced=1 - fi - refused=1 - printf ' -> REFUSED enumeration (FLAGGED, not swallowed; see dual-allocator hazard / #908)\n' >&2 - elif [ "$enum_mode" -eq 1 ]; then - # ENUMERATE the pre-existing/startup obligation into the durable store. - # This path runs ONLY when the safety gate proved the reconciler is the - # SOLE feeder (no detector co-feeding), so its store-cursor allocation - # cannot be aliased. Class = non-coalescible fail-safe `actionable` (§2.3) + if [ "$enumerate" -eq 1 ]; then + # ENUMERATE the pre-existing/startup obligation into the durable store via + # the SINGLE store-side allocator (#908): store.sh enqueue WITHOUT --seq + # allocates the next observed_seq under the store's lock and PRINTS it. Both + # the detector and this reconciler draw from that one cursor, so a co-fed + # store cannot alias. Class = non-coalescible fail-safe `actionable` (§2.3) # so distinct pre-existing obligations never collapse together. - offset=$((offset + 1)) local seq locators - seq=$((obs_base + offset)) locators="$(jq -cn --arg kind "$kind" --arg id "$id" --arg hash "$curhash" \ --argjson sdef "$def" \ '{kind:$kind, id:$id, observed_hash:$hash, reconciled:true} + ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')" - if "$STORE_SH" enqueue --seq "$seq" --class actionable --locators "$locators" >/dev/null 2>&1; then + if seq="$("$STORE_SH" enqueue --class actionable --locators "$locators")"; then printf '%s' "$curhash" | _atomic_write "$ledger" enumerated=$((enumerated + 1)) printf ' -> enumerated into store as observed_seq=%s (class=actionable)\n' "$seq" else - echo "reconcile.sh: enumerate FAILED for '$kind/$id' seq $seq" >&2 + echo "reconcile.sh: enumerate FAILED for '$kind/$id'" >&2 failed=1 fi fi @@ -519,9 +486,7 @@ EOF return 1 fi if [ "$unaccounted" -ne 0 ]; then - if [ "$refused" -eq 1 ]; then - echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) REFUSED enumeration (dual-allocator hazard, #908). They are flagged, NOT swallowed. Resolve co-feeding / assert sole-feeder, then re-run." >&2 - elif [ "$enumerate" -eq 1 ]; then + if [ "$enumerate" -eq 1 ]; then echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) found and enumerated. Re-run to confirm 0." >&2 else echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) (audit mode; nothing enumerated)." >&2 diff --git a/packages/mosaic/framework/tools/wake/store.sh b/packages/mosaic/framework/tools/wake/store.sh index f4577152..066d9571 100755 --- a/packages/mosaic/framework/tools/wake/store.sh +++ b/packages/mosaic/framework/tools/wake/store.sh @@ -8,8 +8,11 @@ # §2.4 cursor semantics: observed_seq is authoritative; SHAs are descriptors. # # Three cursors: -# observed_seq (detector-owned; this lib RECORDS/stores it) — monotonic int -# assigned at observation. Authoritative. Source SHAs are +# observed_seq (store-owned SINGLE ALLOCATOR, #908) — monotonic int assigned by +# THIS lib at enqueue: next = observed_seq + 1, under an exclusive +# lock, committed IFF the durable write succeeds. Authoritative. +# There is exactly ONE allocator (this store); the detector and +# reconciler no longer allocate independently. Source SHAs are # descriptors, NOT the cursor (§2.4). # pending-inbox (durable) — append store; retains until CONSUMED; survives # pane death / park / restart. @@ -48,9 +51,23 @@ Usage: store.sh [options] Commands: init Create/repair the XDG state layout. - enqueue --seq N --class C [opts] Record an observation into the durable - pending-inbox (coalesce or append). - --seq N observed_seq (detector-assigned monotonic int). Required. + enqueue [--class C] [opts] Record an observation into the durable + pending-inbox (coalesce or append) and + ALLOCATE its observed_seq. store.sh is the + SOLE allocator of observed_seq (#908): + under an exclusive enqueue lock it reads + the observed_seq cursor, sets + next = observed_seq + 1, writes the + pending record + observed.set + the cursor + as ONE transaction, and PRINTS next to + stdout. The cursor advances IFF the durable + write succeeds (no burn-before-enqueue). + --seq N LEGACY explicit observed_seq (optional). Normal callers + (detector, reconciler) OMIT this and let the store + allocate. Retained only for tests/tools that must place a + SPECIFIC seq (e.g. construct a gap). An explicit seq that + is <= consumed_seq is REFUSED loudly (anti-swallow): the + idempotent-ignore path can never silently drop a wake. --class C digest|actionable|human|terminal-log|reaction. Absent/empty => actionable (fail-safe). --locators JSON JSON value for the entry's locators (default {}). @@ -109,12 +126,18 @@ cmd_enqueue() { esac done - case "$seq" in - '' | *[!0-9]*) - echo "store.sh enqueue: --seq must be a non-negative integer" >&2 - exit 2 - ;; - esac + # --seq is OPTIONAL (#908): normal callers omit it and the store allocates. + # If supplied (legacy/explicit), it must be a non-negative integer. + local explicit=0 + if [ -n "$seq" ]; then + explicit=1 + case "$seq" in + *[!0-9]*) + echo "store.sh enqueue: --seq must be a non-negative integer" >&2 + exit 2 + ;; + esac + fi # Absent class => actionable (fail-safe, §2.3). [ -n "$class" ] || class="actionable" @@ -142,13 +165,42 @@ cmd_enqueue() { _wake_init_dir "$STATE_DIR" - local consumed - consumed="$(_wake_read_int "$STATE_DIR/consumed_seq" 0)" + # --- serialize the whole allocate+write transaction (#908) ---------------- + # A single store-side allocator means the read of observed_seq, the durable + # write, and the cursor bump MUST be one critical section: two concurrent + # enqueues that both read observed_seq=N would otherwise both allocate N+1 and + # collide. The lock (flock where present) makes concurrent enqueues get + # DISTINCT seqs. + _wake_lock_acquire "$STATE_DIR/.enqueue.lock" || { + echo "store.sh enqueue: could not acquire the enqueue lock ($STATE_DIR/.enqueue.lock)" >&2 + exit 1 + } - # An observation already inside the consumed prefix is idempotently ignored - # (§1.2: cursor never regresses; a re-observed already-consumed seq is a no-op). + local consumed observed + consumed="$(_wake_read_int "$STATE_DIR/consumed_seq" 0)" + observed="$(_wake_read_int "$STATE_DIR/observed_seq" 0)" + + if [ "$explicit" -eq 0 ]; then + # ALLOCATE: the store is the sole allocator. next = observed_seq + 1. + seq=$((observed + 1)) + fi + + # --- anti-swallow FAIL-LOUD (#908, arrow #3) ------------------------------ + # observed_seq >= consumed_seq always (consumed advances only over the gapless + # observed prefix), so an ALLOCATED next = observed+1 is ALWAYS > consumed. The + # old "seq <= consumed => idempotent no-op" path is therefore UNREACHABLE for an + # allocation; if it is ever reached the store is inconsistent (corruption) — we + # FAIL LOUD, never a silent no-op. For a LEGACY explicit --seq, a seq inside the + # consumed prefix is likewise refused loudly so it can never silently swallow a + # fresh obligation (the migration-restart killer must be structurally impossible). if [ "$seq" -le "$consumed" ]; then - return 0 + _wake_lock_release + if [ "$explicit" -eq 1 ]; then + echo "store.sh enqueue: REFUSED explicit --seq $seq <= consumed_seq $consumed — anti-swallow (#908): a seq inside the consumed prefix is never a fresh obligation and must NEVER be silently ignored." >&2 + else + echo "store.sh enqueue: FAIL LOUD (#908 allocation invariant) — computed observed_seq $seq <= consumed_seq $consumed (observed_seq cursor=$observed). The store is inconsistent; refusing to allocate rather than silently no-op." >&2 + fi + exit 1 fi # Build the durable entry. `hmac` is an UNSIGNED PLACEHOLDER (W3/A5 signs it). @@ -159,6 +211,7 @@ cmd_enqueue() { --argjson locators "$locators" \ --argjson emit_ts "$emit_ts" \ '{observed_seq:$seq, locators:$locators, class:$class, emit_ts:$emit_ts, hmac:""}')" || { + _wake_lock_release echo "store.sh enqueue: failed to build entry" >&2 exit 1 } @@ -168,6 +221,11 @@ cmd_enqueue() { # digest lines, then append the newest). Newest subsumes prior. # others: APPEND (never replaced). # In BOTH cases the entry lands in the durable store. + # + # ORDERING (#908, arrow #1 — no burn-before-enqueue): the durable pending write + # comes FIRST and its failure ABORTS before the observed.set / observed_seq + # cursor writes. The cursor advances IFF the durable pending write succeeds, so + # a failed enqueue can never burn a seq that never reached the store. local new_pending if [ "$class" = "digest" ]; then new_pending="$(jq -c 'select(.class != "digest")' "$STATE_DIR/pending.jsonl" 2>/dev/null || true)" @@ -176,20 +234,35 @@ cmd_enqueue() { new_pending="$(cat "$STATE_DIR/pending.jsonl" 2>/dev/null; printf '%s\n' "$entry")" new_pending="$(printf '%s' "$new_pending" | grep -v '^[[:space:]]*$' || true)" fi - printf '%s\n' "$new_pending" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/pending.jsonl" + if ! printf '%s\n' "$new_pending" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/pending.jsonl"; then + _wake_lock_release + echo "store.sh enqueue: durable pending write FAILED for seq $seq — observed_seq cursor NOT advanced (no burned seq, no interior gap)." >&2 + exit 1 + fi # --- observed.set: record the seq in the live window (gap-detector SoT) ---- - { + if ! { cat "$STATE_DIR/observed.set" 2>/dev/null printf '%s\n' "$seq" - } | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set" + } | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set"; then + _wake_lock_release + echo "store.sh enqueue: observed.set write FAILED for seq $seq — observed_seq cursor NOT advanced." >&2 + exit 1 + fi - # --- observed_seq cursor: max monotonic int ------------------------------- - local observed - observed="$(_wake_read_int "$STATE_DIR/observed_seq" 0)" + # --- observed_seq cursor: max monotonic int (COMMIT of the allocation) ----- + # Written LAST: only now, after the durable pending + observed.set writes + # succeeded, does the cursor advance. This is the atomic commit point of the + # allocation. if [ "$seq" -gt "$observed" ]; then printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq" fi + + _wake_lock_release + + # Print the allocated observed_seq to stdout (sole-allocator contract): the + # detector/reconciler capture it for logging/locators/emit. + printf '%s\n' "$seq" } cmd_drain() { diff --git a/packages/mosaic/framework/tools/wake/test-wake-detector.sh b/packages/mosaic/framework/tools/wake/test-wake-detector.sh index 8b2c0660..343b209e 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-detector.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-detector.sh @@ -13,6 +13,9 @@ # observed_seq NOT advanced (the G2a invariant) (§4/G2a) # D7 anchor-scoped hashing: edit OUTSIDE the anchor is a no-op; # edit INSIDE the anchor is a delta (§1.1 (a)) +# D8 MIGRATION-RESTART: post-migration first delta ALLOCATES a seq +# > consumed via the single store-side allocator — never the +# silent seq<=consumed no-op the old private counter caused (#908 arrow 3) # # Uses FAKE/STUB sources only (no live network). Isolated: every test runs # against a fresh WAKE_STATE_HOME temp dir. @@ -62,7 +65,12 @@ fresh_state() { # depth — pending_depth reported by the store for the current namespace. depth() { "$STORE" cursors | sed -n 's/pending_depth=//p'; } -det_seq() { "$DET" cursors | sed -n 's/detector_observed_seq=//p'; } +# observed_seq is the STORE's single source of truth now (#908): the detector +# keeps no private counter, so `detector.sh cursors` reports the store cursors. +# This asserts the SAME invariant the old detector-private counter did (a delta +# advances observed_seq by exactly one; no-change/first-seen/fail-loud do not) — +# just read from the unified store allocator. +det_seq() { "$DET" cursors | sed -n 's/^observed_seq=//p'; } # write_watchlist FILE VERSION — a minimal valid watch-list with one repo source # ("r1") plus optionally a board_file with an anchor ("b1"). @@ -332,6 +340,64 @@ echo "== D7: anchor-scoped hashing (edit outside anchor = no-op; inside = delta) [ "$(depth)" -ge 1 ] || fail_msg "D7: in-anchor edit must enqueue, got depth $(depth)" ) && ok +echo "== D8: MIGRATION-RESTART — post-migration first delta ALLOCATES > consumed, never a silent no-op (#908, arrow #3) ==" +( + WAKE_STATE_HOME="$(fresh_state d8)" + export WAKE_STATE_HOME + unset WAKE_AGENT + stub="$TMP_ROOT/d8stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + # A watch-list with a SINGLE repo source (clean seq accounting). + wl="$TMP_ROOT/d8.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1", "class": "actionable" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + + # --- simulate adopt -> migrate -> restart -------------------------------- + # §5 migration seeds the STORE cursors (consumed_seq=N, observed_seq=N) so the + # already-consumed prefix is preserved. It seeds NO detector-private counter — + # because after #908 the detector HAS none; the store cursor is the single SoT. + # This is the exact pilot state that used to silently swallow the first delta. + sd="$WAKE_STATE_HOME/default" + mkdir -p "$sd" + printf '5' >"$sd/consumed_seq" + printf '5' >"$sd/observed_seq" + : >"$sd/observed.set" # prefix <=5 fully consumed; live window empty + : >"$sd/pending.jsonl" + + # "Restart" the detector: fresh detector-local state (no hash yet, and — the + # killer precondition — NO private observed_seq counter exists). + printf 'STATE-A\n' >"$stub/repo_r1" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "D8: post-restart baseline pass failed" + # First-seen baselines silently (deliver-on-new); the store cursor is untouched. + [ "$(depth)" = "0" ] || fail_msg "D8: baseline must not enqueue, got depth $(depth)" + [ "$(det_seq)" = "5" ] || fail_msg "D8: migration-seeded observed_seq must be preserved at 5, got $(det_seq)" + + # --- the real, un-consumed obligation: ONE delta ------------------------- + printf 'STATE-B\n' >"$stub/repo_r1" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "D8: post-migration delta pass failed" + # THE KILLER ASSERTION: the store (sole allocator) allocates 6 = observed(5)+1, + # which is > consumed(5). Under the OLD private-counter detector this delta got + # seq 1 (counter restarted at 0) which is <= consumed 5 and was SILENTLY + # swallowed by store enqueue's seq<=consumed no-op (depth stayed 0). With the + # unified allocator it CANNOT be swallowed: it enqueues and WOULD wake. + [ "$(det_seq)" = "6" ] || fail_msg "D8: post-migration first delta must ALLOCATE observed_seq 6 (>consumed 5), got $(det_seq) — a restart-at-0 private counter would give <=5 and be swallowed" + [ "$(depth)" = "1" ] || fail_msg "D8: post-migration delta must ENQUEUE a real obligation (depth 1), NOT be a silent no-op, got depth $(depth)" + # And it is genuinely deliverable (a locator-bearing pending entry the consumer + # would wake on), not swallowed. + entry="$("$STORE" drain)" + echo "$entry" | jq -e 'select(.observed_seq==6 and .locators.kind=="repo" and .locators.id=="r1")' >/dev/null \ + || fail_msg "D8: the enqueued post-migration obligation must be a real deliverable at observed_seq=6 [$entry]" + # Contiguous-prefix contract still holds: CONSUMED 6 succeeds (6 is observed, + # 5 is the consumed prefix — no interior gap). + "$STORE" consume --upto 6 >/dev/null 2>&1 || fail_msg "D8: CONSUMED 6 must succeed over the contiguous prefix after the delta" +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake detector harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 diff --git a/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh index 77d248a0..2735b01d 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh @@ -15,6 +15,10 @@ # R7 a source already reflected in the inbox -> ACCOUNTED (no double- # enumeration of detector-delivered state) (§4/G3-ii) # R8 a source adapter error -> FAIL LOUD (G2a parity; never 'no state') (§4/G2a) +# R9 co-feed NO-ALIAS: detector + reconciler share the ONE store-side +# observed_seq allocator, so co-feeding one store yields DISTINCT, +# contiguous seqs (no aliasing). Formerly the fail-closed dual- +# allocator refusal; #908 dissolves the hazard at the root. (§4/G3, #908) # # Uses FAKE/STUB sources only (no live network). Isolated per test. # shellcheck disable=SC2030,SC2031 @@ -232,7 +236,7 @@ EOF [ "$(depth)" = "0" ] || fail_msg "R8: a failed observation must NOT enumerate anything, got depth $(depth)" ) && ok -echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never silent-alias) ==" +echo "== R9: co-feed NO-ALIAS — detector + reconciler share ONE store allocator; every observed_seq distinct + gapless (#908 resolved) ==" ( WAKE_STATE_HOME="$(fresh_state r9)" export WAKE_STATE_HOME @@ -241,7 +245,7 @@ echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never mkdir -p "$stub" make_stub "$stub" export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" - # Pre-existing sources present at startup (the reviewer's alpha/beta repro). + # The reviewer's alpha/beta repro (formerly the dual-allocator ALIASING case). printf 'ALPHA\n' >"$stub/repo_alpha" printf 'BETA\n' >"$stub/repo_beta" wl="$TMP_ROOT/r9.json" @@ -251,26 +255,44 @@ echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "alpha" }, { "kind": "repo", "id": "beta" } ] } ] } EOF export WAKE_WATCH_LIST="$wl" - # A DETECTOR is co-feeding this store: baseline it (creates the detector's - # private-counter state dir). The detector's private observed_seq allocator is - # now live and INDEPENDENT of the store cursor — the exact co-feeding hazard. + + # A DETECTOR is co-feeding this store. Under #908 both the detector and the + # reconciler allocate observed_seq from the ONE store cursor (store.sh enqueue, + # no --seq), so co-feeding is now SAFE — the reconciler ENUMERATES (no refusal), + # and no seq can be aliased. (The old guard failed closed here; that refusal is + # retired exactly to the extent the single allocator makes safe.) "$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector baseline failed" - # The pre-existing state is unaccounted (baseline is silent). Enumerating it - # here would allocate a store-cursor seq the detector's private counter can - # later REISSUE and alias — the serialized collision. The reconciler MUST - # refuse (fail loud), NOT enqueue a collidable seq. - before="$(depth)" + # A real detector delta on alpha -> the store allocates observed_seq 1. + printf 'ALPHA2\n' >"$stub/repo_alpha" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector delta (alpha) failed" + [ "$(depth)" = "1" ] || fail_msg "R9: detector delta should enqueue exactly one entry, got depth $(depth)" + + # The reconciler co-feeds the SAME store: beta is still unaccounted (its + # baseline was silent), so it enumerates beta -> the store allocates the NEXT + # seq (2), NOT a reissue of 1. alpha's current state is already in the inbox + # (accounted), so it is not double-enumerated. out="$("$RECON" reconcile 2>&1)" rc=$? - [ "$rc" -ne 0 ] || fail_msg "R9: enumerating into a detector-co-fed store must FAIL LOUD (non-zero), not silently alias" - echo "$out" | grep -qi 'dual-allocator' || fail_msg "R9: the failure must name the dual-allocator hazard [$out]" - echo "$out" | grep -q '908' || fail_msg "R9: the failure must reference the #908 follow-up [$out]" - echo "$out" | grep -qi 'REFUSED' || fail_msg "R9: the obligation must be REFUSED/flagged, not swallowed [$out]" - # No collidable seq was written: depth is unchanged, so a later detector delta - # cannot alias a reconciler-enumerated entry. - [ "$(depth)" = "$before" ] || fail_msg "R9: a refused reconcile must NOT enqueue any (aliasable) seq, depth changed $before->$(depth)" - # (That the sole-feeder path DOES still enumerate — i.e. the guard is not a - # blanket no-op — is proven by R5/R6, which enumerate under --allow-enumerate.) + [ "$rc" -ne 0 ] || fail_msg "R9: pre-existing beta is unaccounted on this pass -> must FLAG (non-zero) [$out]" + echo "$out" | grep -qi 'UNACCOUNTED=1' || fail_msg "R9: only beta should be unaccounted (alpha is inbox-accounted) [$out]" + [ "$(depth)" = "2" ] || fail_msg "R9: reconciler must ENUMERATE beta into the co-fed store (depth 2), got $(depth)" + + # A further detector delta on beta must allocate 3 — the unified allocator + # advances past the reconciler's seq, it can NEVER reissue/alias 1 or 2. + printf 'BETA2\n' >"$stub/repo_beta" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector delta (beta) failed" + + # THE NO-ALIAS ASSERTION: every observed_seq across BOTH feeders is DISTINCT and + # the observed prefix is CONTIGUOUS/GAPLESS (1,2,3). Under the OLD two-allocator + # design the detector's private counter would have reissued a seq the reconciler + # already used, so this set would contain a duplicate (alias). + seqs="$("$STORE" drain | jq -r '.observed_seq' | sort -n | tr '\n' ' ')" + [ "$seqs" = "1 2 3 " ] || fail_msg "R9: co-fed observed_seqs must be distinct+gapless '1 2 3', got '$seqs' (a duplicate = the aliasing #908 dissolved)" + ndistinct="$("$STORE" drain | jq -r '.observed_seq' | sort -nu | grep -c .)" + ntotal="$("$STORE" drain | jq -r '.observed_seq' | grep -c .)" + [ "$ndistinct" = "$ntotal" ] || fail_msg "R9: aliasing detected — $ntotal entries but only $ndistinct distinct observed_seq" + # The contiguous-prefix CONSUMED contract holds over the co-fed seqs. + "$STORE" consume --upto 3 >/dev/null 2>&1 || fail_msg "R9: CONSUMED 3 must succeed over the gapless co-fed prefix" ) && ok echo diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index ab095ab2..eceecc7e 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -12,6 +12,11 @@ # T5 atomic write-tmp+rename survives crash mid-write (§1.2) # T6 durability survives restart (retain until CONSUMED) (§2.3) # T7 ack local-write NEVER blocks on network (§2.2) +# T8 SOLE store-side allocator: enqueue (no --seq) allocates contiguous +# observed_seq + anti-swallow fail-loud on explicit seq<=consumed (#908) +# T9 burn-before-enqueue: a failed durable write must NOT advance the +# observed_seq cursor (no burned seq / no interior gap) (#908 arrow 1) +# T10 concurrency: two concurrent enqueues get DISTINCT seqs (lock) (#908) # # Isolated: every test runs against a fresh WAKE_STATE_HOME temp dir. set -uo pipefail @@ -261,6 +266,143 @@ EOF true ) && ok +echo "== T8: SOLE store-side allocator — enqueue (no --seq) allocates contiguous seqs + anti-swallow fail-loud (#908) ==" +( + WAKE_STATE_HOME="$(fresh_state t8)" + export WAKE_STATE_HOME + unset WAKE_AGENT + # No --seq: the STORE allocates and PRINTS the assigned observed_seq. + s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')" + s2="$("$STORE" enqueue --class actionable --locators '{"n":2}')" + [ "$s1" = "1" ] || fail_msg "T8: first store-allocated observed_seq must be 1, got '$s1'" + [ "$s2" = "2" ] || fail_msg "T8: second store-allocated observed_seq must be 2 (contiguous), got '$s2'" + cur="$("$STORE" cursors)" + echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T8: observed_seq cursor should be 2 [$cur]" + echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T8: both allocations must be durably enqueued [$cur]" + # ANTI-SWALLOW: consume the prefix, then a LEGACY explicit --seq inside the + # consumed prefix must FAIL LOUD (never the old silent seq<=consumed no-op). + "$STORE" consume --upto 2 >/dev/null + before_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')" + if "$STORE" enqueue --seq 1 --class actionable --locators '{}' >/dev/null 2>&1; then + fail_msg "T8: explicit --seq 1 <= consumed_seq 2 must FAIL LOUD (anti-swallow), not be a silent no-op" + fi + err="$("$STORE" enqueue --seq 1 --class actionable --locators '{}' 2>&1 || true)" + echo "$err" | grep -qi 'anti-swallow' || fail_msg "T8: the refusal must name the anti-swallow guarantee [$err]" + after_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')" + [ "$before_depth" = "$after_depth" ] || fail_msg "T8: a refused enqueue must not mutate the store (depth $before_depth->$after_depth)" + # And the NEXT real allocation is > consumed (2) — never inside the prefix. + s3="$("$STORE" enqueue --class actionable --locators '{"n":3}')" + [ "$s3" = "3" ] || fail_msg "T8: post-consume allocation must be 3 (>consumed 2), got '$s3'" +) && ok + +echo "== T9: burn-before-enqueue — a failed durable write must NOT advance observed_seq (no burned seq / no gap) (#908 arrow 1) ==" +( + WAKE_STATE_HOME="$(fresh_state t9)" + export WAKE_STATE_HOME + unset WAKE_AGENT + STATE_DIR="$WAKE_STATE_HOME/default" + # One successful allocation: observed_seq=1, and the enqueue lock now exists. + s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')" + [ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'" + obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'" + # Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger: + # bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's + # exact rename TARGET for the durable write — see _wake-common.sh) so the + # atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"): + # the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is + # a structural VFS constraint, not a DAC permission check — root does NOT + # bypass it (no uid can rename over a live mountpoint short of an explicit + # umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a + # CI container running as root BYPASSES DAC checks entirely, so the + # "read-only" dir let mktemp/the write through and every assertion below + # went spuriously green. The old chmod also blocked the CURSOR's own + # mktemp (same dir), so it could never have caught a premature cursor + # commit either — narrower coverage than its comment claimed. Here + # observed_seq is left an ordinary writable file, never touched by the + # mount, so a regression that commits the cursor before/independent of the + # durable write is still genuinely caught by the assertions below. + # + # The mount + forced enqueue run inside a private mount+user namespace + # (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL + # whether this harness runs unprivileged (dev) or as root (CI): creating a + # user namespace needs no pre-existing privilege, and the busy-mountpoint + # rule inside it is the real kernel rule, unaffected by uid in or out of a + # namespace. The bind mount is entirely private to that namespace and + # vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain + # writable file again for every assertion below — no manual cleanup step. + rc=0 + if ! command -v unshare >/dev/null 2>&1; then + fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox" + rc=99 + else + ro_src="$TMP_ROOT/t9-ro-src" + mkdir -p "$ro_src" + : >"$ro_src/pending.jsonl" + # shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED + # bash (inside the unshare'd namespace), not this outer single-quoted one. + unshare --mount --user --map-root-user bash -c ' + set -u + ro_src="$1" + state_dir="$2" + store="$3" + if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then + exit 98 + fi + mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true + "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1 + ' _ "$ro_src" "$STATE_DIR" "$STORE" + rc=$? + if [ "$rc" -eq 98 ]; then + fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed" + fi + fi + [ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)" + # THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was + # burned by a write that never reached the store. (RED if the cursor bumps + # before/independent of the durable write.) + obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_after" = "1" ] || fail_msg "T9: a failed enqueue must NOT advance observed_seq (burned seq -> permanent gap), got '$obs_after'" + # No interior gap was created: CONSUMED 1 (the only real obligation) still holds. + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T9: CONSUMED 1 must remain valid — no burned-seq gap" + # And a fresh allocation resumes cleanly at 2 (not 3 — nothing was burned). + s="$("$STORE" enqueue --class actionable --locators '{"n":"2b"}')" + [ "$s" = "2" ] || fail_msg "T9: post-failure allocation must resume at 2 (no burned seq), got '$s'" +) && ok + +echo "== T10: concurrency — two concurrent enqueues get DISTINCT seqs (enqueue lock) (#908) ==" +if command -v flock >/dev/null 2>&1; then + ( + WAKE_STATE_HOME="$(fresh_state t10)" + export WAKE_STATE_HOME + unset WAKE_AGENT + "$STORE" init >/dev/null 2>&1 || true + o1="$TMP_ROOT/t10.o1" + o2="$TMP_ROOT/t10.o2" + # Two enqueues racing on the SAME store. Without the lock both read + # observed_seq=0 and allocate 1 (aliasing + a lost write); the lock serializes + # the allocate+write so they get 1 and 2. + "$STORE" enqueue --class actionable --locators '{"c":1}' >"$o1" 2>/dev/null & + p1=$! + "$STORE" enqueue --class actionable --locators '{"c":2}' >"$o2" 2>/dev/null & + p2=$! + wait "$p1"; wait "$p2" + a="$(tr -d '[:space:]' <"$o1")" + b="$(tr -d '[:space:]' <"$o2")" + [ -n "$a" ] && [ -n "$b" ] || fail_msg "T10: both concurrent enqueues must print an allocated seq (got '$a','$b')" + [ "$a" != "$b" ] || fail_msg "T10: two concurrent enqueues must get DISTINCT seqs, both got '$a' (aliasing / lost write without the lock)" + cur="$("$STORE" cursors)" + echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T10: observed_seq must reach 2 after two enqueues [$cur]" + echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T10: both entries must be durably stored (no lost write) [$cur]" + # The two allocated seqs are exactly {1,2} (distinct, contiguous, gapless). + lo="$a"; hi="$b"; [ "$a" -gt "$b" ] && { lo="$b"; hi="$a"; } + { [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T10: concurrent seqs must be {1,2}, got {$lo,$hi}" + ) && ok +else + echo " SKIP: flock not available (concurrency guarantee needs flock; matches detector single-instance SKIP)" + ok +fi + echo if [ -s "$FAILFILE" ]; then echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2