From 7603cb1c10525fcd6cc5a6b582334cc66ec349e2 Mon Sep 17 00:00:00 2001 From: mosaic-coder Date: Sat, 25 Jul 2026 20:52:45 -0500 Subject: [PATCH] fix(wake): reconciler fails closed on the observed_seq dual-allocator hazard (#908) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #909 found a blocking correctness defect: the reconciler's "serialised => safe" claim was FACTUALLY WRONG. observed_seq has two INDEPENDENT allocators — the detector's private counter ($STATE_DIR/detector/ observed_seq_counter, W4) and the reconciler's store-cursor+offset — so even STRICTLY SEQUENTIAL calls collide: reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2, detector private still 0); the detector's next delta allocates private 0->1 and enqueues seq1, ALIASING alpha. A consumer acking CONSUMED 1 then silently DROPS a distinct obligation — the exact G3 swallow the reconciler exists to prevent, via seq-aliasing. Reproduced with sequential, never-concurrent calls. Fix (W5-scoped; full store-side allocator unification stays deferred to #908 — detector.sh/W4 untouched): * Enumeration now FAILS CLOSED. Before writing any observed_seq, the reconciler REFUSES (loud, non-zero; the obligation is FLAGGED, never swallowed) when a seq it would allocate could be reissued/aliased: - if a detector is co-feeding this store (its private state dir exists) — ALWAYS refuse (read-only detection; W4 never touched); - else require the operator to assert reconciler-SOLE-FEEDER via --allow-enumerate / WAKE_RECONCILE_ALLOW_ENUMERATE=1. --no-enumerate and `check` never write, so they stay unconditionally safe. * Corrected the false "serialised => safe" rationale in the header comment to state the real dual-allocator hazard honestly, reference #908, and record the operational guard: a detector and the reconciler MUST NOT co-feed one store until #908 lands. * RED-FIRST test R9 reproduces the serialized-collision scenario (detector co-feeding + pre-existing state) and asserts the reconciler FAILS LOUD and enqueues NO aliasable seq (depth unchanged). Verified red: removing the guard re-enables the collision (R9 depth 0->2) and turns R9 red. R5/R6 now enumerate under an explicit sole-feeder assertion (no detector co-feeds those stores). test:framework-shell + shellcheck + firewall all green. Part of #892 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0158NZqN2n2ymKFeJAZ4GUCb --- .../mosaic/framework/tools/wake/reconcile.sh | 97 ++++++++++++++++--- .../tools/wake/test-wake-reconcile.sh | 44 +++++++++ 2 files changed, 125 insertions(+), 16 deletions(-) diff --git a/packages/mosaic/framework/tools/wake/reconcile.sh b/packages/mosaic/framework/tools/wake/reconcile.sh index 0cf4423e..0bc1beb3 100755 --- a/packages/mosaic/framework/tools/wake/reconcile.sh +++ b/packages/mosaic/framework/tools/wake/reconcile.sh @@ -29,14 +29,23 @@ # silently "no state". # # CONTRACT NOTES (flagged, not silently guessed — see PR body): -# * observed_seq allocation on enumeration: observed_seq is detector-owned -# (§2.4). The store exposes no "allocate next seq" primitive, so the -# reconciler enumerates using seqs drawn from the STORE's authoritative -# observed_seq cursor (§2.4: "observed_seq is authoritative") + 1.. within a -# pass. A dual-allocator collision window with the detector's private -# counter is possible under true concurrency; in the per-host -# single-instance model reconcile and poll are serialised. Unifying on a -# single store-side allocator is tracked as a cross-wave follow-up. +# * observed_seq DUAL-ALLOCATOR HAZARD (fail-closed; #908 tracks the fix). +# observed_seq has TWO INDEPENDENT allocators today: the detector's PRIVATE +# counter ($STATE_DIR/detector/observed_seq_counter, W4) and — if this +# reconciler enumerated — the STORE cursor (observed_seq+offset). Because the +# detector's next seq is (private_counter+1), NOT (store_observed_seq+1), the +# two are decoupled and SERIALIZING THEM DOES NOT PREVENT COLLISION: after +# the reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2, +# detector private still 0), the detector's next delta allocates private +# 0->1 and enqueues seq1 — ALIASING alpha. A consumer acking CONSUMED 1 +# (trusting §2.4's unique observed_seq) then silently DROPS a distinct +# obligation — the exact G3 swallow this reconciler exists to prevent. +# THEREFORE, until #908 unifies on a single store-side allocator, the +# reconciler FAILS CLOSED: it REFUSES to enumerate (loud, non-zero; the +# obligation is FLAGGED, never silently swallowed) whenever a detector is +# co-feeding this store, and otherwise requires the operator to assert +# reconciler-SOLE-FEEDER mode. OPERATIONAL GUARD: a detector and this +# reconciler MUST NOT co-feed one store until #908 lands. # * "accounted" is judged against observed_seq/inbox (§4/G3 wording) PLUS the # reconciler's OWN durable reconciled-state ledger — NOT the detector's # hash-file. Judging by the detector baseline would let a silent first-seen @@ -58,6 +67,13 @@ STATE_DIR="$(wake_state_dir)" # Reconciler-local state (its durable reconciled-state ledger) lives in its own # subdir under the store's STATE_DIR so it never collides with store/detector. RECON_DIR="$STATE_DIR/reconciler" +# The detector's PRIVATE state dir (W4). Its presence signals a detector is (or +# has been) allocating observed_seq into THIS store from an INDEPENDENT private +# counter — the co-feeding condition under which reconciler enumeration is +# unsafe (see the dual-allocator hazard note above; #908). Read-only coupling: +# reconcile.sh never writes or imports W4, it only OBSERVES this path to fail +# closed rather than silently alias a seq. +DET_STATE="$STATE_DIR/detector" _need_jq() { command -v jq >/dev/null 2>&1 || { @@ -95,7 +111,7 @@ Commands: * if a lane declares `required_sources`, any id missing from its sources. Exit 0 IFF the inventory is complete; non-zero (loud) on any flag. - reconcile [--no-enumerate] + reconcile [--no-enumerate] [--allow-enumerate] Part (ii) — periodic full reconcile. Enumerates each covered source's CURRENT state and compares it against observed_seq/inbox (+ the reconciler's reconciled-state ledger). Any source state NOT reflected => @@ -103,6 +119,17 @@ Commands: ENUMERATED into the durable store (the startup/pre-existing obligation path). Exit 0 IFF 0 unaccounted were FOUND this pass. + DUAL-ALLOCATOR SAFETY (fail-closed; #908): enumeration WRITES observed_seq + from an allocator INDEPENDENT of the detector's private counter, so it + FAILS LOUD (refuses; the obligation is FLAGGED, never swallowed) when it + would risk aliasing a seq: + * if a detector is co-feeding this store (its private state dir exists) + => refuse, ALWAYS (even with the ack below); + * else the operator MUST assert reconciler-SOLE-FEEDER mode via + --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1), promising no + detector co-feeds this store until #908 lands. + --no-enumerate (and `check`) never write, so they are always safe. + check The full §4/G3 gate, side-effect-free: `inventory` AND `reconcile --no-enumerate`. Exit 0 IFF the inventory is complete AND 0 @@ -114,6 +141,8 @@ Environment: contract as the detector: , def on stdin; exit!=0 or empty => FAIL LOUD). WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh). + WAKE_RECONCILE_ALLOW_ENUMERATE set to 1 to assert reconciler-SOLE-FEEDER mode + (see reconcile's dual-allocator safety note; #908). EOF } @@ -346,19 +375,24 @@ _inbox_has() { } cmd_reconcile() { - local enumerate=1 + local enumerate=1 allow_enumerate=0 while [ $# -gt 0 ]; do case "$1" in --no-enumerate) enumerate=0 shift ;; + --allow-enumerate) + allow_enumerate=1 + shift + ;; *) echo "reconcile.sh reconcile: unknown option '$1'" >&2 exit 2 ;; esac done + [ "${WAKE_RECONCILE_ALLOW_ENUMERATE:-0}" = "1" ] && allow_enumerate=1 local json json="$(_load_watchlist)" || exit $? @@ -370,11 +404,30 @@ cmd_reconcile() { "$STORE_SH" init >/dev/null 2>&1 || true mkdir -p "$RECON_DIR" - # Seq allocation base: the store's authoritative observed_seq (§2.4). + # --- DUAL-ALLOCATOR SAFETY GATE (fail-closed; #908) ------------------------ + # Decide, ONCE, whether ENUMERATION (the observed_seq WRITE path) is safe this + # pass. It is unsafe — and we REFUSE (flag, never swallow) — whenever a seq we + # allocate could be REISSUED by the detector's INDEPENDENT private counter and + # alias a distinct obligation (see the header hazard note). --no-enumerate and + # `check` never write, so the gate does not apply to them. + local enum_mode="$enumerate" enum_refuse_reason="" + if [ "$enumerate" -eq 1 ]; then + if [ -e "$DET_STATE" ]; then + enum_mode=0 + enum_refuse_reason="a detector is co-feeding this store ($DET_STATE exists); its private observed_seq counter (W4) is INDEPENDENT of the store cursor this reconciler allocates from, so enumerating would risk REISSUING/ALIASING a seq and silently swallowing a distinct obligation. #908 tracks the store-side allocator unification that makes co-feeding safe. OPERATIONAL GUARD: do not run a detector and the reconciler against one store until #908." + elif [ "$allow_enumerate" -ne 1 ]; then + enum_mode=0 + enum_refuse_reason="enumeration writes observed_seq via an allocator INDEPENDENT of the detector's private counter (W4); until #908 unifies them, co-feeding one store aliases seqs even under strictly-sequential calls. Set --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1) to assert this store is reconciler-SOLE-FEEDER (no detector co-feeds it)." + fi + fi + + # Seq allocation base: the store's authoritative observed_seq (§2.4). Used ONLY + # on the sole-feeder path the gate above has proven safe. local obs_base offset obs_base="$("$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p')" case "$obs_base" in '' | *[!0-9]*) obs_base=0 ;; esac offset=0 + local refuse_announced=0 refused=0 local pairs kind id failed=0 unaccounted=0 enumerated=0 pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')" @@ -425,11 +478,21 @@ cmd_reconcile() { unaccounted=$((unaccounted + 1)) printf ' %s/%s: UNACCOUNTED (current state not reflected in observed_seq/inbox)\n' "$kind" "$id" >&2 - if [ "$enumerate" -eq 1 ]; then + if [ "$enumerate" -eq 1 ] && [ "$enum_mode" -eq 0 ]; then + # REFUSE to enumerate (fail-closed): the obligation is loudly FLAGGED, NOT + # silently swallowed. Announce the dual-allocator reason once. + if [ "$refuse_announced" -eq 0 ]; then + echo "reconcile.sh: FAIL LOUD (dual-allocator hazard, #908) — $enum_refuse_reason" >&2 + refuse_announced=1 + fi + refused=1 + printf ' -> REFUSED enumeration (FLAGGED, not swallowed; see dual-allocator hazard / #908)\n' >&2 + elif [ "$enum_mode" -eq 1 ]; then # ENUMERATE the pre-existing/startup obligation into the durable store. - # Allocate a seq from the store's authoritative observed_seq (see header - # contract note). Class = non-coalescible fail-safe `actionable` (§2.3) so - # distinct pre-existing obligations never collapse together. + # This path runs ONLY when the safety gate proved the reconciler is the + # SOLE feeder (no detector co-feeding), so its store-cursor allocation + # cannot be aliased. Class = non-coalescible fail-safe `actionable` (§2.3) + # so distinct pre-existing obligations never collapse together. offset=$((offset + 1)) local seq locators seq=$((obs_base + offset)) @@ -456,7 +519,9 @@ EOF return 1 fi if [ "$unaccounted" -ne 0 ]; then - if [ "$enumerate" -eq 1 ]; then + if [ "$refused" -eq 1 ]; then + echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) REFUSED enumeration (dual-allocator hazard, #908). They are flagged, NOT swallowed. Resolve co-feeding / assert sole-feeder, then re-run." >&2 + elif [ "$enumerate" -eq 1 ]; then echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) found and enumerated. Re-run to confirm 0." >&2 else echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) (audit mode; nothing enumerated)." >&2 diff --git a/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh index a0c55b6c..77d248a0 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh @@ -158,6 +158,9 @@ echo "== R5/R6: pre-existing state -> UNACCOUNTED flag + enumerated into store; "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r2" } ] } ] } EOF export WAKE_WATCH_LIST="$wl" + # Sole-feeder mode: no detector co-feeds this store (detector.sh is never run + # here), so enumeration is collision-safe once the operator asserts it. + export WAKE_RECONCILE_ALLOW_ENUMERATE=1 [ "$(depth)" = "0" ] || fail_msg "R5: store must start empty" out="$("$RECON" reconcile 2>&1)" rc=$? @@ -229,6 +232,47 @@ EOF [ "$(depth)" = "0" ] || fail_msg "R8: a failed observation must NOT enumerate anything, got depth $(depth)" ) && ok +echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never silent-alias) ==" +( + WAKE_STATE_HOME="$(fresh_state r9)" + export WAKE_STATE_HOME + unset WAKE_AGENT WAKE_RECONCILE_ALLOW_ENUMERATE + stub="$TMP_ROOT/r9stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + # Pre-existing sources present at startup (the reviewer's alpha/beta repro). + printf 'ALPHA\n' >"$stub/repo_alpha" + printf 'BETA\n' >"$stub/repo_beta" + wl="$TMP_ROOT/r9.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "alpha" }, { "id": "beta" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "alpha" }, { "kind": "repo", "id": "beta" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + # A DETECTOR is co-feeding this store: baseline it (creates the detector's + # private-counter state dir). The detector's private observed_seq allocator is + # now live and INDEPENDENT of the store cursor — the exact co-feeding hazard. + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector baseline failed" + # The pre-existing state is unaccounted (baseline is silent). Enumerating it + # here would allocate a store-cursor seq the detector's private counter can + # later REISSUE and alias — the serialized collision. The reconciler MUST + # refuse (fail loud), NOT enqueue a collidable seq. + before="$(depth)" + out="$("$RECON" reconcile 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R9: enumerating into a detector-co-fed store must FAIL LOUD (non-zero), not silently alias" + echo "$out" | grep -qi 'dual-allocator' || fail_msg "R9: the failure must name the dual-allocator hazard [$out]" + echo "$out" | grep -q '908' || fail_msg "R9: the failure must reference the #908 follow-up [$out]" + echo "$out" | grep -qi 'REFUSED' || fail_msg "R9: the obligation must be REFUSED/flagged, not swallowed [$out]" + # No collidable seq was written: depth is unchanged, so a later detector delta + # cannot alias a reconciler-enumerated entry. + [ "$(depth)" = "$before" ] || fail_msg "R9: a refused reconcile must NOT enqueue any (aliasable) seq, depth changed $before->$(depth)" + # (That the sole-feeder path DOES still enumerate — i.e. the guard is not a + # blanket no-op — is proven by R5/R6, which enumerate under --allow-enumerate.) +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake reconcile harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2