mosaicstack/stack
1
0
Fork 0
Code Issues 86 Pull Requests 1 Actions Packages Projects Releases 15 Wiki Activity

Merge remote-tracking branch 'origin/main' into feat/wake-914-digest-render
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
Details

Browse Source 
# Conflicts:
#	packages/mosaic/framework/tools/wake/manifest.txt
This commit is contained in:
mosaic-coder
2026-07-26 01:52:45 -05:00
parent d0bdad93e7 c585ac3326
commit 5d96cbe896
8 changed files with 500 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
packages/mosaic/framework/tools/wake/_wake-common.sh
View File

@@ -65,6 +65,51 @@ _atomic_write() {
fi
}
# ---------------------------------------------------------------------------
# Enqueue serialization (single store-side observed_seq allocator, #908).
#
# store.sh is the SOLE allocator of observed_seq: it reads the observed_seq
# cursor, computes next = observed_seq + 1, and writes the pending record +
# observed.set + the cursor as ONE transaction. That read-modify-write MUST be
# serialized so two concurrent enqueues cannot read the same cursor and collide
# on a seq. These helpers take an exclusive lock for the duration of the
# transaction. flock (Linux/CI) is the load-bearing mechanism; where flock is
# absent the open still succeeds so a single-threaded enqueue is unaffected (the
# concurrency guarantee then degrades — the concurrency test SKIPs without flock,
# exactly as the detector's single-instance test already does).
#
# The lock uses a FIXED fd (8) within one store.sh process. Each store.sh
# invocation is its own process (the detector/reconciler call it as a
# subprocess), so fd 8 is always free here and never clashes with the detector
# run-loop lock (fd 9, a DIFFERENT process).
# ---------------------------------------------------------------------------
_wake_lock_acquire() {
# _wake_lock_acquire LOCKFILE — open fd 8 on LOCKFILE and take an exclusive
# (blocking) lock. Returns non-zero if the lock cannot be taken.
local lf="$1" dir
dir="$(dirname "$lf")"
[ -d "$dir" ] || mkdir -p "$dir" 2>/dev/null || true
# Open the lock fd. If the file does not yet exist and the dir is writable it
# is created; if the dir is read-only but the file exists, opening it O_WRONLY
# still succeeds (write perm on the file, not the dir).
# NB: a command-less `exec` redirection persists for the WHOLE shell, so we must
# NOT append `2>/dev/null` here (it would permanently silence the caller's
# stderr and swallow every later fail-loud diagnostic). Redirect only fd 8.
exec 8>"$lf" || return 1
if command -v flock >/dev/null 2>&1; then
flock 8 || return 1
fi
return 0
}
_wake_lock_release() {
# _wake_lock_release — drop the enqueue lock (closing fd 8 releases the flock).
# The `2>/dev/null` is SCOPED to the brace group (suppressing a "bad fd" close
# error) — it must NOT sit on a bare `exec`, where the redirection would persist
# for the whole shell and silence every later fail-loud diagnostic.
{ exec 8>&-; } 2>/dev/null || true
}
# _wake_clean_stale_tmp DIR — remove leftover atomic-write temp files (e.g. from
# a crash mid-write). Safe: these are never the live store.
_wake_clean_stale_tmp() {

64
packages/mosaic/framework/tools/wake/detector.sh
View File

@@ -9,11 +9,14 @@
# proven delta-gated poller (gitea-pr-watch: poll -> hash -> deliver only
# on delta; 0-wasted) and adds (a) repo-section/anchor-scoped hashing so
# human-decision FILE edits are caught, not just API-visible state.
# §1.2 observe -> assign observed_seq -> enqueue into the W2 durable store.
# §2.4 Cursor semantics: `observed_seq` is a detector-local monotonic int
# assigned AT OBSERVATION and is AUTHORITATIVE; source SHAs are
# DESCRIPTORS, not the cursor. A per-watch LAST-OBSERVED HASH is compared
# each poll so a revert (A->B->A across polls) is caught as a delta.
# §1.2 observe -> enqueue into the W2 durable store (the store ALLOCATES the
# observed_seq — single store-side allocator, #908).
# §2.4 Cursor semantics: `observed_seq` is a monotonic int, AUTHORITATIVE, now
# allocated SOLELY by store.sh enqueue (#908 unified the two former
# allocators). The detector no longer keeps a private counter: it enqueues
# and captures the store-assigned seq. Source SHAs are DESCRIPTORS, not the
# cursor. A per-watch LAST-OBSERVED HASH is compared each poll so a revert
# (A->B->A across polls) is caught as a delta.
# §4/G2a FAIL-LOUD source semantics: a source/target failure OR a 401 / 403 /
# privacy-404 / partial / ambiguous-empty response MUST fail loud, MUST
# NOT advance the authoritative cursor (`observed_seq`), and MUST NOT be
@@ -100,8 +103,9 @@ Commands:
validate Load + validate the watch-list and its
schema_version against the manifest;
exit 0 if usable, non-zero (loud) if not.
cursors Print the detector-local observed_seq
counter and the store cursors.
cursors Print the store cursors (observed_seq is
the store's single source of truth, #908;
the detector keeps no private counter).
Environment:
WAKE_WATCH_LIST Path to the operator watch-list JSON (required).
@@ -171,19 +175,15 @@ _load_watchlist() {
printf '%s' "$json"
}
# --- detector-local observed_seq (§2.4, authoritative monotonic int) --------
# _next_observed_seq — atomically increment + return the detector-local counter.
# Assigned AT OBSERVATION (only on a real delta). A single global monotonic
# sequence across ALL watches (source SHAs are descriptors, NOT the cursor).
_next_observed_seq() {
mkdir -p "$DET_DIR"
local cur nxt
cur="$(_wake_read_int "$DET_DIR/observed_seq_counter" 0)"
nxt=$((cur + 1))
printf '%s' "$nxt" | _atomic_write "$DET_DIR/observed_seq_counter"
printf '%s' "$nxt"
}
# --- observed_seq: the store is the SOLE allocator (#908) -------------------
#
# The detector NO LONGER keeps a private observed_seq counter. That private
# counter was the shared root of three defects (burn-before-enqueue, W5
# dual-allocator aliasing, and the migration-restart silent-swallow); see the PR
# for #908. observed_seq is now allocated EXCLUSIVELY by store.sh enqueue (which
# reads its own cursor and returns the assigned seq). The detector calls enqueue
# WITHOUT --seq and captures the store-returned seq for logging/locators — it
# never allocates a seq itself, so the seam is gone.
# --- per-watch last-observed hash (§2.4 revert/ABA-at-rest detection) --------
@@ -274,10 +274,11 @@ _poll_source() {
return 0
fi
# --- DELTA: assign the next observed_seq AT OBSERVATION and enqueue --------
# (a revert A->B->A lands here because $h != the stored $last).
local seq locators emit_ts
seq="$(_next_observed_seq)"
# --- DELTA: enqueue and let the STORE allocate observed_seq (#908) ---------
# (a revert A->B->A lands here because $h != the stored $last). The store is
# the sole allocator: we call enqueue WITHOUT --seq and capture the seq it
# assigns (for logging/locators/emit). We do NOT allocate a seq ourselves.
local locators emit_ts
emit_ts="$(date +%s)"
# NB: `def` is a reserved word in jq — the source-definition arg is `$sdef`.
locators="$(jq -cn \
@@ -288,16 +289,19 @@ _poll_source() {
'{kind:$kind, id:$id, observed_hash:$hash}
+ ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')"
local args=(enqueue --seq "$seq" --locators "$locators" --emit-ts "$emit_ts")
local args=(enqueue --locators "$locators" --emit-ts "$emit_ts")
# Pass class through only when the source declares one; absent => the store
# defaults to `actionable` (fail-safe, §2.3). Never guess a coalescible class.
[ -n "$class" ] && args+=(--class "$class")
if ! "$STORE_SH" "${args[@]}" >/dev/null; then
echo "detector.sh: store enqueue FAILED for '$kind/$id' seq $seq" >&2
local seq
if ! seq="$("$STORE_SH" "${args[@]}")"; then
echo "detector.sh: store enqueue FAILED for '$kind/$id' (observed_seq NOT advanced; hash NOT advanced)" >&2
return 1
fi
# Only AFTER a durable enqueue do we advance the per-watch last-observed hash.
# (store-allocated seq $seq is available for logging/emit if needed.)
: "$seq"
printf '%s' "$h" | _atomic_write "$hf"
return 0
}
@@ -420,9 +424,9 @@ cmd_run() {
}
cmd_cursors() {
local counter
counter="$(_wake_read_int "$DET_DIR/observed_seq_counter" 0)"
printf 'detector_observed_seq=%s\n' "$counter"
# observed_seq has a SINGLE source of truth now (#908): the STORE cursor. The
# detector keeps no private counter, so cursors simply reports the store's
# authoritative cursors (observed_seq / consumed_seq / pending_depth).
"$STORE_SH" cursors 2>/dev/null || true
}

23
packages/mosaic/framework/tools/wake/manifest.txt
View File

@@ -24,7 +24,17 @@
# alarm-target/HMAC-key install-validation. Also folds in the two W6
# monitor-integration observations (monitor-side ingested_ts
# staleness + beacon HMAC-verify at record).
# 0.6.1 #914 digest.sh renderer fixes (live wake-pilot findings): (a) the
# 0.6.1 #908 — UNIFY observed_seq on a SINGLE store-side allocator. store.sh
# enqueue is now the sole allocator (reads its own cursor, next=+1
# under an exclusive lock, prints the seq; commits IFF the durable
# write succeeds). The detector-private observed_seq_counter and its
# --seq hand-off are DELETED; the reconciler enumerates via the same
# store allocator (its dual-allocator fail-closed guard retired). This
# dissolves the three defects rooted in the private-counter seam:
# burn-before-enqueue (arrow 1), W5 co-feed aliasing (arrow 2), and
# the migration-restart silent-swallow (arrow 3, now structurally
# impossible — allocation is always > consumed or fails loud).
# 0.6.2 #914 digest.sh renderer fixes (live wake-pilot findings): (a) the
# embedded ack copy-run line now bakes an explicit
# WAKE_AGENT=<render-time-agent> prefix (shell-quoted) so an
# env-less copy-run resolves to the correct per-agent namespace
@@ -36,7 +46,7 @@
# empty. Display-only: the ACTIONABLE-tier hard-locator FAIL-LOUD
# gate (_has_hard_locator, exit 4) is unchanged.
component=wake
version=0.6.1
version=0.6.2
# Watch-list schema this component consumes, and the INCLUSIVE range of
# schema_version values it supports. A wake-watch-list.json whose schema_version
@@ -54,8 +64,9 @@ schema_max=1
# sign.sh A5 — non-circular HMAC signer (independent wake_id,
# load_credentials by-name; fills the hmac placeholder). (W3)
# detector.sh A1 — per-host single-instance delta-gated detector daemon
# (flock, anchor-scoped hashing, detector-local observed_seq,
# fail-loud source semantics; enqueues deltas to store.sh). (W4)
# (flock, anchor-scoped hashing, fail-loud source semantics;
# enqueues deltas to store.sh and captures the store-allocated
# observed_seq — no private counter, #908). (W4)
# fn-oracle.sh A6 — synthetic-canary FN-oracle: injects a KNOWN delta at the
# source boundary, drives the pipeline through the detector's
# public poll-once, asserts CONSUMED within the per-class SLO
@@ -64,7 +75,9 @@ schema_max=1
# reconcile.sh A7 — source-parity reconciler: (i) source-coverage parity
# inventory (an omitted source cannot pass the vector
# vacuously) + (ii) periodic full reconcile to 0-unaccounted,
# enumerating pre-existing/startup state into the store. (W5)
# enumerating pre-existing/startup state into the store via the
# SINGLE store-side allocator (co-feed is safe; the former
# dual-allocator fail-closed guard retired, #908). (W5)
# beacon.sh A8 — off-host DEAD-MAN liveness beacon: a monotonic beacon
# EMITTER (emit — the primitive the detector run-loop calls
# each cycle), the off-host monitor's RECEIVER + beacon-ABSENCE

127
packages/mosaic/framework/tools/wake/reconcile.sh
View File

@@ -29,23 +29,24 @@
# silently "no state".
#
# CONTRACT NOTES (flagged, not silently guessed — see PR body):
# * observed_seq DUAL-ALLOCATOR HAZARD (fail-closed; #908 tracks the fix).
# observed_seq has TWO INDEPENDENT allocators today: the detector's PRIVATE
# counter ($STATE_DIR/detector/observed_seq_counter, W4) and — if this
# reconciler enumerated — the STORE cursor (observed_seq+offset). Because the
# detector's next seq is (private_counter+1), NOT (store_observed_seq+1), the
# two are decoupled and SERIALIZING THEM DOES NOT PREVENT COLLISION: after
# the reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2,
# detector private still 0), the detector's next delta allocates private
# 0->1 and enqueues seq1 — ALIASING alpha. A consumer acking CONSUMED 1
# (trusting §2.4's unique observed_seq) then silently DROPS a distinct
# obligation — the exact G3 swallow this reconciler exists to prevent.
# THEREFORE, until #908 unifies on a single store-side allocator, the
# reconciler FAILS CLOSED: it REFUSES to enumerate (loud, non-zero; the
# obligation is FLAGGED, never silently swallowed) whenever a detector is
# co-feeding this store, and otherwise requires the operator to assert
# reconciler-SOLE-FEEDER mode. OPERATIONAL GUARD: a detector and this
# reconciler MUST NOT co-feed one store until #908 lands.
# * observed_seq DUAL-ALLOCATOR HAZARD — RESOLVED by #908 (single store-side
# allocator). HISTORY: observed_seq once had TWO INDEPENDENT allocators — the
# detector's PRIVATE counter (W4) and the STORE cursor (observed_seq+offset)
# this reconciler used to enumerate from. Because the detector's next seq was
# (private_counter+1), NOT (store_observed_seq+1), the two were decoupled and
# even serializing them did NOT prevent collision: the reconciler enumerating
# alpha->seq1, beta->seq2 while the detector's private counter was still 0
# meant the detector's next delta re-allocated seq1, ALIASING alpha, and a
# consumer acking CONSUMED 1 silently dropped a distinct obligation.
# #908 DISSOLVES this at the root: store.sh enqueue is now the SOLE allocator
# (reads its own observed_seq cursor, next=+1, under an exclusive lock). This
# reconciler now enumerates by calling store.sh enqueue WITHOUT --seq — the
# SAME single allocator the detector uses — so co-feeding one store is SAFE:
# both paths draw distinct, contiguous seqs from the one cursor. The former
# fail-closed refusal (and the reconciler-SOLE-FEEDER assertion) are therefore
# RETIRED; --allow-enumerate / WAKE_RECONCILE_ALLOW_ENUMERATE remain accepted
# as deprecated no-ops for caller compatibility. The G3 accounting invariants
# (0-UNACCOUNTED, vacuous-pass prevention) are unaffected and still enforced.
# * "accounted" is judged against observed_seq/inbox (§4/G3 wording) PLUS the
# reconciler's OWN durable reconciled-state ledger — NOT the detector's
# hash-file. Judging by the detector baseline would let a silent first-seen
@@ -67,13 +68,6 @@ STATE_DIR="$(wake_state_dir)"
# Reconciler-local state (its durable reconciled-state ledger) lives in its own
# subdir under the store's STATE_DIR so it never collides with store/detector.
RECON_DIR="$STATE_DIR/reconciler"
# The detector's PRIVATE state dir (W4). Its presence signals a detector is (or
# has been) allocating observed_seq into THIS store from an INDEPENDENT private
# counter — the co-feeding condition under which reconciler enumeration is
# unsafe (see the dual-allocator hazard note above; #908). Read-only coupling:
# reconcile.sh never writes or imports W4, it only OBSERVES this path to fail
# closed rather than silently alias a seq.
DET_STATE="$STATE_DIR/detector"
_need_jq() {
command -v jq >/dev/null 2>&1 || {
@@ -119,16 +113,14 @@ Commands:
ENUMERATED into the durable store (the startup/pre-existing obligation
path). Exit 0 IFF 0 unaccounted were FOUND this pass.
DUAL-ALLOCATOR SAFETY (fail-closed; #908): enumeration WRITES observed_seq
from an allocator INDEPENDENT of the detector's private counter, so it
FAILS LOUD (refuses; the obligation is FLAGGED, never swallowed) when it
would risk aliasing a seq:
* if a detector is co-feeding this store (its private state dir exists)
=> refuse, ALWAYS (even with the ack below);
* else the operator MUST assert reconciler-SOLE-FEEDER mode via
--allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1), promising no
detector co-feeds this store until #908 lands.
--no-enumerate (and `check`) never write, so they are always safe.
SINGLE STORE-SIDE ALLOCATOR (#908, resolved): enumeration allocates
observed_seq by calling store.sh enqueue WITHOUT --seq — the SAME sole
allocator the detector uses (store cursor +1 under an exclusive lock). A
detector co-feeding this store is therefore SAFE: both draw distinct,
contiguous seqs from the one cursor, so no aliasing is possible. The old
fail-closed refusal is retired; --allow-enumerate (and
WAKE_RECONCILE_ALLOW_ENUMERATE=1) are accepted as deprecated no-ops.
--no-enumerate (and `check`) never write, so they remain side-effect-free.
check
The full §4/G3 gate, side-effect-free: `inventory` AND
@@ -141,8 +133,9 @@ Environment:
contract as the detector: <cmd> <kind> <id>, def on
stdin; exit!=0 or empty => FAIL LOUD).
WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh).
WAKE_RECONCILE_ALLOW_ENUMERATE set to 1 to assert reconciler-SOLE-FEEDER mode
(see reconcile's dual-allocator safety note; #908).
WAKE_RECONCILE_ALLOW_ENUMERATE DEPRECATED no-op (#908 made co-feeding safe via
the single store-side allocator); accepted for
backward compatibility, no longer required.
EOF
}
@@ -404,30 +397,14 @@ cmd_reconcile() {
"$STORE_SH" init >/dev/null 2>&1 || true
mkdir -p "$RECON_DIR"
# --- DUAL-ALLOCATOR SAFETY GATE (fail-closed; #908) ------------------------
# Decide, ONCE, whether ENUMERATION (the observed_seq WRITE path) is safe this
# pass. It is unsafe — and we REFUSE (flag, never swallow) — whenever a seq we
# allocate could be REISSUED by the detector's INDEPENDENT private counter and
# alias a distinct obligation (see the header hazard note). --no-enumerate and
# `check` never write, so the gate does not apply to them.
local enum_mode="$enumerate" enum_refuse_reason=""
if [ "$enumerate" -eq 1 ]; then
if [ -e "$DET_STATE" ]; then
enum_mode=0
enum_refuse_reason="a detector is co-feeding this store ($DET_STATE exists); its private observed_seq counter (W4) is INDEPENDENT of the store cursor this reconciler allocates from, so enumerating would risk REISSUING/ALIASING a seq and silently swallowing a distinct obligation. #908 tracks the store-side allocator unification that makes co-feeding safe. OPERATIONAL GUARD: do not run a detector and the reconciler against one store until #908."
elif [ "$allow_enumerate" -ne 1 ]; then
enum_mode=0
enum_refuse_reason="enumeration writes observed_seq via an allocator INDEPENDENT of the detector's private counter (W4); until #908 unifies them, co-feeding one store aliases seqs even under strictly-sequential calls. Set --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1) to assert this store is reconciler-SOLE-FEEDER (no detector co-feeds it)."
fi
fi
# Seq allocation base: the store's authoritative observed_seq (§2.4). Used ONLY
# on the sole-feeder path the gate above has proven safe.
local obs_base offset
obs_base="$("$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p')"
case "$obs_base" in '' | *[!0-9]*) obs_base=0 ;; esac
offset=0
local refuse_announced=0 refused=0
# --- SINGLE STORE-SIDE ALLOCATOR (#908) -----------------------------------
# Enumeration allocates observed_seq by calling store.sh enqueue WITHOUT --seq
# (the sole allocator the detector also uses). Co-feeding one store is safe —
# both paths draw distinct, contiguous seqs from the one store cursor under its
# exclusive lock — so the former dual-allocator fail-closed refusal is retired.
# allow_enumerate is now a deprecated no-op, retained only so old callers that
# pass --allow-enumerate / WAKE_RECONCILE_ALLOW_ENUMERATE=1 still work.
: "$allow_enumerate"
local pairs kind id failed=0 unaccounted=0 enumerated=0
pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')"
@@ -478,34 +455,24 @@ cmd_reconcile() {
unaccounted=$((unaccounted + 1))
printf ' %s/%s: UNACCOUNTED (current state not reflected in observed_seq/inbox)\n' "$kind" "$id" >&2
if [ "$enumerate" -eq 1 ] && [ "$enum_mode" -eq 0 ]; then
# REFUSE to enumerate (fail-closed): the obligation is loudly FLAGGED, NOT
# silently swallowed. Announce the dual-allocator reason once.
if [ "$refuse_announced" -eq 0 ]; then
echo "reconcile.sh: FAIL LOUD (dual-allocator hazard, #908) — $enum_refuse_reason" >&2
refuse_announced=1
fi
refused=1
printf ' -> REFUSED enumeration (FLAGGED, not swallowed; see dual-allocator hazard / #908)\n' >&2
elif [ "$enum_mode" -eq 1 ]; then
# ENUMERATE the pre-existing/startup obligation into the durable store.
# This path runs ONLY when the safety gate proved the reconciler is the
# SOLE feeder (no detector co-feeding), so its store-cursor allocation
# cannot be aliased. Class = non-coalescible fail-safe `actionable` (§2.3)
if [ "$enumerate" -eq 1 ]; then
# ENUMERATE the pre-existing/startup obligation into the durable store via
# the SINGLE store-side allocator (#908): store.sh enqueue WITHOUT --seq
# allocates the next observed_seq under the store's lock and PRINTS it. Both
# the detector and this reconciler draw from that one cursor, so a co-fed
# store cannot alias. Class = non-coalescible fail-safe `actionable` (§2.3)
# so distinct pre-existing obligations never collapse together.
offset=$((offset + 1))
local seq locators
seq=$((obs_base + offset))
locators="$(jq -cn --arg kind "$kind" --arg id "$id" --arg hash "$curhash" \
--argjson sdef "$def" \
'{kind:$kind, id:$id, observed_hash:$hash, reconciled:true}
+ ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')"
if "$STORE_SH" enqueue --seq "$seq" --class actionable --locators "$locators" >/dev/null 2>&1; then
if seq="$("$STORE_SH" enqueue --class actionable --locators "$locators")"; then
printf '%s' "$curhash" | _atomic_write "$ledger"
enumerated=$((enumerated + 1))
printf ' -> enumerated into store as observed_seq=%s (class=actionable)\n' "$seq"
else
echo "reconcile.sh: enumerate FAILED for '$kind/$id' seq $seq" >&2
echo "reconcile.sh: enumerate FAILED for '$kind/$id'" >&2
failed=1
fi
fi
@@ -519,9 +486,7 @@ EOF
return 1
fi
if [ "$unaccounted" -ne 0 ]; then
if [ "$refused" -eq 1 ]; then
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) REFUSED enumeration (dual-allocator hazard, #908). They are flagged, NOT swallowed. Resolve co-feeding / assert sole-feeder, then re-run." >&2
elif [ "$enumerate" -eq 1 ]; then
if [ "$enumerate" -eq 1 ]; then
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) found and enumerated. Re-run to confirm 0." >&2
else
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) (audit mode; nothing enumerated)." >&2

117
packages/mosaic/framework/tools/wake/store.sh
View File

@@ -8,8 +8,11 @@
# §2.4 cursor semantics: observed_seq is authoritative; SHAs are descriptors.
#
# Three cursors:
# observed_seq (detector-owned; this lib RECORDS/stores it) — monotonic int
# assigned at observation. Authoritative. Source SHAs are
# observed_seq (store-owned SINGLE ALLOCATOR, #908) — monotonic int assigned by
# THIS lib at enqueue: next = observed_seq + 1, under an exclusive
# lock, committed IFF the durable write succeeds. Authoritative.
# There is exactly ONE allocator (this store); the detector and
# reconciler no longer allocate independently. Source SHAs are
# descriptors, NOT the cursor (§2.4).
# pending-inbox (durable) — append store; retains until CONSUMED; survives
# pane death / park / restart.
@@ -48,9 +51,23 @@ Usage: store.sh <command> [options]
Commands:
init Create/repair the XDG state layout.
enqueue --seq N --class C [opts] Record an observation into the durable
pending-inbox (coalesce or append).
--seq N observed_seq (detector-assigned monotonic int). Required.
enqueue [--class C] [opts] Record an observation into the durable
pending-inbox (coalesce or append) and
ALLOCATE its observed_seq. store.sh is the
SOLE allocator of observed_seq (#908):
under an exclusive enqueue lock it reads
the observed_seq cursor, sets
next = observed_seq + 1, writes the
pending record + observed.set + the cursor
as ONE transaction, and PRINTS next to
stdout. The cursor advances IFF the durable
write succeeds (no burn-before-enqueue).
--seq N LEGACY explicit observed_seq (optional). Normal callers
(detector, reconciler) OMIT this and let the store
allocate. Retained only for tests/tools that must place a
SPECIFIC seq (e.g. construct a gap). An explicit seq that
is <= consumed_seq is REFUSED loudly (anti-swallow): the
idempotent-ignore path can never silently drop a wake.
--class C digest|actionable|human|terminal-log|reaction.
Absent/empty => actionable (fail-safe).
--locators JSON JSON value for the entry's locators (default {}).
@@ -109,12 +126,18 @@ cmd_enqueue() {
esac
done
case "$seq" in
'' | *[!0-9]*)
echo "store.sh enqueue: --seq must be a non-negative integer" >&2
exit 2
;;
esac
# --seq is OPTIONAL (#908): normal callers omit it and the store allocates.
# If supplied (legacy/explicit), it must be a non-negative integer.
local explicit=0
if [ -n "$seq" ]; then
explicit=1
case "$seq" in
*[!0-9]*)
echo "store.sh enqueue: --seq must be a non-negative integer" >&2
exit 2
;;
esac
fi
# Absent class => actionable (fail-safe, §2.3).
[ -n "$class" ] || class="actionable"
@@ -142,13 +165,42 @@ cmd_enqueue() {
_wake_init_dir "$STATE_DIR"
local consumed
consumed="$(_wake_read_int "$STATE_DIR/consumed_seq" 0)"
# --- serialize the whole allocate+write transaction (#908) ----------------
# A single store-side allocator means the read of observed_seq, the durable
# write, and the cursor bump MUST be one critical section: two concurrent
# enqueues that both read observed_seq=N would otherwise both allocate N+1 and
# collide. The lock (flock where present) makes concurrent enqueues get
# DISTINCT seqs.
_wake_lock_acquire "$STATE_DIR/.enqueue.lock" || {
echo "store.sh enqueue: could not acquire the enqueue lock ($STATE_DIR/.enqueue.lock)" >&2
exit 1
}
# An observation already inside the consumed prefix is idempotently ignored
# (§1.2: cursor never regresses; a re-observed already-consumed seq is a no-op).
local consumed observed
consumed="$(_wake_read_int "$STATE_DIR/consumed_seq" 0)"
observed="$(_wake_read_int "$STATE_DIR/observed_seq" 0)"
if [ "$explicit" -eq 0 ]; then
# ALLOCATE: the store is the sole allocator. next = observed_seq + 1.
seq=$((observed + 1))
fi
# --- anti-swallow FAIL-LOUD (#908, arrow #3) ------------------------------
# observed_seq >= consumed_seq always (consumed advances only over the gapless
# observed prefix), so an ALLOCATED next = observed+1 is ALWAYS > consumed. The
# old "seq <= consumed => idempotent no-op" path is therefore UNREACHABLE for an
# allocation; if it is ever reached the store is inconsistent (corruption) — we
# FAIL LOUD, never a silent no-op. For a LEGACY explicit --seq, a seq inside the
# consumed prefix is likewise refused loudly so it can never silently swallow a
# fresh obligation (the migration-restart killer must be structurally impossible).
if [ "$seq" -le "$consumed" ]; then
return 0
_wake_lock_release
if [ "$explicit" -eq 1 ]; then
echo "store.sh enqueue: REFUSED explicit --seq $seq <= consumed_seq $consumed — anti-swallow (#908): a seq inside the consumed prefix is never a fresh obligation and must NEVER be silently ignored." >&2
else
echo "store.sh enqueue: FAIL LOUD (#908 allocation invariant) — computed observed_seq $seq <= consumed_seq $consumed (observed_seq cursor=$observed). The store is inconsistent; refusing to allocate rather than silently no-op." >&2
fi
exit 1
fi
# Build the durable entry. `hmac` is an UNSIGNED PLACEHOLDER (W3/A5 signs it).
@@ -159,6 +211,7 @@ cmd_enqueue() {
--argjson locators "$locators" \
--argjson emit_ts "$emit_ts" \
'{observed_seq:$seq, locators:$locators, class:$class, emit_ts:$emit_ts, hmac:""}')" || {
_wake_lock_release
echo "store.sh enqueue: failed to build entry" >&2
exit 1
}
@@ -168,6 +221,11 @@ cmd_enqueue() {
# digest lines, then append the newest). Newest subsumes prior.
# others: APPEND (never replaced).
# In BOTH cases the entry lands in the durable store.
#
# ORDERING (#908, arrow #1 — no burn-before-enqueue): the durable pending write
# comes FIRST and its failure ABORTS before the observed.set / observed_seq
# cursor writes. The cursor advances IFF the durable pending write succeeds, so
# a failed enqueue can never burn a seq that never reached the store.
local new_pending
if [ "$class" = "digest" ]; then
new_pending="$(jq -c 'select(.class != "digest")' "$STATE_DIR/pending.jsonl" 2>/dev/null || true)"
@@ -176,20 +234,35 @@ cmd_enqueue() {
new_pending="$(cat "$STATE_DIR/pending.jsonl" 2>/dev/null; printf '%s\n' "$entry")"
new_pending="$(printf '%s' "$new_pending" | grep -v '^[[:space:]]*$' || true)"
fi
printf '%s\n' "$new_pending" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/pending.jsonl"
if ! printf '%s\n' "$new_pending" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/pending.jsonl"; then
_wake_lock_release
echo "store.sh enqueue: durable pending write FAILED for seq $seq — observed_seq cursor NOT advanced (no burned seq, no interior gap)." >&2
exit 1
fi
# --- observed.set: record the seq in the live window (gap-detector SoT) ----
{
if ! {
cat "$STATE_DIR/observed.set" 2>/dev/null
printf '%s\n' "$seq"
} | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set"
} | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set"; then
_wake_lock_release
echo "store.sh enqueue: observed.set write FAILED for seq $seq — observed_seq cursor NOT advanced." >&2
exit 1
fi
# --- observed_seq cursor: max monotonic int -------------------------------
local observed
observed="$(_wake_read_int "$STATE_DIR/observed_seq" 0)"
# --- observed_seq cursor: max monotonic int (COMMIT of the allocation) -----
# Written LAST: only now, after the durable pending + observed.set writes
# succeeded, does the cursor advance. This is the atomic commit point of the
# allocation.
if [ "$seq" -gt "$observed" ]; then
printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq"
fi
_wake_lock_release
# Print the allocated observed_seq to stdout (sole-allocator contract): the
# detector/reconciler capture it for logging/locators/emit.
printf '%s\n' "$seq"
}
cmd_drain() {

68
packages/mosaic/framework/tools/wake/test-wake-detector.sh
View File

@@ -13,6 +13,9 @@
# observed_seq NOT advanced (the G2a invariant) (§4/G2a)
# D7 anchor-scoped hashing: edit OUTSIDE the anchor is a no-op;
# edit INSIDE the anchor is a delta (§1.1 (a))
# D8 MIGRATION-RESTART: post-migration first delta ALLOCATES a seq
# > consumed via the single store-side allocator — never the
# silent seq<=consumed no-op the old private counter caused (#908 arrow 3)
#
# Uses FAKE/STUB sources only (no live network). Isolated: every test runs
# against a fresh WAKE_STATE_HOME temp dir.
@@ -62,7 +65,12 @@ fresh_state() {
# depth — pending_depth reported by the store for the current namespace.
depth() { "$STORE" cursors | sed -n 's/pending_depth=//p'; }
det_seq() { "$DET" cursors | sed -n 's/detector_observed_seq=//p'; }
# observed_seq is the STORE's single source of truth now (#908): the detector
# keeps no private counter, so `detector.sh cursors` reports the store cursors.
# This asserts the SAME invariant the old detector-private counter did (a delta
# advances observed_seq by exactly one; no-change/first-seen/fail-loud do not) —
# just read from the unified store allocator.
det_seq() { "$DET" cursors | sed -n 's/^observed_seq=//p'; }
# write_watchlist FILE VERSION — a minimal valid watch-list with one repo source
# ("r1") plus optionally a board_file with an anchor ("b1").
@@ -332,6 +340,64 @@ echo "== D7: anchor-scoped hashing (edit outside anchor = no-op; inside = delta)
[ "$(depth)" -ge 1 ] || fail_msg "D7: in-anchor edit must enqueue, got depth $(depth)"
) && ok
echo "== D8: MIGRATION-RESTART — post-migration first delta ALLOCATES > consumed, never a silent no-op (#908, arrow #3) =="
(
WAKE_STATE_HOME="$(fresh_state d8)"
export WAKE_STATE_HOME
unset WAKE_AGENT
stub="$TMP_ROOT/d8stub"
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
# A watch-list with a SINGLE repo source (clean seq accounting).
wl="$TMP_ROOT/d8.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1", "class": "actionable" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
# --- simulate adopt -> migrate -> restart --------------------------------
# §5 migration seeds the STORE cursors (consumed_seq=N, observed_seq=N) so the
# already-consumed prefix is preserved. It seeds NO detector-private counter —
# because after #908 the detector HAS none; the store cursor is the single SoT.
# This is the exact pilot state that used to silently swallow the first delta.
sd="$WAKE_STATE_HOME/default"
mkdir -p "$sd"
printf '5' >"$sd/consumed_seq"
printf '5' >"$sd/observed_seq"
: >"$sd/observed.set" # prefix <=5 fully consumed; live window empty
: >"$sd/pending.jsonl"
# "Restart" the detector: fresh detector-local state (no hash yet, and — the
# killer precondition — NO private observed_seq counter exists).
printf 'STATE-A\n' >"$stub/repo_r1"
"$DET" poll-once >/dev/null 2>&1 || fail_msg "D8: post-restart baseline pass failed"
# First-seen baselines silently (deliver-on-new); the store cursor is untouched.
[ "$(depth)" = "0" ] || fail_msg "D8: baseline must not enqueue, got depth $(depth)"
[ "$(det_seq)" = "5" ] || fail_msg "D8: migration-seeded observed_seq must be preserved at 5, got $(det_seq)"
# --- the real, un-consumed obligation: ONE delta -------------------------
printf 'STATE-B\n' >"$stub/repo_r1"
"$DET" poll-once >/dev/null 2>&1 || fail_msg "D8: post-migration delta pass failed"
# THE KILLER ASSERTION: the store (sole allocator) allocates 6 = observed(5)+1,
# which is > consumed(5). Under the OLD private-counter detector this delta got
# seq 1 (counter restarted at 0) which is <= consumed 5 and was SILENTLY
# swallowed by store enqueue's seq<=consumed no-op (depth stayed 0). With the
# unified allocator it CANNOT be swallowed: it enqueues and WOULD wake.
[ "$(det_seq)" = "6" ] || fail_msg "D8: post-migration first delta must ALLOCATE observed_seq 6 (>consumed 5), got $(det_seq) — a restart-at-0 private counter would give <=5 and be swallowed"
[ "$(depth)" = "1" ] || fail_msg "D8: post-migration delta must ENQUEUE a real obligation (depth 1), NOT be a silent no-op, got depth $(depth)"
# And it is genuinely deliverable (a locator-bearing pending entry the consumer
# would wake on), not swallowed.
entry="$("$STORE" drain)"
echo "$entry" | jq -e 'select(.observed_seq==6 and .locators.kind=="repo" and .locators.id=="r1")' >/dev/null \
|| fail_msg "D8: the enqueued post-migration obligation must be a real deliverable at observed_seq=6 [$entry]"
# Contiguous-prefix contract still holds: CONSUMED 6 succeeds (6 is observed,
# 5 is the consumed prefix — no interior gap).
"$STORE" consume --upto 6 >/dev/null 2>&1 || fail_msg "D8: CONSUMED 6 must succeed over the contiguous prefix after the delta"
) && ok
echo
if [ -s "$FAILFILE" ]; then
echo "wake detector harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2

60
packages/mosaic/framework/tools/wake/test-wake-reconcile.sh
View File

@@ -15,6 +15,10 @@
# R7 a source already reflected in the inbox -> ACCOUNTED (no double-
# enumeration of detector-delivered state) (§4/G3-ii)
# R8 a source adapter error -> FAIL LOUD (G2a parity; never 'no state') (§4/G2a)
# R9 co-feed NO-ALIAS: detector + reconciler share the ONE store-side
# observed_seq allocator, so co-feeding one store yields DISTINCT,
# contiguous seqs (no aliasing). Formerly the fail-closed dual-
# allocator refusal; #908 dissolves the hazard at the root. (§4/G3, #908)
#
# Uses FAKE/STUB sources only (no live network). Isolated per test.
# shellcheck disable=SC2030,SC2031
@@ -232,7 +236,7 @@ EOF
[ "$(depth)" = "0" ] || fail_msg "R8: a failed observation must NOT enumerate anything, got depth $(depth)"
) && ok
echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never silent-alias) =="
echo "== R9: co-feed NO-ALIAS — detector + reconciler share ONE store allocator; every observed_seq distinct + gapless (#908 resolved) =="
(
WAKE_STATE_HOME="$(fresh_state r9)"
export WAKE_STATE_HOME
@@ -241,7 +245,7 @@ echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
# Pre-existing sources present at startup (the reviewer's alpha/beta repro).
# The reviewer's alpha/beta repro (formerly the dual-allocator ALIASING case).
printf 'ALPHA\n' >"$stub/repo_alpha"
printf 'BETA\n' >"$stub/repo_beta"
wl="$TMP_ROOT/r9.json"
@@ -251,26 +255,44 @@ echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "alpha" }, { "kind": "repo", "id": "beta" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
# A DETECTOR is co-feeding this store: baseline it (creates the detector's
# private-counter state dir). The detector's private observed_seq allocator is
# now live and INDEPENDENT of the store cursor — the exact co-feeding hazard.
# A DETECTOR is co-feeding this store. Under #908 both the detector and the
# reconciler allocate observed_seq from the ONE store cursor (store.sh enqueue,
# no --seq), so co-feeding is now SAFE — the reconciler ENUMERATES (no refusal),
# and no seq can be aliased. (The old guard failed closed here; that refusal is
# retired exactly to the extent the single allocator makes safe.)
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector baseline failed"
# The pre-existing state is unaccounted (baseline is silent). Enumerating it
# here would allocate a store-cursor seq the detector's private counter can
# later REISSUE and alias — the serialized collision. The reconciler MUST
# refuse (fail loud), NOT enqueue a collidable seq.
before="$(depth)"
# A real detector delta on alpha -> the store allocates observed_seq 1.
printf 'ALPHA2\n' >"$stub/repo_alpha"
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector delta (alpha) failed"
[ "$(depth)" = "1" ] || fail_msg "R9: detector delta should enqueue exactly one entry, got depth $(depth)"
# The reconciler co-feeds the SAME store: beta is still unaccounted (its
# baseline was silent), so it enumerates beta -> the store allocates the NEXT
# seq (2), NOT a reissue of 1. alpha's current state is already in the inbox
# (accounted), so it is not double-enumerated.
out="$("$RECON" reconcile 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R9: enumerating into a detector-co-fed store must FAIL LOUD (non-zero), not silently alias"
echo "$out" | grep -qi 'dual-allocator' || fail_msg "R9: the failure must name the dual-allocator hazard [$out]"
echo "$out" | grep -q '908' || fail_msg "R9: the failure must reference the #908 follow-up [$out]"
echo "$out" | grep -qi 'REFUSED' || fail_msg "R9: the obligation must be REFUSED/flagged, not swallowed [$out]"
# No collidable seq was written: depth is unchanged, so a later detector delta
# cannot alias a reconciler-enumerated entry.
[ "$(depth)" = "$before" ] || fail_msg "R9: a refused reconcile must NOT enqueue any (aliasable) seq, depth changed $before->$(depth)"
# (That the sole-feeder path DOES still enumerate — i.e. the guard is not a
# blanket no-op — is proven by R5/R6, which enumerate under --allow-enumerate.)
[ "$rc" -ne 0 ] || fail_msg "R9: pre-existing beta is unaccounted on this pass -> must FLAG (non-zero) [$out]"
echo "$out" | grep -qi 'UNACCOUNTED=1' || fail_msg "R9: only beta should be unaccounted (alpha is inbox-accounted) [$out]"
[ "$(depth)" = "2" ] || fail_msg "R9: reconciler must ENUMERATE beta into the co-fed store (depth 2), got $(depth)"
# A further detector delta on beta must allocate 3 — the unified allocator
# advances past the reconciler's seq, it can NEVER reissue/alias 1 or 2.
printf 'BETA2\n' >"$stub/repo_beta"
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector delta (beta) failed"
# THE NO-ALIAS ASSERTION: every observed_seq across BOTH feeders is DISTINCT and
# the observed prefix is CONTIGUOUS/GAPLESS (1,2,3). Under the OLD two-allocator
# design the detector's private counter would have reissued a seq the reconciler
# already used, so this set would contain a duplicate (alias).
seqs="$("$STORE" drain | jq -r '.observed_seq' | sort -n | tr '\n' ' ')"
[ "$seqs" = "1 2 3 " ] || fail_msg "R9: co-fed observed_seqs must be distinct+gapless '1 2 3', got '$seqs' (a duplicate = the aliasing #908 dissolved)"
ndistinct="$("$STORE" drain | jq -r '.observed_seq' | sort -nu | grep -c .)"
ntotal="$("$STORE" drain | jq -r '.observed_seq' | grep -c .)"
[ "$ndistinct" = "$ntotal" ] || fail_msg "R9: aliasing detected — $ntotal entries but only $ndistinct distinct observed_seq"
# The contiguous-prefix CONSUMED contract holds over the co-fed seqs.
"$STORE" consume --upto 3 >/dev/null 2>&1 || fail_msg "R9: CONSUMED 3 must succeed over the gapless co-fed prefix"
) && ok
echo

158
packages/mosaic/framework/tools/wake/test-wake-store-ack.sh
View File

@@ -12,6 +12,11 @@
# T5 atomic write-tmp+rename survives crash mid-write (§1.2)
# T6 durability survives restart (retain until CONSUMED) (§2.3)
# T7 ack local-write NEVER blocks on network (§2.2)
# T8 SOLE store-side allocator: enqueue (no --seq) allocates contiguous
# observed_seq + anti-swallow fail-loud on explicit seq<=consumed (#908)
# T9 burn-before-enqueue: a failed durable write must NOT advance the
# observed_seq cursor (no burned seq / no interior gap) (#908 arrow 1)
# T10 concurrency: two concurrent enqueues get DISTINCT seqs (lock) (#908)
#
# Isolated: every test runs against a fresh WAKE_STATE_HOME temp dir.
set -uo pipefail
@@ -242,11 +247,19 @@ EOF
done
# Background sync deliberately SLOW: proves the ack does not wait for it.
export WAKE_ACK_SYNC_CMD="sleep 3; touch '$WAKE_STATE_HOME/SYNC_DONE'"
start="$(date +%s)"
# Sub-second timing (#918): `date +%s` is SECOND-granularity, so a
# legitimately fast (sub-second) ack that straddles a wall-clock second
# boundary can misreport elapsed=2s and flake under CI load (observed on
# Woodpecker pipeline 2057; 20/20 passes locally elsewhere). Use
# `date +%s.%N` and compare in milliseconds via awk (bash `[ ]` can't do
# float/ms math) against a threshold comfortably below the 3s background
# sync — proving the ack did NOT wait on it — while tolerating a
# legitimate sub-second-to-~1s ack across a boundary.
start="$(date +%s.%N)"
out="$(PATH="$bin:$PATH" "$ACK" consumed --upto 1 --wake-id WID-1)"
end="$(date +%s)"
elapsed=$((end - start))
[ "$elapsed" -lt 2 ] || fail_msg "T7: ack path blocked ${elapsed}s — must return without waiting on the background sync"
end="$(date +%s.%N)"
elapsed_ms="$(awk -v s="$start" -v e="$end" 'BEGIN { printf "%d", (e - s) * 1000 }')"
[ "$elapsed_ms" -lt 1500 ] || fail_msg "T7: ack path blocked ${elapsed_ms}ms — must return without waiting on the background sync"
echo "$out" | grep -q 'CONSUMED 1' || fail_msg "T7: CONSUMED not reported [$out]"
# Local write is durable IMMEDIATELY (no network needed to record the ack).
STATE_DIR="$WAKE_STATE_HOME/default"
@@ -261,6 +274,143 @@ EOF
true
) && ok
echo "== T8: SOLE store-side allocator — enqueue (no --seq) allocates contiguous seqs + anti-swallow fail-loud (#908) =="
(
WAKE_STATE_HOME="$(fresh_state t8)"
export WAKE_STATE_HOME
unset WAKE_AGENT
# No --seq: the STORE allocates and PRINTS the assigned observed_seq.
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
s2="$("$STORE" enqueue --class actionable --locators '{"n":2}')"
[ "$s1" = "1" ] || fail_msg "T8: first store-allocated observed_seq must be 1, got '$s1'"
[ "$s2" = "2" ] || fail_msg "T8: second store-allocated observed_seq must be 2 (contiguous), got '$s2'"
cur="$("$STORE" cursors)"
echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T8: observed_seq cursor should be 2 [$cur]"
echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T8: both allocations must be durably enqueued [$cur]"
# ANTI-SWALLOW: consume the prefix, then a LEGACY explicit --seq inside the
# consumed prefix must FAIL LOUD (never the old silent seq<=consumed no-op).
"$STORE" consume --upto 2 >/dev/null
before_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
if "$STORE" enqueue --seq 1 --class actionable --locators '{}' >/dev/null 2>&1; then
fail_msg "T8: explicit --seq 1 <= consumed_seq 2 must FAIL LOUD (anti-swallow), not be a silent no-op"
fi
err="$("$STORE" enqueue --seq 1 --class actionable --locators '{}' 2>&1 || true)"
echo "$err" | grep -qi 'anti-swallow' || fail_msg "T8: the refusal must name the anti-swallow guarantee [$err]"
after_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
[ "$before_depth" = "$after_depth" ] || fail_msg "T8: a refused enqueue must not mutate the store (depth $before_depth->$after_depth)"
# And the NEXT real allocation is > consumed (2) — never inside the prefix.
s3="$("$STORE" enqueue --class actionable --locators '{"n":3}')"
[ "$s3" = "3" ] || fail_msg "T8: post-consume allocation must be 3 (>consumed 2), got '$s3'"
) && ok
echo "== T9: burn-before-enqueue — a failed durable write must NOT advance observed_seq (no burned seq / no gap) (#908 arrow 1) =="
(
WAKE_STATE_HOME="$(fresh_state t9)"
export WAKE_STATE_HOME
unset WAKE_AGENT
STATE_DIR="$WAKE_STATE_HOME/default"
# One successful allocation: observed_seq=1, and the enqueue lock now exists.
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
[ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'"
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
[ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'"
# Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger:
# bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's
# exact rename TARGET for the durable write — see _wake-common.sh) so the
# atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"):
# the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is
# a structural VFS constraint, not a DAC permission check — root does NOT
# bypass it (no uid can rename over a live mountpoint short of an explicit
# umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a
# CI container running as root BYPASSES DAC checks entirely, so the
# "read-only" dir let mktemp/the write through and every assertion below
# went spuriously green. The old chmod also blocked the CURSOR's own
# mktemp (same dir), so it could never have caught a premature cursor
# commit either — narrower coverage than its comment claimed. Here
# observed_seq is left an ordinary writable file, never touched by the
# mount, so a regression that commits the cursor before/independent of the
# durable write is still genuinely caught by the assertions below.
#
# The mount + forced enqueue run inside a private mount+user namespace
# (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL
# whether this harness runs unprivileged (dev) or as root (CI): creating a
# user namespace needs no pre-existing privilege, and the busy-mountpoint
# rule inside it is the real kernel rule, unaffected by uid in or out of a
# namespace. The bind mount is entirely private to that namespace and
# vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain
# writable file again for every assertion below — no manual cleanup step.
rc=0
if ! command -v unshare >/dev/null 2>&1; then
fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
rc=99
else
ro_src="$TMP_ROOT/t9-ro-src"
mkdir -p "$ro_src"
: >"$ro_src/pending.jsonl"
# shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED
# bash (inside the unshare'd namespace), not this outer single-quoted one.
unshare --mount --user --map-root-user bash -c '
set -u
ro_src="$1"
state_dir="$2"
store="$3"
if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then
exit 98
fi
mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1
' _ "$ro_src" "$STATE_DIR" "$STORE"
rc=$?
if [ "$rc" -eq 98 ]; then
fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed"
fi
fi
[ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)"
# THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was
# burned by a write that never reached the store. (RED if the cursor bumps
# before/independent of the durable write.)
obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
[ "$obs_after" = "1" ] || fail_msg "T9: a failed enqueue must NOT advance observed_seq (burned seq -> permanent gap), got '$obs_after'"
# No interior gap was created: CONSUMED 1 (the only real obligation) still holds.
"$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T9: CONSUMED 1 must remain valid — no burned-seq gap"
# And a fresh allocation resumes cleanly at 2 (not 3 — nothing was burned).
s="$("$STORE" enqueue --class actionable --locators '{"n":"2b"}')"
[ "$s" = "2" ] || fail_msg "T9: post-failure allocation must resume at 2 (no burned seq), got '$s'"
) && ok
echo "== T10: concurrency — two concurrent enqueues get DISTINCT seqs (enqueue lock) (#908) =="
if command -v flock >/dev/null 2>&1; then
(
WAKE_STATE_HOME="$(fresh_state t10)"
export WAKE_STATE_HOME
unset WAKE_AGENT
"$STORE" init >/dev/null 2>&1 || true
o1="$TMP_ROOT/t10.o1"
o2="$TMP_ROOT/t10.o2"
# Two enqueues racing on the SAME store. Without the lock both read
# observed_seq=0 and allocate 1 (aliasing + a lost write); the lock serializes
# the allocate+write so they get 1 and 2.
"$STORE" enqueue --class actionable --locators '{"c":1}' >"$o1" 2>/dev/null &
p1=$!
"$STORE" enqueue --class actionable --locators '{"c":2}' >"$o2" 2>/dev/null &
p2=$!
wait "$p1"; wait "$p2"
a="$(tr -d '[:space:]' <"$o1")"
b="$(tr -d '[:space:]' <"$o2")"
[ -n "$a" ] && [ -n "$b" ] || fail_msg "T10: both concurrent enqueues must print an allocated seq (got '$a','$b')"
[ "$a" != "$b" ] || fail_msg "T10: two concurrent enqueues must get DISTINCT seqs, both got '$a' (aliasing / lost write without the lock)"
cur="$("$STORE" cursors)"
echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T10: observed_seq must reach 2 after two enqueues [$cur]"
echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T10: both entries must be durably stored (no lost write) [$cur]"
# The two allocated seqs are exactly {1,2} (distinct, contiguous, gapless).
lo="$a"; hi="$b"; [ "$a" -gt "$b" ] && { lo="$b"; hi="$a"; }
{ [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T10: concurrent seqs must be {1,2}, got {$lo,$hi}"
) && ok
else
echo " SKIP: flock not available (concurrency guarantee needs flock; matches detector single-instance SKIP)"
ok
fi
echo
if [ -s "$FAILFILE" ]; then
echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2