mosaicstack/stack
1
0
Fork 0
Code Issues 90 Pull Requests Actions Packages Projects Releases 15 Wiki Activity

feat(wake): W5 — synthetic-canary FN-oracle + source-parity reconciler (#909)
All checks were successful
ci/woodpecker/push/publish Pipeline was successful
Details
ci/woodpecker/push/ci Pipeline was successful
Details

Browse Source 
Co-authored-by: jason.woltje <jason@diversecanvas.com>
Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #909.
This commit is contained in:
jason.woltje
2026-07-26 02:04:19 +00:00
committed by Mos
parent 5df47e735e
commit 320f5bfb6f
6 changed files with 1346 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

352
packages/mosaic/framework/tools/wake/fn-oracle.sh Executable file
View File

@@ -0,0 +1,352 @@
#!/usr/bin/env bash
# fn-oracle.sh — A6 of the wake canon (EPIC #892, W5): the FN-oracle /
# synthetic-canary.
#
# CONTRACT ANCHORS (docs/scratchpads/heartbeat-planning/CONVERGED-DESIGN.md):
# §4 vector { synthetic-canary FN-rate = 0 } — the oracle injects a KNOWN
# synthetic-canary delta into the wake pipeline and ASSERTS it
# reaches CONSUMED within its per-class SLO; it measures the
# FALSE-NEGATIVE rate. FN-rate MUST be 0 for the timer to retire.
# §4/§7-res5 The FN-oracle is a real, NON-OPTIONAL operating cost — the price
# of being allowed to retire the fixed timer, not a free extra.
# §1.4 Framework component, operator-agnostic (~/.config/mosaic/tools/).
#
# OFF-DOMAIN / DETECTOR-INDEPENDENT (the §4/A8 false-negative-blindspot killer):
# The injection and the CONSUMED-assertion do NOT depend on the detector's
# own hashing / poll internals. The oracle injects a delta at the SOURCE
# boundary (the bytes an operator adapter reports), drives the pipeline
# through the detector's PUBLIC API only (poll-once — a black box), and then
# renders its verdict SOLELY from the terminal store/ack state (did a CONSUMED
# ack advance the cursor to cover the canary's observed_seq?). It never reads a
# detector hash-file, poll counter, or self-report. CONSEQUENCE: a detector
# that silently DROPS changes — including a FULLY-DISABLED detector at a
# "perfect" no-op rate (0 wakes/day, which looks great on the wakes metric) —
# still FAILS the oracle, because the canary never reaches CONSUMED. That is
# the whole point: the success metric is a VECTOR, not a wake-count.
#
# ISOLATION: the probe runs in its OWN XDG state namespace (WAKE_ORACLE_HOME),
# NOT the operator's live queue, so a synthetic canary never delivers a fake
# wake to a real agent. It exercises the REAL detector.sh / store.sh / ack.sh
# CODE paths against isolated synthetic data — the standard synthetic-canary
# shape (synthetic transaction, real code, isolated data).
#
# SCOPE: calls the PUBLIC APIs of detector.sh (poll-once), store.sh
# (cursors/drain), and ack.sh (received/consumed). It does NOT reimplement or
# modify any of them.
#
# Operator-agnostic: all state via XDG/env; no operator paths/names/secrets.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./_wake-common.sh disable=SC1091
. "$SCRIPT_DIR/_wake-common.sh"
DETECTOR_SH="$SCRIPT_DIR/detector.sh"
STORE_SH="$SCRIPT_DIR/store.sh"
ACK_SH="$SCRIPT_DIR/ack.sh"
# Isolated oracle state root (XDG). NEVER the operator's live wake queue.
ORACLE_HOME="${WAKE_ORACLE_HOME:-${XDG_STATE_HOME:-$HOME/.local/state}/mosaic/wake/fn-oracle}"
# The canary's own store namespace lives UNDER the oracle root.
CANARY_STATE_HOME="$ORACLE_HOME/canary-store"
CANARY_AGENT="canary"
CANARY_SRC="$ORACLE_HOME/canary.src"
CANARY_ADAPTER="$ORACLE_HOME/canary-adapter.sh"
CANARY_WATCHLIST="$ORACLE_HOME/canary-watch-list.json"
METRICS="$ORACLE_HOME/metrics.jsonl"
_need_jq() {
command -v jq >/dev/null 2>&1 || {
echo "fn-oracle.sh: jq is required" >&2
exit 3
}
}
usage() {
cat >&2 <<'EOF'
Usage: fn-oracle.sh run --slo-seconds N [options]
Commands:
run --slo-seconds N [--count K] [--class C]
Run K synthetic-canary cycles (default 1). Each cycle injects a KNOWN
delta at the source boundary, drives the pipeline through the detector's
public poll-once, then asserts the canary reaches CONSUMED within the
per-class SLO (N seconds). Emits per-canary PASS/FN-DETECTED, the
FN-rate metric, and an overall VERDICT. Exit 0 IFF FN-rate == 0.
status
Print the tail of the recorded metrics ledger.
Options:
--slo-seconds N REQUIRED for run. The per-class SLO (operator-tuned; symbolic
tiers per §4 — no default is invented). event->CONSUMED must
land within N seconds or the canary counts as a false-negative.
--count K Number of canary cycles this run (default 1).
--class C Wake class the canary flows as (default digest — the primary
machine-wake path §4 certifies). digest|actionable|human.
Environment:
WAKE_ORACLE_HOME Isolated oracle state root (XDG by default). NEVER the
operator's live queue.
WAKE_ORACLE_DETECTOR_CMD Command the oracle runs to advance ONE observe cycle
of the pipeline (default: detector.sh poll-once). The
canary env (watch-list/adapter/state) is exported to
it. Pluggable so a disabled/dropping detector can be
exercised (it MUST then be caught as FN-DETECTED).
WAKE_ORACLE_SLO_SECONDS Fallback for --slo-seconds.
EOF
}
# --- one-time synthetic-canary scaffold (idempotent) -----------------------
# The oracle SUPPLIES its own source adapter + watch-list so it needs no
# operator config and stays fully self-contained. The adapter simply echoes the
# current canary source bytes — the synthetic "source of truth" the oracle
# controls end-to-end.
_write_scaffold() {
local class="$1"
mkdir -p "$ORACLE_HOME"
cat >"$CANARY_ADAPTER" <<EOF
#!/usr/bin/env bash
# Synthetic-canary source adapter (fn-oracle). Reports the current canary bytes.
set -u
[ -f "$CANARY_SRC" ] && cat "$CANARY_SRC"
exit 0
EOF
chmod +x "$CANARY_ADAPTER"
# A minimal, schema-valid watch-list with EXACTLY ONE synthetic source, no
# anchor. One source in an isolated store => any enqueue is unambiguously the
# canary. The class is carried so the canary flows on the class under test.
jq -n --arg class "$class" '{
schema_version: 1,
repos: [ { id: "fn-canary", remote: "synthetic/fn-oracle-canary", class: $class } ],
watches: [ { lane: "fn-oracle", sources: [ { kind: "repo", id: "fn-canary" } ] } ]
}' >"$CANARY_WATCHLIST"
}
# _canary_env — export the isolated canary namespace + adapter + watch-list for
# a child detector/store/ack invocation.
_canary_env() {
export WAKE_STATE_HOME="$CANARY_STATE_HOME"
export WAKE_AGENT="$CANARY_AGENT"
export WAKE_WATCH_LIST="$CANARY_WATCHLIST"
export WAKE_DETECTOR_SOURCE_CMD="$CANARY_ADAPTER"
}
# _store_observed / _store_consumed — read the terminal cursors (public API).
_store_observed() { "$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p'; }
_store_consumed() { "$STORE_SH" cursors 2>/dev/null | sed -n 's/^consumed_seq=//p'; }
# _drive_pipeline — advance ONE observe cycle. Black-box: whatever
# WAKE_ORACLE_DETECTOR_CMD is (default: the real detector's public poll-once).
# A source failure inside the detector is loud there; the oracle judges only the
# terminal state, so a drop of ANY kind surfaces as an unmet CONSUMED.
_drive_pipeline() {
local cmd="${WAKE_ORACLE_DETECTOR_CMD:-"$DETECTOR_SH poll-once"}"
sh -c "$cmd" >/dev/null 2>&1 || true
}
# _seat_baseline — seat the source's first-seen baseline ONCE (detector
# first-seen is silent by design). Every subsequent canary token is then a
# genuine DELTA off the prior token, so no spurious baseline wakes are produced.
_seat_baseline() {
_canary_env
printf '%s' "FN-ORACLE-BASELINE-$$-$(date +%s)" >"$CANARY_SRC"
_drive_pipeline
}
# --- one canary cycle -------------------------------------------------------
# Returns 0 = PASS (reached CONSUMED within SLO), 1 = FN-DETECTED.
# Prints a one-line verdict for the cycle.
_run_one() {
local idx="$1" slo="$2" class="$3"
_canary_env
local nonce token inject_ts
nonce="$$-$idx-$(date +%s%N 2>/dev/null || date +%s)"
token="FN-ORACLE-CANARY-$nonce"
local obs_before
obs_before="$(_store_observed)"
case "$obs_before" in '' | *[!0-9]*) obs_before=0 ;; esac
# INJECT the known delta at the source boundary + stamp the clock.
inject_ts="$(date +%s)"
printf '%s' "$token" >"$CANARY_SRC"
# Drive the pipeline (through the detector's public API — a black box).
_drive_pipeline
local obs_after
obs_after="$(_store_observed)"
case "$obs_after" in '' | *[!0-9]*) obs_after=0 ;; esac
# ---- OFF-DOMAIN verdict, part 1: was the delta OBSERVED at all? ----------
# In an isolated single-source store, any observed_seq advance is the canary.
# NO advance => the pipeline (a dropping / disabled detector) swallowed a KNOWN
# real change => FALSE NEGATIVE. This is the killer: a perfect no-op rate does
# not rescue a detector that fails to see a real delta.
if [ "$obs_after" -le "$obs_before" ]; then
printf ' canary %s: FN-DETECTED (injected delta never OBSERVED — pipeline dropped a known change; observed_seq %s unchanged)\n' \
"$idx" "$obs_before" >&2
return 1
fi
# The delta was observed and durably enqueued. Simulate the consumer contract
# (§2.2): deliver (drain), RECEIVED-ack, capture, then CONSUMED-ack the
# contiguous prefix. All via the PUBLIC ack/store API.
local delivered
delivered="$("$STORE_SH" drain 2>/dev/null | grep -c '.' || echo 0)"
if [ "${delivered:-0}" -lt 1 ]; then
printf ' canary %s: FN-DETECTED (observed but NOT delivered — drain returned nothing to consume)\n' "$idx" >&2
return 1
fi
"$ACK_SH" received --wake-id "fn-oracle-$nonce" >/dev/null 2>&1 || true
# CONSUMED over the contiguous prefix up to the canary's observed_seq.
"$ACK_SH" consumed --upto "$obs_after" --no-sync >/dev/null 2>&1 || true
# ---- OFF-DOMAIN verdict, part 2: did it reach CONSUMED, within SLO? ------
local consumed elapsed now
consumed="$(_store_consumed)"
case "$consumed" in '' | *[!0-9]*) consumed=0 ;; esac
now="$(date +%s)"
elapsed=$((now - inject_ts))
if [ "$consumed" -lt "$obs_after" ]; then
printf ' canary %s: FN-DETECTED (delivered but consumed_seq=%s never reached the canary observed_seq=%s)\n' \
"$idx" "$consumed" "$obs_after" >&2
return 1
fi
if [ "$elapsed" -gt "$slo" ]; then
printf ' canary %s: FN-DETECTED (reached CONSUMED but in %ss > per-class SLO %ss)\n' \
"$idx" "$elapsed" "$slo" >&2
return 1
fi
printf ' canary %s: PASS (event->CONSUMED in %ss <= SLO %ss; observed_seq=%s consumed_seq=%s class=%s)\n' \
"$idx" "$elapsed" "$slo" "$obs_after" "$consumed" "$class"
return 0
}
cmd_run() {
_need_jq
local slo="${WAKE_ORACLE_SLO_SECONDS:-}" count=1 class="digest"
while [ $# -gt 0 ]; do
case "$1" in
--slo-seconds)
slo="${2:-}"
shift 2
;;
--count)
count="${2:-}"
shift 2
;;
--class)
class="${2:-}"
shift 2
;;
*)
echo "fn-oracle.sh run: unknown option '$1'" >&2
exit 2
;;
esac
done
# No invented numeric SLO (design law): the per-class SLO MUST be supplied.
case "$slo" in
'' | *[!0-9]*)
echo "fn-oracle.sh run: --slo-seconds (or WAKE_ORACLE_SLO_SECONDS) is REQUIRED and must be a non-negative integer (per-class SLO is operator-tuned; no default is invented)" >&2
exit 2
;;
esac
case "$count" in
'' | *[!0-9]* | 0)
echo "fn-oracle.sh run: --count must be a positive integer" >&2
exit 2
;;
esac
case "$class" in
digest | actionable | human) : ;;
*)
echo "fn-oracle.sh run: --class must be one of digest|actionable|human" >&2
exit 2
;;
esac
# Fresh isolated probe state each run => a deterministic baseline. This is the
# oracle's OWN synthetic store, never the operator's live queue.
rm -rf "$CANARY_STATE_HOME"
mkdir -p "$ORACLE_HOME"
_write_scaffold "$class"
echo "fn-oracle: synthetic-canary run (count=$count, class=$class, per-class SLO=${slo}s)"
_seat_baseline # one-time first-seen baseline (silent by design)
local fn=0 i
i=1
while [ "$i" -le "$count" ]; do
if _run_one "$i" "$slo" "$class"; then
:
else
fn=$((fn + 1))
fi
i=$((i + 1))
done
# FN-rate metric. §4 requires FN-rate == 0.
local rate verdict rc
if [ "$fn" -eq 0 ]; then
verdict="PASS"
rc=0
else
verdict="FN-DETECTED"
rc=1
fi
# Rational FN-rate rendered without bc (portable): integer numerator/denominator
# plus a scaled decimal.
rate="$(awk -v f="$fn" -v n="$count" 'BEGIN { printf "%.4f", (n>0? f/n : 0) }')"
local rec
rec="$(jq -cn \
--argjson ts "$(date +%s)" \
--arg class "$class" \
--argjson slo "$slo" \
--argjson count "$count" \
--argjson fn "$fn" \
--arg rate "$rate" \
--arg verdict "$verdict" \
'{ts:$ts, class:$class, slo_seconds:$slo, count:$count, fn:$fn, fn_rate:($rate|tonumber), verdict:$verdict}')"
{ cat "$METRICS" 2>/dev/null; printf '%s\n' "$rec"; } | grep -v '^[[:space:]]*$' >"$METRICS.tmp" 2>/dev/null || true
mv -f "$METRICS.tmp" "$METRICS" 2>/dev/null || true
echo "fn-oracle: FN-RATE = $fn/$count = $rate (class=$class, SLO=${slo}s)"
echo "fn-oracle: VERDICT = $verdict"
[ "$rc" -eq 0 ] || echo "fn-oracle: §4 vector FAILS — synthetic-canary FN-rate must be 0 to retire the timer." >&2
return "$rc"
}
cmd_status() {
echo "# fn-oracle metrics (tail)"
tail -n 10 "$METRICS" 2>/dev/null || echo "(no runs recorded)"
}
main() {
[ $# -ge 1 ] || {
usage
exit 2
}
local cmd="$1"
shift
case "$cmd" in
run) cmd_run "$@" ;;
status) cmd_status "$@" ;;
-h | --help | help) usage ;;
*)
echo "fn-oracle.sh: unknown command '$cmd'" >&2
usage
exit 2
;;
esac
}
main "$@"

15
packages/mosaic/framework/tools/wake/manifest.txt
View File

@@ -14,8 +14,9 @@
# 0.1.0 W2 — store+drain lib + ack-wrapper. # 0.1.0 W2 — store+drain lib + ack-wrapper.
# 0.2.0 W3 — cumulative-state digest renderer + non-circular HMAC signer. # 0.2.0 W3 — cumulative-state digest renderer + non-circular HMAC signer.
# 0.3.0 W4 — per-host single-instance delta-gated detector daemon. # 0.3.0 W4 — per-host single-instance delta-gated detector daemon.
# 0.4.0 W5 — synthetic-canary FN-oracle + source-parity reconciler.
component=wake component=wake
version=0.3.0 version=0.4.0
# Watch-list schema this component consumes, and the INCLUSIVE range of # Watch-list schema this component consumes, and the INCLUSIVE range of
# schema_version values it supports. A wake-watch-list.json whose schema_version # schema_version values it supports. A wake-watch-list.json whose schema_version
@@ -35,5 +36,13 @@ schema_max=1
# detector.sh A1 — per-host single-instance delta-gated detector daemon # detector.sh A1 — per-host single-instance delta-gated detector daemon
# (flock, anchor-scoped hashing, detector-local observed_seq, # (flock, anchor-scoped hashing, detector-local observed_seq,
# fail-loud source semantics; enqueues deltas to store.sh). (W4) # fail-loud source semantics; enqueues deltas to store.sh). (W4)
# Out of scope (later waves): FN-oracle/reconciler (W5), beacon (W6), # fn-oracle.sh A6 — synthetic-canary FN-oracle: injects a KNOWN delta at the
# installer (W7). # source boundary, drives the pipeline through the detector's
# public poll-once, asserts CONSUMED within the per-class SLO
# (off-domain verdict from the terminal store cursor). §4
# requires FN-rate=0; a dropping/disabled detector FAILS. (W5)
# reconcile.sh A7 — source-parity reconciler: (i) source-coverage parity
# inventory (an omitted source cannot pass the vector
# vacuously) + (ii) periodic full reconcile to 0-unaccounted,
# enumerating pre-existing/startup state into the store. (W5)
# Out of scope (later waves): off-host beacon (W6), installer (W7).

568
packages/mosaic/framework/tools/wake/reconcile.sh Executable file
View File

@@ -0,0 +1,568 @@
#!/usr/bin/env bash
# reconcile.sh — A7 of the wake canon (EPIC #892, W5): the source-parity
# reconciler.
#
# CONTRACT ANCHORS (docs/scratchpads/heartbeat-planning/CONVERGED-DESIGN.md):
# §4/G3 Source parity / reconciliation — the GO condition has TWO ordered
# parts:
# (i) a source-coverage PARITY INVENTORY FIRST: a lane-by-lane
# DECLARED inventory of every operational source the lane
# depends on, so an OMITTED source cannot make the §4 vector pass
# VACUOUSLY. An incomplete / undeclared inventory FLAGS (it must
# NOT silently pass).
# (ii) then a periodic full RECONCILE: enumerate declared+configured
# source state vs observed_seq/inbox => 0 UNACCOUNTED. Any source
# state not reflected in observed_seq/inbox is a gap => FLAG.
# §4 vector { source-parity inventory complete AND reconcile = 0 unaccounted }
# §7-res5 FN-oracle + reconciliation are a real, NON-OPTIONAL operating cost.
# W4 review division: the detector's first-seen-baseline does NOT wake; the
# RECONCILER is what enumerates PRE-EXISTING / startup source state into
# the durable store (at startup and periodically). Without it, state
# that already existed when the detector first saw it (baselined
# silently) would never enter the inbox and would be lost.
#
# SCOPE: calls the PUBLIC APIs of store.sh (cursors/drain/enqueue) only. It does
# NOT reimplement or modify the store, ack, digest, signer, or detector. It
# OBSERVES current source state through the SAME operator adapter contract the
# detector uses (WAKE_DETECTOR_SOURCE_CMD <kind> <id>, def on stdin), with the
# SAME fail-loud (G2a) discipline: a source error / ambiguous-empty is never
# silently "no state".
#
# CONTRACT NOTES (flagged, not silently guessed — see PR body):
# * observed_seq DUAL-ALLOCATOR HAZARD (fail-closed; #908 tracks the fix).
# observed_seq has TWO INDEPENDENT allocators today: the detector's PRIVATE
# counter ($STATE_DIR/detector/observed_seq_counter, W4) and — if this
# reconciler enumerated — the STORE cursor (observed_seq+offset). Because the
# detector's next seq is (private_counter+1), NOT (store_observed_seq+1), the
# two are decoupled and SERIALIZING THEM DOES NOT PREVENT COLLISION: after
# the reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2,
# detector private still 0), the detector's next delta allocates private
# 0->1 and enqueues seq1 — ALIASING alpha. A consumer acking CONSUMED 1
# (trusting §2.4's unique observed_seq) then silently DROPS a distinct
# obligation — the exact G3 swallow this reconciler exists to prevent.
# THEREFORE, until #908 unifies on a single store-side allocator, the
# reconciler FAILS CLOSED: it REFUSES to enumerate (loud, non-zero; the
# obligation is FLAGGED, never silently swallowed) whenever a detector is
# co-feeding this store, and otherwise requires the operator to assert
# reconciler-SOLE-FEEDER mode. OPERATIONAL GUARD: a detector and this
# reconciler MUST NOT co-feed one store until #908 lands.
# * "accounted" is judged against observed_seq/inbox (§4/G3 wording) PLUS the
# reconciler's OWN durable reconciled-state ledger — NOT the detector's
# hash-file. Judging by the detector baseline would let a silent first-seen
# baseline swallow a pre-existing obligation (the exact hole G3 closes).
# * enumeration uses the non-coalescible fail-safe class `actionable` (§2.3)
# so two distinct pre-existing obligations can never coalesce into one.
#
# Operator-agnostic: all state via XDG/env; no operator paths/names/secrets.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./_wake-common.sh disable=SC1091
. "$SCRIPT_DIR/_wake-common.sh"
STORE_SH="$SCRIPT_DIR/store.sh"
MANIFEST="$SCRIPT_DIR/manifest.txt"
STATE_DIR="$(wake_state_dir)"
# Reconciler-local state (its durable reconciled-state ledger) lives in its own
# subdir under the store's STATE_DIR so it never collides with store/detector.
RECON_DIR="$STATE_DIR/reconciler"
# The detector's PRIVATE state dir (W4). Its presence signals a detector is (or
# has been) allocating observed_seq into THIS store from an INDEPENDENT private
# counter — the co-feeding condition under which reconciler enumeration is
# unsafe (see the dual-allocator hazard note above; #908). Read-only coupling:
# reconcile.sh never writes or imports W4, it only OBSERVES this path to fail
# closed rather than silently alias a seq.
DET_STATE="$STATE_DIR/detector"
_need_jq() {
command -v jq >/dev/null 2>&1 || {
echo "reconcile.sh: jq is required" >&2
exit 3
}
}
_hash() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum | awk '{print $1}'
elif command -v shasum >/dev/null 2>&1; then
shasum -a 256 | awk '{print $1}'
elif command -v openssl >/dev/null 2>&1; then
openssl dgst -sha256 | awk '{print $NF}'
else
echo "reconcile.sh: no sha256 tool (sha256sum/shasum/openssl) available" >&2
exit 3
fi
}
usage() {
cat >&2 <<'EOF'
Usage: reconcile.sh <command> [options]
Commands:
inventory
Part (i) — source-coverage PARITY INVENTORY. Prints a lane-by-lane
declared inventory and FLAGS any incompleteness that could let the §4
vector pass VACUOUSLY:
* a declared operational source (repos/board_files/lane_anchors) that
NO watch covers (an OMITTED source);
* a watch that references a source with no declared definition (dangling);
* a lane with zero sources; an empty watch-list;
* if a lane declares `required_sources`, any id missing from its sources.
Exit 0 IFF the inventory is complete; non-zero (loud) on any flag.
reconcile [--no-enumerate] [--allow-enumerate]
Part (ii) — periodic full reconcile. Enumerates each covered source's
CURRENT state and compares it against observed_seq/inbox (+ the
reconciler's reconciled-state ledger). Any source state NOT reflected =>
UNACCOUNTED => FLAG. Unless --no-enumerate, each unaccounted source is
ENUMERATED into the durable store (the startup/pre-existing obligation
path). Exit 0 IFF 0 unaccounted were FOUND this pass.
DUAL-ALLOCATOR SAFETY (fail-closed; #908): enumeration WRITES observed_seq
from an allocator INDEPENDENT of the detector's private counter, so it
FAILS LOUD (refuses; the obligation is FLAGGED, never swallowed) when it
would risk aliasing a seq:
* if a detector is co-feeding this store (its private state dir exists)
=> refuse, ALWAYS (even with the ack below);
* else the operator MUST assert reconciler-SOLE-FEEDER mode via
--allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1), promising no
detector co-feeds this store until #908 lands.
--no-enumerate (and `check`) never write, so they are always safe.
check
The full §4/G3 gate, side-effect-free: `inventory` AND
`reconcile --no-enumerate`. Exit 0 IFF the inventory is complete AND 0
unaccounted. Use this to evaluate the vector's G3 clause without writing.
Environment:
WAKE_WATCH_LIST Operator watch-list JSON (required).
WAKE_DETECTOR_SOURCE_CMD Source adapter (required for reconcile/check; same
contract as the detector: <cmd> <kind> <id>, def on
stdin; exit!=0 or empty => FAIL LOUD).
WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh).
WAKE_RECONCILE_ALLOW_ENUMERATE set to 1 to assert reconciler-SOLE-FEEDER mode
(see reconcile's dual-allocator safety note; #908).
EOF
}
_manifest_val() {
local key="$1"
[ -f "$MANIFEST" ] || return 0
sed -n "s/^${key}=//p" "$MANIFEST" | head -n1 | tr -d '[:space:]'
}
# _load_watchlist — validate path + JSON + shape + Gate B schema range (mirrors
# the detector's fail-loud validation so the reconciler certifies the SAME
# watch-list the detector runs). Echoes validated JSON on stdout.
_load_watchlist() {
_need_jq
local wl="${WAKE_WATCH_LIST:-}"
if [ -z "$wl" ]; then
echo "reconcile.sh: WAKE_WATCH_LIST is not set (no watch-list to reconcile)" >&2
return 2
fi
if [ ! -f "$wl" ]; then
echo "reconcile.sh: watch-list not found: $wl" >&2
return 2
fi
local json
if ! json="$(jq -e . "$wl" 2>/dev/null)"; then
echo "reconcile.sh: watch-list is not valid JSON: $wl" >&2
return 2
fi
if ! printf '%s' "$json" | jq -e 'has("schema_version") and has("watches")' >/dev/null 2>&1; then
echo "reconcile.sh: watch-list missing required 'schema_version' or 'watches'" >&2
return 2
fi
local ver smin smax
ver="$(printf '%s' "$json" | jq -r '.schema_version')"
smin="$(_manifest_val schema_min)"
smax="$(_manifest_val schema_max)"
case "$ver" in
'' | *[!0-9]*)
echo "reconcile.sh: watch-list schema_version must be an integer (got '$ver')" >&2
return 2
;;
esac
if [ -z "$smin" ] || [ -z "$smax" ]; then
echo "reconcile.sh: manifest is missing schema_min/schema_max (cannot validate Gate B)" >&2
return 2
fi
if [ "$ver" -lt "$smin" ] || [ "$ver" -gt "$smax" ]; then
echo "reconcile.sh: FAIL LOUD (Gate B) — watch-list schema_version $ver is OUTSIDE the supported range [$smin, $smax]" >&2
return 2
fi
printf '%s' "$json"
}
# _coll_for_kind KIND — the top-level collection a source kind is defined in.
_coll_for_kind() {
case "$1" in
repo) printf 'repos' ;;
board_file) printf 'board_files' ;;
lane_anchor) printf 'lane_anchors' ;;
*) return 1 ;;
esac
}
# --- part (i): source-coverage parity inventory ----------------------------
cmd_inventory() {
local json
json="$(_load_watchlist)" || exit $?
local flags=0
# An empty watch-list can never certify coverage of anything.
local nwatch
nwatch="$(printf '%s' "$json" | jq -r '.watches | length')"
if [ "${nwatch:-0}" -eq 0 ]; then
echo "reconcile.sh: FLAG (G3 parity) — watch-list declares NO watches; coverage is vacuous." >&2
flags=$((flags + 1))
fi
echo "# source-coverage parity inventory (lane-by-lane)"
# Per-lane declared inventory + empty-lane + required_sources checks.
local nlanes li
nlanes="$(printf '%s' "$json" | jq -r '.watches | length')"
li=0
while [ "$li" -lt "${nlanes:-0}" ]; do
local lane nsrc
lane="$(printf '%s' "$json" | jq -r --argjson i "$li" '.watches[$i].lane // "(unnamed)"')"
nsrc="$(printf '%s' "$json" | jq -r --argjson i "$li" '.watches[$i].sources | length')"
printf ' lane %s: %s declared source(s)\n' "$lane" "${nsrc:-0}"
if [ "${nsrc:-0}" -eq 0 ]; then
echo "reconcile.sh: FLAG (G3 parity) — lane '$lane' declares ZERO sources (empty coverage)." >&2
flags=$((flags + 1))
fi
# required_sources (optional, additive; ignored by the detector). If the
# operator declares the lane's true dependency set here, every id MUST be
# covered by the lane's sources — an OMITTED dependency FLAGS.
local missing
missing="$(printf '%s' "$json" | jq -r --argjson i "$li" '
(.watches[$i].required_sources // []) as $req
| ($req - [ .watches[$i].sources[].id ]) | .[]')"
if [ -n "$missing" ]; then
local m
while IFS= read -r m; do
[ -n "$m" ] || continue
echo "reconcile.sh: FLAG (G3 parity) — lane '$lane' requires source '$m' but it is OMITTED from the lane's watched sources (vacuous-pass prevented)." >&2
flags=$((flags + 1))
done <<EOF
$missing
EOF
fi
li=$((li + 1))
done
# Dangling: every watched (kind,id) MUST have a declared definition.
local pairs kind id
pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')"
while IFS=$'\t' read -r kind id; do
[ -n "$kind" ] || continue
local coll def
if ! coll="$(_coll_for_kind "$kind")"; then
echo "reconcile.sh: FLAG (G3 parity) — unknown source kind '$kind' in a watch." >&2
flags=$((flags + 1))
continue
fi
def="$(printf '%s' "$json" | jq -c --arg c "$coll" --arg id "$id" '(.[$c] // []) | map(select(.id == $id)) | .[0] // empty')"
if [ -z "$def" ]; then
echo "reconcile.sh: FLAG (G3 parity) — watch references '$kind/$id' but no such entry is declared in '$coll' (dangling)." >&2
flags=$((flags + 1))
fi
done <<EOF
$pairs
EOF
# Omitted-from-coverage: every DECLARED operational source must be covered by
# at least one watch. A declared-but-unwatched source is exactly the OMITTED
# source that would let the vector pass VACUOUSLY (the detector never checks it).
local coll kinds
kinds="repos:repo board_files:board_file lane_anchors:lane_anchor"
for coll in $kinds; do
local cname ckind ids
cname="${coll%%:*}"
ckind="${coll##*:}"
ids="$(printf '%s' "$json" | jq -r --arg c "$cname" '(.[$c] // [])[].id')"
local sid covered
while IFS= read -r sid; do
[ -n "$sid" ] || continue
covered="$(printf '%s' "$json" | jq -r --arg k "$ckind" --arg id "$sid" \
'[ .watches[].sources[] | select(.kind==$k and .id==$id) ] | length')"
if [ "${covered:-0}" -eq 0 ]; then
echo "reconcile.sh: FLAG (G3 parity) — declared source '$ckind/$sid' is NOT covered by any watch (OMITTED from coverage; vacuous-pass prevented)." >&2
flags=$((flags + 1))
fi
done <<EOF
$ids
EOF
done
if [ "$flags" -ne 0 ]; then
echo "reconcile.sh: source-parity inventory INCOMPLETE — $flags flag(s). The §4 vector's G3 clause does NOT pass." >&2
return 1
fi
echo "reconcile.sh: source-parity inventory COMPLETE (every declared source is covered; no dangling/empty/omitted)."
return 0
}
# --- part (ii): full reconcile (0 unaccounted) -----------------------------
# _scope_anchor ANCHOR (content on stdin) — mirror of the detector's anchor
# scoping so the reconciler hashes the SAME region the detector would. Extract
# from the first line containing the anchor up to the next heading (or EOF).
_scope_anchor() {
local anchor="$1"
awk -v a="$anchor" '
index($0, a) > 0 && !inzone { inzone=1; print; next }
inzone && /^#/ { exit }
inzone { print }
'
}
# _recon_key KIND ID — stable per-source ledger key.
_recon_key() {
printf '%s\037%s' "$1" "$2" | _hash
}
# _observe_source KIND ID DEF — observe current scoped state; echo the hash on
# success (rc 0). rc 1 = FAIL LOUD (source error / ambiguous-empty), same G2a
# discipline as the detector: an error is never silently "no state".
_observe_source() {
local kind="$1" id="$2" def="$3"
local anchor raw rc deftmp
anchor="$(printf '%s' "$def" | jq -r '.anchor // empty')"
mkdir -p "$RECON_DIR"
# shellcheck disable=SC2154
deftmp="$(mktemp "$RECON_DIR/${_wake_tmp_prefix}defXXXXXX")" || return 1
printf '%s' "$def" >"$deftmp"
raw="$("$WAKE_DETECTOR_SOURCE_CMD" "$kind" "$id" <"$deftmp" 2>/dev/null)"
rc=$?
rm -f "$deftmp"
if [ "$rc" -ne 0 ]; then
echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' errored (adapter exit $rc). Cannot certify parity; not treated as 'no state'." >&2
return 1
fi
if [ -z "$raw" ]; then
echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' returned AMBIGUOUS-EMPTY (empty-that-might-mean-hidden is never 'no state')." >&2
return 1
fi
local scoped
if [ -n "$anchor" ]; then
scoped="$(printf '%s' "$raw" | _scope_anchor "$anchor")"
if [ -z "$scoped" ]; then
echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' anchor '$anchor' not present (ambiguous section state)." >&2
return 1
fi
else
scoped="$raw"
fi
printf '%s' "$scoped" | _hash
}
# _inbox_has KIND ID HASH — true iff a pending inbox entry reflects this exact
# source state (observed_seq/inbox is the §4/G3 accounting universe).
_inbox_has() {
local kind="$1" id="$2" h="$3" n
n="$("$STORE_SH" drain 2>/dev/null | jq -s --arg k "$kind" --arg id "$id" --arg h "$h" \
'[ .[] | select(.locators.kind==$k and .locators.id==$id and .locators.observed_hash==$h) ] | length' 2>/dev/null || echo 0)"
[ "${n:-0}" -gt 0 ]
}
cmd_reconcile() {
local enumerate=1 allow_enumerate=0
while [ $# -gt 0 ]; do
case "$1" in
--no-enumerate)
enumerate=0
shift
;;
--allow-enumerate)
allow_enumerate=1
shift
;;
*)
echo "reconcile.sh reconcile: unknown option '$1'" >&2
exit 2
;;
esac
done
[ "${WAKE_RECONCILE_ALLOW_ENUMERATE:-0}" = "1" ] && allow_enumerate=1
local json
json="$(_load_watchlist)" || exit $?
if [ -z "${WAKE_DETECTOR_SOURCE_CMD:-}" ]; then
echo "reconcile.sh: WAKE_DETECTOR_SOURCE_CMD is not set (no adapter to observe source state)" >&2
exit 2
fi
"$STORE_SH" init >/dev/null 2>&1 || true
mkdir -p "$RECON_DIR"
# --- DUAL-ALLOCATOR SAFETY GATE (fail-closed; #908) ------------------------
# Decide, ONCE, whether ENUMERATION (the observed_seq WRITE path) is safe this
# pass. It is unsafe — and we REFUSE (flag, never swallow) — whenever a seq we
# allocate could be REISSUED by the detector's INDEPENDENT private counter and
# alias a distinct obligation (see the header hazard note). --no-enumerate and
# `check` never write, so the gate does not apply to them.
local enum_mode="$enumerate" enum_refuse_reason=""
if [ "$enumerate" -eq 1 ]; then
if [ -e "$DET_STATE" ]; then
enum_mode=0
enum_refuse_reason="a detector is co-feeding this store ($DET_STATE exists); its private observed_seq counter (W4) is INDEPENDENT of the store cursor this reconciler allocates from, so enumerating would risk REISSUING/ALIASING a seq and silently swallowing a distinct obligation. #908 tracks the store-side allocator unification that makes co-feeding safe. OPERATIONAL GUARD: do not run a detector and the reconciler against one store until #908."
elif [ "$allow_enumerate" -ne 1 ]; then
enum_mode=0
enum_refuse_reason="enumeration writes observed_seq via an allocator INDEPENDENT of the detector's private counter (W4); until #908 unifies them, co-feeding one store aliases seqs even under strictly-sequential calls. Set --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1) to assert this store is reconciler-SOLE-FEEDER (no detector co-feeds it)."
fi
fi
# Seq allocation base: the store's authoritative observed_seq (§2.4). Used ONLY
# on the sole-feeder path the gate above has proven safe.
local obs_base offset
obs_base="$("$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p')"
case "$obs_base" in '' | *[!0-9]*) obs_base=0 ;; esac
offset=0
local refuse_announced=0 refused=0
local pairs kind id failed=0 unaccounted=0 enumerated=0
pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')"
if [ -z "$pairs" ]; then
echo "reconcile.sh: watch-list declares no sources under watches[].sources[]" >&2
exit 2
fi
echo "# full reconcile (declared+configured source state vs observed_seq/inbox)"
while IFS=$'\t' read -r kind id; do
[ -n "$kind" ] || continue
local coll def
if ! coll="$(_coll_for_kind "$kind")"; then
echo "reconcile.sh: FAIL LOUD — unknown source kind '$kind' in watch-list" >&2
failed=1
continue
fi
def="$(printf '%s' "$json" | jq -c --arg c "$coll" --arg id "$id" '(.[$c] // []) | map(select(.id == $id)) | .[0] // empty')"
if [ -z "$def" ]; then
echo "reconcile.sh: FAIL LOUD (G3 parity) — watch references '$kind/$id' but no such entry is declared in '$coll'" >&2
failed=1
continue
fi
local curhash key ledger accounted
if ! curhash="$(_observe_source "$kind" "$id" "$def")"; then
failed=1
continue
fi
key="$(_recon_key "$kind" "$id")"
ledger="$RECON_DIR/seen-$key"
accounted=0
# Reflected in the durable inbox (delivered, awaiting consume)?
if _inbox_has "$kind" "$id" "$curhash"; then
accounted=1
# Or already reconciled to this exact state before (durable ledger)?
elif [ -f "$ledger" ] && [ "$(tr -d '[:space:]' <"$ledger")" = "$curhash" ]; then
accounted=1
fi
if [ "$accounted" -eq 1 ]; then
printf ' %s/%s: ACCOUNTED\n' "$kind" "$id"
continue
fi
# UNACCOUNTED: a source state not reflected in observed_seq/inbox => a gap.
unaccounted=$((unaccounted + 1))
printf ' %s/%s: UNACCOUNTED (current state not reflected in observed_seq/inbox)\n' "$kind" "$id" >&2
if [ "$enumerate" -eq 1 ] && [ "$enum_mode" -eq 0 ]; then
# REFUSE to enumerate (fail-closed): the obligation is loudly FLAGGED, NOT
# silently swallowed. Announce the dual-allocator reason once.
if [ "$refuse_announced" -eq 0 ]; then
echo "reconcile.sh: FAIL LOUD (dual-allocator hazard, #908) — $enum_refuse_reason" >&2
refuse_announced=1
fi
refused=1
printf ' -> REFUSED enumeration (FLAGGED, not swallowed; see dual-allocator hazard / #908)\n' >&2
elif [ "$enum_mode" -eq 1 ]; then
# ENUMERATE the pre-existing/startup obligation into the durable store.
# This path runs ONLY when the safety gate proved the reconciler is the
# SOLE feeder (no detector co-feeding), so its store-cursor allocation
# cannot be aliased. Class = non-coalescible fail-safe `actionable` (§2.3)
# so distinct pre-existing obligations never collapse together.
offset=$((offset + 1))
local seq locators
seq=$((obs_base + offset))
locators="$(jq -cn --arg kind "$kind" --arg id "$id" --arg hash "$curhash" \
--argjson sdef "$def" \
'{kind:$kind, id:$id, observed_hash:$hash, reconciled:true}
+ ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')"
if "$STORE_SH" enqueue --seq "$seq" --class actionable --locators "$locators" >/dev/null 2>&1; then
printf '%s' "$curhash" | _atomic_write "$ledger"
enumerated=$((enumerated + 1))
printf ' -> enumerated into store as observed_seq=%s (class=actionable)\n' "$seq"
else
echo "reconcile.sh: enumerate FAILED for '$kind/$id' seq $seq" >&2
failed=1
fi
fi
done <<EOF
$pairs
EOF
echo "reconcile.sh: UNACCOUNTED=$unaccounted ENUMERATED=$enumerated"
if [ "$failed" -ne 0 ]; then
echo "reconcile.sh: reconcile FAILED (a source could not be observed/enumerated)." >&2
return 1
fi
if [ "$unaccounted" -ne 0 ]; then
if [ "$refused" -eq 1 ]; then
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) REFUSED enumeration (dual-allocator hazard, #908). They are flagged, NOT swallowed. Resolve co-feeding / assert sole-feeder, then re-run." >&2
elif [ "$enumerate" -eq 1 ]; then
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) found and enumerated. Re-run to confirm 0." >&2
else
echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) (audit mode; nothing enumerated)." >&2
fi
return 1
fi
echo "reconcile.sh: reconcile CLEAN — 0 unaccounted (every declared source state is reflected in observed_seq/inbox)."
return 0
}
cmd_check() {
# The full §4/G3 gate, side-effect-free.
local rc=0
cmd_inventory || rc=1
cmd_reconcile --no-enumerate || rc=1
if [ "$rc" -eq 0 ]; then
echo "reconcile.sh: G3 GATE PASS — inventory complete AND 0 unaccounted."
else
echo "reconcile.sh: G3 GATE FAIL — inventory incomplete OR unaccounted state present." >&2
fi
return "$rc"
}
main() {
[ $# -ge 1 ] || {
usage
exit 2
}
local cmd="$1"
shift
case "$cmd" in
inventory) cmd_inventory "$@" ;;
reconcile) cmd_reconcile "$@" ;;
check) cmd_check "$@" ;;
-h | --help | help) usage ;;
*)
echo "reconcile.sh: unknown command '$cmd'" >&2
usage
exit 2
;;
esac
}
main "$@"

132
packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh Executable file
View File

@@ -0,0 +1,132 @@
#!/usr/bin/env bash
# test-wake-fn-oracle.sh — RED-FIRST invariant harness for W5 (EPIC #892):
# the FN-oracle / synthetic-canary (fn-oracle.sh, A6).
#
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
# that invariant regresses:
# O1 healthy pipeline -> synthetic-canary FN-rate = 0 (PASS, exit 0) (§4 vector)
# O2 a DISABLED / dropping detector (perfect no-op rate) -> FN-DETECTED —
# the off-domain false-negative-blindspot killer: a detector that silently
# drops a KNOWN injected delta FAILS the oracle even at 0 wakes/day (§4/A8)
# O3 reached-CONSUMED-but-too-slow -> FN-DETECTED (the "within its per-class
# SLO" clause; a late delivery is still a false negative) (§4)
# O4 the verdict is OFF-DOMAIN: it is rendered from the terminal store cursor
# (consumed_seq), independent of any detector self-report (§4/A8)
# O5 no invented SLO: --slo-seconds is REQUIRED (fail-loud usage guard) (design law)
#
# Uses the oracle's OWN isolated state namespace + a pluggable drive command
# (WAKE_ORACLE_DETECTOR_CMD) so a disabled detector can be exercised. No live
# network, no operator queue touched.
#
# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell
# and re-exports the per-test env, so environments are isolated by design (the
# same idiom as test-wake-detector.sh).
# shellcheck disable=SC2030,SC2031
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ORACLE="$SCRIPT_DIR/fn-oracle.sh"
DET="$SCRIPT_DIR/detector.sh"
command -v jq >/dev/null 2>&1 || {
echo "SKIP: jq not available" >&2
exit 0
}
TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT
# Failures recorded to a FILE (subshell-safe — a var counter would silently
# swallow failures across the per-test subshells; mirrors the W2/W4 harnesses).
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
echo " FAIL: $*" >&2
echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }
fresh_home() {
local d="$TMP_ROOT/$1"
rm -rf "$d"
mkdir -p "$d"
printf '%s' "$d"
}
echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 =="
(
WAKE_ORACLE_HOME="$(fresh_home o1)"
export WAKE_ORACLE_HOME
unset WAKE_ORACLE_DETECTOR_CMD
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]"
echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]"
echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]"
) && ok
echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) =="
(
WAKE_ORACLE_HOME="$(fresh_home o2)"
export WAKE_ORACLE_HOME
# A fully-disabled detector: it observes nothing, enqueues nothing — a
# "perfect" 0-wake rate that would look ideal on the wakes/day metric alone.
export WAKE_ORACLE_DETECTOR_CMD="true"
out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate"
echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]"
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]"
echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]"
) && ok
echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) =="
(
WAKE_ORACLE_HOME="$(fresh_home o3)"
export WAKE_ORACLE_HOME
# The REAL detector delivers the canary, but a delay pushes event->CONSUMED
# past a tight per-class SLO. A late delivery is still a false negative.
export WAKE_ORACLE_DETECTOR_CMD="sleep 2 && '$DET' poll-once"
out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)"
echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]"
# It must NOT be the 'never observed' branch: the delta WAS observed/delivered,
# just too slowly. This proves O3 exercises the SLO clause specifically.
if echo "$out" | grep -qi 'never OBSERVED'; then
fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]"
fi
) && ok
echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector self-report) =="
(
WAKE_ORACLE_HOME="$(fresh_home o4)"
export WAKE_ORACLE_HOME
# A drive that LIES 'success' (exit 0) but enqueues nothing. If the oracle
# trusted the drive's exit code it would pass; because it judges only the
# terminal store state, it correctly reports FN-DETECTED.
export WAKE_ORACLE_DETECTOR_CMD="exit 0"
out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)"
echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]"
) && ok
echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) =="
(
WAKE_ORACLE_HOME="$(fresh_home o5)"
export WAKE_ORACLE_HOME
unset WAKE_ORACLE_DETECTOR_CMD WAKE_ORACLE_SLO_SECONDS
err="$("$ORACLE" run --count 1 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)"
echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]"
) && ok
echo
if [ -s "$FAILFILE" ]; then
echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
exit 1
fi
echo "wake fn-oracle harness: all invariants passed ($pass groups)"

281
packages/mosaic/framework/tools/wake/test-wake-reconcile.sh Executable file
View File

@@ -0,0 +1,281 @@
#!/usr/bin/env bash
# test-wake-reconcile.sh — RED-FIRST invariant harness for W5 (EPIC #892):
# the source-parity reconciler (reconcile.sh, A7).
#
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
# that invariant regresses:
# R1 complete inventory -> PASS (exit 0) (§4/G3-i)
# R2 an OMITTED source (declared but unwatched) -> FLAG; the §4 vector
# cannot pass VACUOUSLY, and it is NOT silently green (§4/G3-i)
# R3 a required_sources omission -> FLAG (explicit lane-dependency form) (§4/G3-i)
# R4 a dangling watch reference -> FLAG (§4/G3-i)
# R5 an UNACCOUNTED source-state -> FLAGGED (found + non-zero exit) (§4/G3-ii)
# R6 pre-existing state at startup -> ENUMERATED into the durable store;
# a follow-up reconcile then reports 0 unaccounted (W4-division)
# R7 a source already reflected in the inbox -> ACCOUNTED (no double-
# enumeration of detector-delivered state) (§4/G3-ii)
# R8 a source adapter error -> FAIL LOUD (G2a parity; never 'no state') (§4/G2a)
#
# Uses FAKE/STUB sources only (no live network). Isolated per test.
# shellcheck disable=SC2030,SC2031
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RECON="$SCRIPT_DIR/reconcile.sh"
STORE="$SCRIPT_DIR/store.sh"
DET="$SCRIPT_DIR/detector.sh"
command -v jq >/dev/null 2>&1 || {
echo "SKIP: jq not available" >&2
exit 0
}
TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
echo " FAIL: $*" >&2
echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }
fresh_state() {
local d="$TMP_ROOT/$1"
rm -rf "$d"
mkdir -p "$d"
printf '%s' "$d"
}
depth() { "$STORE" cursors | sed -n 's/pending_depth=//p'; }
# make_stub DIR — a source adapter reading $DIR/<kind>_<id> with optional .rc.
make_stub() {
local dir="$1"
cat >"$dir/adapter.sh" <<EOF
#!/usr/bin/env bash
set -u
kind="\$1"; id="\$2"; base="$dir/\${kind}_\${id}"; rc=0
[ -f "\$base.rc" ] && rc="\$(cat "\$base.rc")"
[ -f "\$base" ] && cat "\$base"
exit "\$rc"
EOF
chmod +x "$dir/adapter.sh"
}
echo "== R1: complete inventory -> PASS =="
(
WAKE_STATE_HOME="$(fresh_state r1)"
export WAKE_STATE_HOME
unset WAKE_AGENT
wl="$TMP_ROOT/r1.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" }, { "id": "r2" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r2" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
out="$("$RECON" inventory 2>&1)"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "R1: a complete inventory must PASS (exit 0) [$out]"
echo "$out" | grep -qi 'COMPLETE' || fail_msg "R1: a complete inventory must report COMPLETE [$out]"
) && ok
echo "== R2: OMITTED source -> FLAG (vacuous-pass prevented, not silently green) =="
(
WAKE_STATE_HOME="$(fresh_state r2)"
export WAKE_STATE_HOME
unset WAKE_AGENT
# r2 is DECLARED (operationally depended-on) but no watch covers it.
wl="$TMP_ROOT/r2.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" }, { "id": "r2" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
out="$("$RECON" inventory 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R2: an omitted (declared-but-unwatched) source must FLAG (non-zero), not silently pass"
echo "$out" | grep -qi 'r2' || fail_msg "R2: the flag must name the omitted source r2 [$out]"
echo "$out" | grep -qi 'vacuous' || fail_msg "R2: the flag must state the vacuous-pass is prevented [$out]"
) && ok
echo "== R3: required_sources omission -> FLAG =="
(
WAKE_STATE_HOME="$(fresh_state r3)"
export WAKE_STATE_HOME
unset WAKE_AGENT
# The lane explicitly declares it depends on r1 AND r2, but only watches r1.
wl="$TMP_ROOT/r3.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" }, { "id": "r2" } ],
"watches": [ { "lane": "L", "required_sources": [ "r1", "r2" ], "sources": [ { "kind": "repo", "id": "r1" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
out="$("$RECON" inventory 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R3: a required_sources omission must FLAG (non-zero)"
echo "$out" | grep -qi "requires source 'r2'" || fail_msg "R3: the flag must name the required-but-omitted source r2 [$out]"
) && ok
echo "== R4: dangling watch reference -> FLAG =="
(
WAKE_STATE_HOME="$(fresh_state r4)"
export WAKE_STATE_HOME
unset WAKE_AGENT
wl="$TMP_ROOT/r4.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r9" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
out="$("$RECON" inventory 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R4: a dangling reference must FLAG (non-zero)"
echo "$out" | grep -qi 'dangling' || fail_msg "R4: the flag must state it is dangling [$out]"
) && ok
echo "== R5/R6: pre-existing state -> UNACCOUNTED flag + enumerated into store; re-run 0 =="
(
WAKE_STATE_HOME="$(fresh_state r56)"
export WAKE_STATE_HOME
unset WAKE_AGENT
stub="$TMP_ROOT/r56stub"
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
# TWO pre-existing sources with content already present at startup.
printf 'PRE-EXISTING-1\n' >"$stub/repo_r1"
printf 'PRE-EXISTING-2\n' >"$stub/repo_r2"
wl="$TMP_ROOT/r56.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" }, { "id": "r2" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r2" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
# Sole-feeder mode: no detector co-feeds this store (detector.sh is never run
# here), so enumeration is collision-safe once the operator asserts it.
export WAKE_RECONCILE_ALLOW_ENUMERATE=1
[ "$(depth)" = "0" ] || fail_msg "R5: store must start empty"
out="$("$RECON" reconcile 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R5: pre-existing unaccounted state must FLAG (non-zero) on first reconcile"
echo "$out" | grep -qi 'UNACCOUNTED=2' || fail_msg "R5: both pre-existing sources must be UNACCOUNTED [$out]"
# R6: enumerated into the durable store — one entry per pre-existing source.
[ "$(depth)" = "2" ] || fail_msg "R6: pre-existing state must be ENUMERATED into the store (depth 2), got $(depth)"
# A follow-up reconcile now finds everything accounted -> 0 unaccounted.
out2="$("$RECON" reconcile 2>&1)"
rc2=$?
[ "$rc2" -eq 0 ] || fail_msg "R6: a second reconcile must report 0 unaccounted (exit 0) [$out2]"
echo "$out2" | grep -qi 'UNACCOUNTED=0' || fail_msg "R6: second reconcile must be 0 unaccounted [$out2]"
[ "$(depth)" = "2" ] || fail_msg "R6: a clean reconcile must NOT re-enumerate (depth still 2), got $(depth)"
) && ok
echo "== R7: source already in the inbox -> ACCOUNTED (no double-enumeration) =="
(
WAKE_STATE_HOME="$(fresh_state r7)"
export WAKE_STATE_HOME
unset WAKE_AGENT
stub="$TMP_ROOT/r7stub"
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
printf 'V0\n' >"$stub/repo_r1"
wl="$TMP_ROOT/r7.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
# Detector baselines (silent) then observes a real delta -> pending inbox entry.
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R7: detector baseline failed"
printf 'V1\n' >"$stub/repo_r1"
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R7: detector delta failed"
[ "$(depth)" = "1" ] || fail_msg "R7: detector delta should leave one pending entry, got $(depth)"
# The source's CURRENT state IS the pending inbox entry -> ACCOUNTED, and the
# reconciler must NOT enumerate a duplicate.
out="$("$RECON" reconcile 2>&1)"
rc=$?
[ "$rc" -eq 0 ] || fail_msg "R7: state reflected in the inbox must be ACCOUNTED (exit 0) [$out]"
echo "$out" | grep -qi 'UNACCOUNTED=0' || fail_msg "R7: inbox-reflected state must be 0 unaccounted [$out]"
[ "$(depth)" = "1" ] || fail_msg "R7: reconciler must NOT double-enumerate inbox-reflected state (depth still 1), got $(depth)"
) && ok
echo "== R8: source adapter error -> FAIL LOUD (G2a; never 'no state') =="
(
WAKE_STATE_HOME="$(fresh_state r8)"
export WAKE_STATE_HOME
unset WAKE_AGENT
stub="$TMP_ROOT/r8stub"
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
printf 'FORBIDDEN\n' >"$stub/repo_r1"
printf '3\n' >"$stub/repo_r1.rc" # adapter exits non-zero (403/partial class)
wl="$TMP_ROOT/r8.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "r1" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
out="$("$RECON" reconcile 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R8: a source error must FAIL LOUD (non-zero exit)"
echo "$out" | grep -qi 'FAIL LOUD' || fail_msg "R8: the source error must be loud [$out]"
[ "$(depth)" = "0" ] || fail_msg "R8: a failed observation must NOT enumerate anything, got depth $(depth)"
) && ok
echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never silent-alias) =="
(
WAKE_STATE_HOME="$(fresh_state r9)"
export WAKE_STATE_HOME
unset WAKE_AGENT WAKE_RECONCILE_ALLOW_ENUMERATE
stub="$TMP_ROOT/r9stub"
mkdir -p "$stub"
make_stub "$stub"
export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh"
# Pre-existing sources present at startup (the reviewer's alpha/beta repro).
printf 'ALPHA\n' >"$stub/repo_alpha"
printf 'BETA\n' >"$stub/repo_beta"
wl="$TMP_ROOT/r9.json"
cat >"$wl" <<'EOF'
{ "schema_version": 1,
"repos": [ { "id": "alpha" }, { "id": "beta" } ],
"watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "alpha" }, { "kind": "repo", "id": "beta" } ] } ] }
EOF
export WAKE_WATCH_LIST="$wl"
# A DETECTOR is co-feeding this store: baseline it (creates the detector's
# private-counter state dir). The detector's private observed_seq allocator is
# now live and INDEPENDENT of the store cursor — the exact co-feeding hazard.
"$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector baseline failed"
# The pre-existing state is unaccounted (baseline is silent). Enumerating it
# here would allocate a store-cursor seq the detector's private counter can
# later REISSUE and alias — the serialized collision. The reconciler MUST
# refuse (fail loud), NOT enqueue a collidable seq.
before="$(depth)"
out="$("$RECON" reconcile 2>&1)"
rc=$?
[ "$rc" -ne 0 ] || fail_msg "R9: enumerating into a detector-co-fed store must FAIL LOUD (non-zero), not silently alias"
echo "$out" | grep -qi 'dual-allocator' || fail_msg "R9: the failure must name the dual-allocator hazard [$out]"
echo "$out" | grep -q '908' || fail_msg "R9: the failure must reference the #908 follow-up [$out]"
echo "$out" | grep -qi 'REFUSED' || fail_msg "R9: the obligation must be REFUSED/flagged, not swallowed [$out]"
# No collidable seq was written: depth is unchanged, so a later detector delta
# cannot alias a reconciler-enumerated entry.
[ "$(depth)" = "$before" ] || fail_msg "R9: a refused reconcile must NOT enqueue any (aliasable) seq, depth changed $before->$(depth)"
# (That the sole-feeder path DOES still enumerate — i.e. the guard is not a
# blanket no-op — is proven by R5/R6, which enumerate under --allow-enumerate.)
) && ok
echo
if [ -s "$FAILFILE" ]; then
echo "wake reconcile harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
exit 1
fi
echo "wake reconcile harness: all invariants passed ($pass groups)"

2
packages/mosaic/package.json
View File

@@ -25,7 +25,7 @@
"lint": "eslint src", "lint": "eslint src",
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"test": "vitest run --passWithNoTests && pnpm run test:framework-shell", "test": "vitest run --passWithNoTests && pnpm run test:framework-shell",
"test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh" "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh"
}, },
"dependencies": { "dependencies": {
"@mosaicstack/brain": "workspace:*", "@mosaicstack/brain": "workspace:*",