From 320f5bfb6f6274c39c696baa0b48714d9e07969e Mon Sep 17 00:00:00 2001 From: "jason.woltje" Date: Sun, 26 Jul 2026 02:04:19 +0000 Subject: [PATCH] =?UTF-8?q?feat(wake):=20W5=20=E2=80=94=20synthetic-canary?= =?UTF-8?q?=20FN-oracle=20+=20source-parity=20reconciler=20(#909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: jason.woltje Co-committed-by: jason.woltje --- .../mosaic/framework/tools/wake/fn-oracle.sh | 352 +++++++++++ .../mosaic/framework/tools/wake/manifest.txt | 15 +- .../mosaic/framework/tools/wake/reconcile.sh | 568 ++++++++++++++++++ .../tools/wake/test-wake-fn-oracle.sh | 132 ++++ .../tools/wake/test-wake-reconcile.sh | 281 +++++++++ packages/mosaic/package.json | 2 +- 6 files changed, 1346 insertions(+), 4 deletions(-) create mode 100755 packages/mosaic/framework/tools/wake/fn-oracle.sh create mode 100755 packages/mosaic/framework/tools/wake/reconcile.sh create mode 100755 packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh create mode 100755 packages/mosaic/framework/tools/wake/test-wake-reconcile.sh diff --git a/packages/mosaic/framework/tools/wake/fn-oracle.sh b/packages/mosaic/framework/tools/wake/fn-oracle.sh new file mode 100755 index 00000000..b08c21af --- /dev/null +++ b/packages/mosaic/framework/tools/wake/fn-oracle.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash +# fn-oracle.sh — A6 of the wake canon (EPIC #892, W5): the FN-oracle / +# synthetic-canary. +# +# CONTRACT ANCHORS (docs/scratchpads/heartbeat-planning/CONVERGED-DESIGN.md): +# §4 vector { synthetic-canary FN-rate = 0 } — the oracle injects a KNOWN +# synthetic-canary delta into the wake pipeline and ASSERTS it +# reaches CONSUMED within its per-class SLO; it measures the +# FALSE-NEGATIVE rate. FN-rate MUST be 0 for the timer to retire. +# §4/§7-res5 The FN-oracle is a real, NON-OPTIONAL operating cost — the price +# of being allowed to retire the fixed timer, not a free extra. +# §1.4 Framework component, operator-agnostic (~/.config/mosaic/tools/). +# +# OFF-DOMAIN / DETECTOR-INDEPENDENT (the §4/A8 false-negative-blindspot killer): +# The injection and the CONSUMED-assertion do NOT depend on the detector's +# own hashing / poll internals. The oracle injects a delta at the SOURCE +# boundary (the bytes an operator adapter reports), drives the pipeline +# through the detector's PUBLIC API only (poll-once — a black box), and then +# renders its verdict SOLELY from the terminal store/ack state (did a CONSUMED +# ack advance the cursor to cover the canary's observed_seq?). It never reads a +# detector hash-file, poll counter, or self-report. CONSEQUENCE: a detector +# that silently DROPS changes — including a FULLY-DISABLED detector at a +# "perfect" no-op rate (0 wakes/day, which looks great on the wakes metric) — +# still FAILS the oracle, because the canary never reaches CONSUMED. That is +# the whole point: the success metric is a VECTOR, not a wake-count. +# +# ISOLATION: the probe runs in its OWN XDG state namespace (WAKE_ORACLE_HOME), +# NOT the operator's live queue, so a synthetic canary never delivers a fake +# wake to a real agent. It exercises the REAL detector.sh / store.sh / ack.sh +# CODE paths against isolated synthetic data — the standard synthetic-canary +# shape (synthetic transaction, real code, isolated data). +# +# SCOPE: calls the PUBLIC APIs of detector.sh (poll-once), store.sh +# (cursors/drain), and ack.sh (received/consumed). It does NOT reimplement or +# modify any of them. +# +# Operator-agnostic: all state via XDG/env; no operator paths/names/secrets. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./_wake-common.sh disable=SC1091 +. "$SCRIPT_DIR/_wake-common.sh" + +DETECTOR_SH="$SCRIPT_DIR/detector.sh" +STORE_SH="$SCRIPT_DIR/store.sh" +ACK_SH="$SCRIPT_DIR/ack.sh" + +# Isolated oracle state root (XDG). NEVER the operator's live wake queue. +ORACLE_HOME="${WAKE_ORACLE_HOME:-${XDG_STATE_HOME:-$HOME/.local/state}/mosaic/wake/fn-oracle}" +# The canary's own store namespace lives UNDER the oracle root. +CANARY_STATE_HOME="$ORACLE_HOME/canary-store" +CANARY_AGENT="canary" +CANARY_SRC="$ORACLE_HOME/canary.src" +CANARY_ADAPTER="$ORACLE_HOME/canary-adapter.sh" +CANARY_WATCHLIST="$ORACLE_HOME/canary-watch-list.json" +METRICS="$ORACLE_HOME/metrics.jsonl" + +_need_jq() { + command -v jq >/dev/null 2>&1 || { + echo "fn-oracle.sh: jq is required" >&2 + exit 3 + } +} + +usage() { + cat >&2 <<'EOF' +Usage: fn-oracle.sh run --slo-seconds N [options] + +Commands: + run --slo-seconds N [--count K] [--class C] + Run K synthetic-canary cycles (default 1). Each cycle injects a KNOWN + delta at the source boundary, drives the pipeline through the detector's + public poll-once, then asserts the canary reaches CONSUMED within the + per-class SLO (N seconds). Emits per-canary PASS/FN-DETECTED, the + FN-rate metric, and an overall VERDICT. Exit 0 IFF FN-rate == 0. + status + Print the tail of the recorded metrics ledger. + +Options: + --slo-seconds N REQUIRED for run. The per-class SLO (operator-tuned; symbolic + tiers per §4 — no default is invented). event->CONSUMED must + land within N seconds or the canary counts as a false-negative. + --count K Number of canary cycles this run (default 1). + --class C Wake class the canary flows as (default digest — the primary + machine-wake path §4 certifies). digest|actionable|human. + +Environment: + WAKE_ORACLE_HOME Isolated oracle state root (XDG by default). NEVER the + operator's live queue. + WAKE_ORACLE_DETECTOR_CMD Command the oracle runs to advance ONE observe cycle + of the pipeline (default: detector.sh poll-once). The + canary env (watch-list/adapter/state) is exported to + it. Pluggable so a disabled/dropping detector can be + exercised (it MUST then be caught as FN-DETECTED). + WAKE_ORACLE_SLO_SECONDS Fallback for --slo-seconds. +EOF +} + +# --- one-time synthetic-canary scaffold (idempotent) ----------------------- +# The oracle SUPPLIES its own source adapter + watch-list so it needs no +# operator config and stays fully self-contained. The adapter simply echoes the +# current canary source bytes — the synthetic "source of truth" the oracle +# controls end-to-end. +_write_scaffold() { + local class="$1" + mkdir -p "$ORACLE_HOME" + + cat >"$CANARY_ADAPTER" < any enqueue is unambiguously the + # canary. The class is carried so the canary flows on the class under test. + jq -n --arg class "$class" '{ + schema_version: 1, + repos: [ { id: "fn-canary", remote: "synthetic/fn-oracle-canary", class: $class } ], + watches: [ { lane: "fn-oracle", sources: [ { kind: "repo", id: "fn-canary" } ] } ] + }' >"$CANARY_WATCHLIST" +} + +# _canary_env — export the isolated canary namespace + adapter + watch-list for +# a child detector/store/ack invocation. +_canary_env() { + export WAKE_STATE_HOME="$CANARY_STATE_HOME" + export WAKE_AGENT="$CANARY_AGENT" + export WAKE_WATCH_LIST="$CANARY_WATCHLIST" + export WAKE_DETECTOR_SOURCE_CMD="$CANARY_ADAPTER" +} + +# _store_observed / _store_consumed — read the terminal cursors (public API). +_store_observed() { "$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p'; } +_store_consumed() { "$STORE_SH" cursors 2>/dev/null | sed -n 's/^consumed_seq=//p'; } + +# _drive_pipeline — advance ONE observe cycle. Black-box: whatever +# WAKE_ORACLE_DETECTOR_CMD is (default: the real detector's public poll-once). +# A source failure inside the detector is loud there; the oracle judges only the +# terminal state, so a drop of ANY kind surfaces as an unmet CONSUMED. +_drive_pipeline() { + local cmd="${WAKE_ORACLE_DETECTOR_CMD:-"$DETECTOR_SH poll-once"}" + sh -c "$cmd" >/dev/null 2>&1 || true +} + +# _seat_baseline — seat the source's first-seen baseline ONCE (detector +# first-seen is silent by design). Every subsequent canary token is then a +# genuine DELTA off the prior token, so no spurious baseline wakes are produced. +_seat_baseline() { + _canary_env + printf '%s' "FN-ORACLE-BASELINE-$$-$(date +%s)" >"$CANARY_SRC" + _drive_pipeline +} + +# --- one canary cycle ------------------------------------------------------- +# Returns 0 = PASS (reached CONSUMED within SLO), 1 = FN-DETECTED. +# Prints a one-line verdict for the cycle. +_run_one() { + local idx="$1" slo="$2" class="$3" + _canary_env + + local nonce token inject_ts + nonce="$$-$idx-$(date +%s%N 2>/dev/null || date +%s)" + token="FN-ORACLE-CANARY-$nonce" + + local obs_before + obs_before="$(_store_observed)" + case "$obs_before" in '' | *[!0-9]*) obs_before=0 ;; esac + + # INJECT the known delta at the source boundary + stamp the clock. + inject_ts="$(date +%s)" + printf '%s' "$token" >"$CANARY_SRC" + + # Drive the pipeline (through the detector's public API — a black box). + _drive_pipeline + + local obs_after + obs_after="$(_store_observed)" + case "$obs_after" in '' | *[!0-9]*) obs_after=0 ;; esac + + # ---- OFF-DOMAIN verdict, part 1: was the delta OBSERVED at all? ---------- + # In an isolated single-source store, any observed_seq advance is the canary. + # NO advance => the pipeline (a dropping / disabled detector) swallowed a KNOWN + # real change => FALSE NEGATIVE. This is the killer: a perfect no-op rate does + # not rescue a detector that fails to see a real delta. + if [ "$obs_after" -le "$obs_before" ]; then + printf ' canary %s: FN-DETECTED (injected delta never OBSERVED — pipeline dropped a known change; observed_seq %s unchanged)\n' \ + "$idx" "$obs_before" >&2 + return 1 + fi + + # The delta was observed and durably enqueued. Simulate the consumer contract + # (§2.2): deliver (drain), RECEIVED-ack, capture, then CONSUMED-ack the + # contiguous prefix. All via the PUBLIC ack/store API. + local delivered + delivered="$("$STORE_SH" drain 2>/dev/null | grep -c '.' || echo 0)" + if [ "${delivered:-0}" -lt 1 ]; then + printf ' canary %s: FN-DETECTED (observed but NOT delivered — drain returned nothing to consume)\n' "$idx" >&2 + return 1 + fi + + "$ACK_SH" received --wake-id "fn-oracle-$nonce" >/dev/null 2>&1 || true + # CONSUMED over the contiguous prefix up to the canary's observed_seq. + "$ACK_SH" consumed --upto "$obs_after" --no-sync >/dev/null 2>&1 || true + + # ---- OFF-DOMAIN verdict, part 2: did it reach CONSUMED, within SLO? ------ + local consumed elapsed now + consumed="$(_store_consumed)" + case "$consumed" in '' | *[!0-9]*) consumed=0 ;; esac + now="$(date +%s)" + elapsed=$((now - inject_ts)) + + if [ "$consumed" -lt "$obs_after" ]; then + printf ' canary %s: FN-DETECTED (delivered but consumed_seq=%s never reached the canary observed_seq=%s)\n' \ + "$idx" "$consumed" "$obs_after" >&2 + return 1 + fi + if [ "$elapsed" -gt "$slo" ]; then + printf ' canary %s: FN-DETECTED (reached CONSUMED but in %ss > per-class SLO %ss)\n' \ + "$idx" "$elapsed" "$slo" >&2 + return 1 + fi + + printf ' canary %s: PASS (event->CONSUMED in %ss <= SLO %ss; observed_seq=%s consumed_seq=%s class=%s)\n' \ + "$idx" "$elapsed" "$slo" "$obs_after" "$consumed" "$class" + return 0 +} + +cmd_run() { + _need_jq + local slo="${WAKE_ORACLE_SLO_SECONDS:-}" count=1 class="digest" + while [ $# -gt 0 ]; do + case "$1" in + --slo-seconds) + slo="${2:-}" + shift 2 + ;; + --count) + count="${2:-}" + shift 2 + ;; + --class) + class="${2:-}" + shift 2 + ;; + *) + echo "fn-oracle.sh run: unknown option '$1'" >&2 + exit 2 + ;; + esac + done + + # No invented numeric SLO (design law): the per-class SLO MUST be supplied. + case "$slo" in + '' | *[!0-9]*) + echo "fn-oracle.sh run: --slo-seconds (or WAKE_ORACLE_SLO_SECONDS) is REQUIRED and must be a non-negative integer (per-class SLO is operator-tuned; no default is invented)" >&2 + exit 2 + ;; + esac + case "$count" in + '' | *[!0-9]* | 0) + echo "fn-oracle.sh run: --count must be a positive integer" >&2 + exit 2 + ;; + esac + case "$class" in + digest | actionable | human) : ;; + *) + echo "fn-oracle.sh run: --class must be one of digest|actionable|human" >&2 + exit 2 + ;; + esac + + # Fresh isolated probe state each run => a deterministic baseline. This is the + # oracle's OWN synthetic store, never the operator's live queue. + rm -rf "$CANARY_STATE_HOME" + mkdir -p "$ORACLE_HOME" + _write_scaffold "$class" + + echo "fn-oracle: synthetic-canary run (count=$count, class=$class, per-class SLO=${slo}s)" + _seat_baseline # one-time first-seen baseline (silent by design) + local fn=0 i + i=1 + while [ "$i" -le "$count" ]; do + if _run_one "$i" "$slo" "$class"; then + : + else + fn=$((fn + 1)) + fi + i=$((i + 1)) + done + + # FN-rate metric. §4 requires FN-rate == 0. + local rate verdict rc + if [ "$fn" -eq 0 ]; then + verdict="PASS" + rc=0 + else + verdict="FN-DETECTED" + rc=1 + fi + # Rational FN-rate rendered without bc (portable): integer numerator/denominator + # plus a scaled decimal. + rate="$(awk -v f="$fn" -v n="$count" 'BEGIN { printf "%.4f", (n>0? f/n : 0) }')" + + local rec + rec="$(jq -cn \ + --argjson ts "$(date +%s)" \ + --arg class "$class" \ + --argjson slo "$slo" \ + --argjson count "$count" \ + --argjson fn "$fn" \ + --arg rate "$rate" \ + --arg verdict "$verdict" \ + '{ts:$ts, class:$class, slo_seconds:$slo, count:$count, fn:$fn, fn_rate:($rate|tonumber), verdict:$verdict}')" + { cat "$METRICS" 2>/dev/null; printf '%s\n' "$rec"; } | grep -v '^[[:space:]]*$' >"$METRICS.tmp" 2>/dev/null || true + mv -f "$METRICS.tmp" "$METRICS" 2>/dev/null || true + + echo "fn-oracle: FN-RATE = $fn/$count = $rate (class=$class, SLO=${slo}s)" + echo "fn-oracle: VERDICT = $verdict" + [ "$rc" -eq 0 ] || echo "fn-oracle: §4 vector FAILS — synthetic-canary FN-rate must be 0 to retire the timer." >&2 + return "$rc" +} + +cmd_status() { + echo "# fn-oracle metrics (tail)" + tail -n 10 "$METRICS" 2>/dev/null || echo "(no runs recorded)" +} + +main() { + [ $# -ge 1 ] || { + usage + exit 2 + } + local cmd="$1" + shift + case "$cmd" in + run) cmd_run "$@" ;; + status) cmd_status "$@" ;; + -h | --help | help) usage ;; + *) + echo "fn-oracle.sh: unknown command '$cmd'" >&2 + usage + exit 2 + ;; + esac +} + +main "$@" diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index 069c1930..2be98a9e 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -14,8 +14,9 @@ # 0.1.0 W2 — store+drain lib + ack-wrapper. # 0.2.0 W3 — cumulative-state digest renderer + non-circular HMAC signer. # 0.3.0 W4 — per-host single-instance delta-gated detector daemon. +# 0.4.0 W5 — synthetic-canary FN-oracle + source-parity reconciler. component=wake -version=0.3.0 +version=0.4.0 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -35,5 +36,13 @@ schema_max=1 # detector.sh A1 — per-host single-instance delta-gated detector daemon # (flock, anchor-scoped hashing, detector-local observed_seq, # fail-loud source semantics; enqueues deltas to store.sh). (W4) -# Out of scope (later waves): FN-oracle/reconciler (W5), beacon (W6), -# installer (W7). +# fn-oracle.sh A6 — synthetic-canary FN-oracle: injects a KNOWN delta at the +# source boundary, drives the pipeline through the detector's +# public poll-once, asserts CONSUMED within the per-class SLO +# (off-domain verdict from the terminal store cursor). §4 +# requires FN-rate=0; a dropping/disabled detector FAILS. (W5) +# reconcile.sh A7 — source-parity reconciler: (i) source-coverage parity +# inventory (an omitted source cannot pass the vector +# vacuously) + (ii) periodic full reconcile to 0-unaccounted, +# enumerating pre-existing/startup state into the store. (W5) +# Out of scope (later waves): off-host beacon (W6), installer (W7). diff --git a/packages/mosaic/framework/tools/wake/reconcile.sh b/packages/mosaic/framework/tools/wake/reconcile.sh new file mode 100755 index 00000000..0bc1beb3 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/reconcile.sh @@ -0,0 +1,568 @@ +#!/usr/bin/env bash +# reconcile.sh — A7 of the wake canon (EPIC #892, W5): the source-parity +# reconciler. +# +# CONTRACT ANCHORS (docs/scratchpads/heartbeat-planning/CONVERGED-DESIGN.md): +# §4/G3 Source parity / reconciliation — the GO condition has TWO ordered +# parts: +# (i) a source-coverage PARITY INVENTORY FIRST: a lane-by-lane +# DECLARED inventory of every operational source the lane +# depends on, so an OMITTED source cannot make the §4 vector pass +# VACUOUSLY. An incomplete / undeclared inventory FLAGS (it must +# NOT silently pass). +# (ii) then a periodic full RECONCILE: enumerate declared+configured +# source state vs observed_seq/inbox => 0 UNACCOUNTED. Any source +# state not reflected in observed_seq/inbox is a gap => FLAG. +# §4 vector { source-parity inventory complete AND reconcile = 0 unaccounted } +# §7-res5 FN-oracle + reconciliation are a real, NON-OPTIONAL operating cost. +# W4 review division: the detector's first-seen-baseline does NOT wake; the +# RECONCILER is what enumerates PRE-EXISTING / startup source state into +# the durable store (at startup and periodically). Without it, state +# that already existed when the detector first saw it (baselined +# silently) would never enter the inbox and would be lost. +# +# SCOPE: calls the PUBLIC APIs of store.sh (cursors/drain/enqueue) only. It does +# NOT reimplement or modify the store, ack, digest, signer, or detector. It +# OBSERVES current source state through the SAME operator adapter contract the +# detector uses (WAKE_DETECTOR_SOURCE_CMD , def on stdin), with the +# SAME fail-loud (G2a) discipline: a source error / ambiguous-empty is never +# silently "no state". +# +# CONTRACT NOTES (flagged, not silently guessed — see PR body): +# * observed_seq DUAL-ALLOCATOR HAZARD (fail-closed; #908 tracks the fix). +# observed_seq has TWO INDEPENDENT allocators today: the detector's PRIVATE +# counter ($STATE_DIR/detector/observed_seq_counter, W4) and — if this +# reconciler enumerated — the STORE cursor (observed_seq+offset). Because the +# detector's next seq is (private_counter+1), NOT (store_observed_seq+1), the +# two are decoupled and SERIALIZING THEM DOES NOT PREVENT COLLISION: after +# the reconciler enumerates alpha->seq1, beta->seq2 (store observed_seq=2, +# detector private still 0), the detector's next delta allocates private +# 0->1 and enqueues seq1 — ALIASING alpha. A consumer acking CONSUMED 1 +# (trusting §2.4's unique observed_seq) then silently DROPS a distinct +# obligation — the exact G3 swallow this reconciler exists to prevent. +# THEREFORE, until #908 unifies on a single store-side allocator, the +# reconciler FAILS CLOSED: it REFUSES to enumerate (loud, non-zero; the +# obligation is FLAGGED, never silently swallowed) whenever a detector is +# co-feeding this store, and otherwise requires the operator to assert +# reconciler-SOLE-FEEDER mode. OPERATIONAL GUARD: a detector and this +# reconciler MUST NOT co-feed one store until #908 lands. +# * "accounted" is judged against observed_seq/inbox (§4/G3 wording) PLUS the +# reconciler's OWN durable reconciled-state ledger — NOT the detector's +# hash-file. Judging by the detector baseline would let a silent first-seen +# baseline swallow a pre-existing obligation (the exact hole G3 closes). +# * enumeration uses the non-coalescible fail-safe class `actionable` (§2.3) +# so two distinct pre-existing obligations can never coalesce into one. +# +# Operator-agnostic: all state via XDG/env; no operator paths/names/secrets. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=./_wake-common.sh disable=SC1091 +. "$SCRIPT_DIR/_wake-common.sh" + +STORE_SH="$SCRIPT_DIR/store.sh" +MANIFEST="$SCRIPT_DIR/manifest.txt" + +STATE_DIR="$(wake_state_dir)" +# Reconciler-local state (its durable reconciled-state ledger) lives in its own +# subdir under the store's STATE_DIR so it never collides with store/detector. +RECON_DIR="$STATE_DIR/reconciler" +# The detector's PRIVATE state dir (W4). Its presence signals a detector is (or +# has been) allocating observed_seq into THIS store from an INDEPENDENT private +# counter — the co-feeding condition under which reconciler enumeration is +# unsafe (see the dual-allocator hazard note above; #908). Read-only coupling: +# reconcile.sh never writes or imports W4, it only OBSERVES this path to fail +# closed rather than silently alias a seq. +DET_STATE="$STATE_DIR/detector" + +_need_jq() { + command -v jq >/dev/null 2>&1 || { + echo "reconcile.sh: jq is required" >&2 + exit 3 + } +} + +_hash() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 | awk '{print $1}' + elif command -v openssl >/dev/null 2>&1; then + openssl dgst -sha256 | awk '{print $NF}' + else + echo "reconcile.sh: no sha256 tool (sha256sum/shasum/openssl) available" >&2 + exit 3 + fi +} + +usage() { + cat >&2 <<'EOF' +Usage: reconcile.sh [options] + +Commands: + inventory + Part (i) — source-coverage PARITY INVENTORY. Prints a lane-by-lane + declared inventory and FLAGS any incompleteness that could let the §4 + vector pass VACUOUSLY: + * a declared operational source (repos/board_files/lane_anchors) that + NO watch covers (an OMITTED source); + * a watch that references a source with no declared definition (dangling); + * a lane with zero sources; an empty watch-list; + * if a lane declares `required_sources`, any id missing from its sources. + Exit 0 IFF the inventory is complete; non-zero (loud) on any flag. + + reconcile [--no-enumerate] [--allow-enumerate] + Part (ii) — periodic full reconcile. Enumerates each covered source's + CURRENT state and compares it against observed_seq/inbox (+ the + reconciler's reconciled-state ledger). Any source state NOT reflected => + UNACCOUNTED => FLAG. Unless --no-enumerate, each unaccounted source is + ENUMERATED into the durable store (the startup/pre-existing obligation + path). Exit 0 IFF 0 unaccounted were FOUND this pass. + + DUAL-ALLOCATOR SAFETY (fail-closed; #908): enumeration WRITES observed_seq + from an allocator INDEPENDENT of the detector's private counter, so it + FAILS LOUD (refuses; the obligation is FLAGGED, never swallowed) when it + would risk aliasing a seq: + * if a detector is co-feeding this store (its private state dir exists) + => refuse, ALWAYS (even with the ack below); + * else the operator MUST assert reconciler-SOLE-FEEDER mode via + --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1), promising no + detector co-feeds this store until #908 lands. + --no-enumerate (and `check`) never write, so they are always safe. + + check + The full §4/G3 gate, side-effect-free: `inventory` AND + `reconcile --no-enumerate`. Exit 0 IFF the inventory is complete AND 0 + unaccounted. Use this to evaluate the vector's G3 clause without writing. + +Environment: + WAKE_WATCH_LIST Operator watch-list JSON (required). + WAKE_DETECTOR_SOURCE_CMD Source adapter (required for reconcile/check; same + contract as the detector: , def on + stdin; exit!=0 or empty => FAIL LOUD). + WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh). + WAKE_RECONCILE_ALLOW_ENUMERATE set to 1 to assert reconciler-SOLE-FEEDER mode + (see reconcile's dual-allocator safety note; #908). +EOF +} + +_manifest_val() { + local key="$1" + [ -f "$MANIFEST" ] || return 0 + sed -n "s/^${key}=//p" "$MANIFEST" | head -n1 | tr -d '[:space:]' +} + +# _load_watchlist — validate path + JSON + shape + Gate B schema range (mirrors +# the detector's fail-loud validation so the reconciler certifies the SAME +# watch-list the detector runs). Echoes validated JSON on stdout. +_load_watchlist() { + _need_jq + local wl="${WAKE_WATCH_LIST:-}" + if [ -z "$wl" ]; then + echo "reconcile.sh: WAKE_WATCH_LIST is not set (no watch-list to reconcile)" >&2 + return 2 + fi + if [ ! -f "$wl" ]; then + echo "reconcile.sh: watch-list not found: $wl" >&2 + return 2 + fi + local json + if ! json="$(jq -e . "$wl" 2>/dev/null)"; then + echo "reconcile.sh: watch-list is not valid JSON: $wl" >&2 + return 2 + fi + if ! printf '%s' "$json" | jq -e 'has("schema_version") and has("watches")' >/dev/null 2>&1; then + echo "reconcile.sh: watch-list missing required 'schema_version' or 'watches'" >&2 + return 2 + fi + local ver smin smax + ver="$(printf '%s' "$json" | jq -r '.schema_version')" + smin="$(_manifest_val schema_min)" + smax="$(_manifest_val schema_max)" + case "$ver" in + '' | *[!0-9]*) + echo "reconcile.sh: watch-list schema_version must be an integer (got '$ver')" >&2 + return 2 + ;; + esac + if [ -z "$smin" ] || [ -z "$smax" ]; then + echo "reconcile.sh: manifest is missing schema_min/schema_max (cannot validate Gate B)" >&2 + return 2 + fi + if [ "$ver" -lt "$smin" ] || [ "$ver" -gt "$smax" ]; then + echo "reconcile.sh: FAIL LOUD (Gate B) — watch-list schema_version $ver is OUTSIDE the supported range [$smin, $smax]" >&2 + return 2 + fi + printf '%s' "$json" +} + +# _coll_for_kind KIND — the top-level collection a source kind is defined in. +_coll_for_kind() { + case "$1" in + repo) printf 'repos' ;; + board_file) printf 'board_files' ;; + lane_anchor) printf 'lane_anchors' ;; + *) return 1 ;; + esac +} + +# --- part (i): source-coverage parity inventory ---------------------------- + +cmd_inventory() { + local json + json="$(_load_watchlist)" || exit $? + + local flags=0 + + # An empty watch-list can never certify coverage of anything. + local nwatch + nwatch="$(printf '%s' "$json" | jq -r '.watches | length')" + if [ "${nwatch:-0}" -eq 0 ]; then + echo "reconcile.sh: FLAG (G3 parity) — watch-list declares NO watches; coverage is vacuous." >&2 + flags=$((flags + 1)) + fi + + echo "# source-coverage parity inventory (lane-by-lane)" + + # Per-lane declared inventory + empty-lane + required_sources checks. + local nlanes li + nlanes="$(printf '%s' "$json" | jq -r '.watches | length')" + li=0 + while [ "$li" -lt "${nlanes:-0}" ]; do + local lane nsrc + lane="$(printf '%s' "$json" | jq -r --argjson i "$li" '.watches[$i].lane // "(unnamed)"')" + nsrc="$(printf '%s' "$json" | jq -r --argjson i "$li" '.watches[$i].sources | length')" + printf ' lane %s: %s declared source(s)\n' "$lane" "${nsrc:-0}" + + if [ "${nsrc:-0}" -eq 0 ]; then + echo "reconcile.sh: FLAG (G3 parity) — lane '$lane' declares ZERO sources (empty coverage)." >&2 + flags=$((flags + 1)) + fi + + # required_sources (optional, additive; ignored by the detector). If the + # operator declares the lane's true dependency set here, every id MUST be + # covered by the lane's sources — an OMITTED dependency FLAGS. + local missing + missing="$(printf '%s' "$json" | jq -r --argjson i "$li" ' + (.watches[$i].required_sources // []) as $req + | ($req - [ .watches[$i].sources[].id ]) | .[]')" + if [ -n "$missing" ]; then + local m + while IFS= read -r m; do + [ -n "$m" ] || continue + echo "reconcile.sh: FLAG (G3 parity) — lane '$lane' requires source '$m' but it is OMITTED from the lane's watched sources (vacuous-pass prevented)." >&2 + flags=$((flags + 1)) + done <&2 + flags=$((flags + 1)) + continue + fi + def="$(printf '%s' "$json" | jq -c --arg c "$coll" --arg id "$id" '(.[$c] // []) | map(select(.id == $id)) | .[0] // empty')" + if [ -z "$def" ]; then + echo "reconcile.sh: FLAG (G3 parity) — watch references '$kind/$id' but no such entry is declared in '$coll' (dangling)." >&2 + flags=$((flags + 1)) + fi + done <&2 + flags=$((flags + 1)) + fi + done <&2 + return 1 + fi + echo "reconcile.sh: source-parity inventory COMPLETE (every declared source is covered; no dangling/empty/omitted)." + return 0 +} + +# --- part (ii): full reconcile (0 unaccounted) ----------------------------- + +# _scope_anchor ANCHOR (content on stdin) — mirror of the detector's anchor +# scoping so the reconciler hashes the SAME region the detector would. Extract +# from the first line containing the anchor up to the next heading (or EOF). +_scope_anchor() { + local anchor="$1" + awk -v a="$anchor" ' + index($0, a) > 0 && !inzone { inzone=1; print; next } + inzone && /^#/ { exit } + inzone { print } + ' +} + +# _recon_key KIND ID — stable per-source ledger key. +_recon_key() { + printf '%s\037%s' "$1" "$2" | _hash +} + +# _observe_source KIND ID DEF — observe current scoped state; echo the hash on +# success (rc 0). rc 1 = FAIL LOUD (source error / ambiguous-empty), same G2a +# discipline as the detector: an error is never silently "no state". +_observe_source() { + local kind="$1" id="$2" def="$3" + local anchor raw rc deftmp + anchor="$(printf '%s' "$def" | jq -r '.anchor // empty')" + mkdir -p "$RECON_DIR" + # shellcheck disable=SC2154 + deftmp="$(mktemp "$RECON_DIR/${_wake_tmp_prefix}defXXXXXX")" || return 1 + printf '%s' "$def" >"$deftmp" + raw="$("$WAKE_DETECTOR_SOURCE_CMD" "$kind" "$id" <"$deftmp" 2>/dev/null)" + rc=$? + rm -f "$deftmp" + if [ "$rc" -ne 0 ]; then + echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' errored (adapter exit $rc). Cannot certify parity; not treated as 'no state'." >&2 + return 1 + fi + if [ -z "$raw" ]; then + echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' returned AMBIGUOUS-EMPTY (empty-that-might-mean-hidden is never 'no state')." >&2 + return 1 + fi + local scoped + if [ -n "$anchor" ]; then + scoped="$(printf '%s' "$raw" | _scope_anchor "$anchor")" + if [ -z "$scoped" ]; then + echo "reconcile.sh: FAIL LOUD (G2a) — source '$kind/$id' anchor '$anchor' not present (ambiguous section state)." >&2 + return 1 + fi + else + scoped="$raw" + fi + printf '%s' "$scoped" | _hash +} + +# _inbox_has KIND ID HASH — true iff a pending inbox entry reflects this exact +# source state (observed_seq/inbox is the §4/G3 accounting universe). +_inbox_has() { + local kind="$1" id="$2" h="$3" n + n="$("$STORE_SH" drain 2>/dev/null | jq -s --arg k "$kind" --arg id "$id" --arg h "$h" \ + '[ .[] | select(.locators.kind==$k and .locators.id==$id and .locators.observed_hash==$h) ] | length' 2>/dev/null || echo 0)" + [ "${n:-0}" -gt 0 ] +} + +cmd_reconcile() { + local enumerate=1 allow_enumerate=0 + while [ $# -gt 0 ]; do + case "$1" in + --no-enumerate) + enumerate=0 + shift + ;; + --allow-enumerate) + allow_enumerate=1 + shift + ;; + *) + echo "reconcile.sh reconcile: unknown option '$1'" >&2 + exit 2 + ;; + esac + done + [ "${WAKE_RECONCILE_ALLOW_ENUMERATE:-0}" = "1" ] && allow_enumerate=1 + + local json + json="$(_load_watchlist)" || exit $? + if [ -z "${WAKE_DETECTOR_SOURCE_CMD:-}" ]; then + echo "reconcile.sh: WAKE_DETECTOR_SOURCE_CMD is not set (no adapter to observe source state)" >&2 + exit 2 + fi + + "$STORE_SH" init >/dev/null 2>&1 || true + mkdir -p "$RECON_DIR" + + # --- DUAL-ALLOCATOR SAFETY GATE (fail-closed; #908) ------------------------ + # Decide, ONCE, whether ENUMERATION (the observed_seq WRITE path) is safe this + # pass. It is unsafe — and we REFUSE (flag, never swallow) — whenever a seq we + # allocate could be REISSUED by the detector's INDEPENDENT private counter and + # alias a distinct obligation (see the header hazard note). --no-enumerate and + # `check` never write, so the gate does not apply to them. + local enum_mode="$enumerate" enum_refuse_reason="" + if [ "$enumerate" -eq 1 ]; then + if [ -e "$DET_STATE" ]; then + enum_mode=0 + enum_refuse_reason="a detector is co-feeding this store ($DET_STATE exists); its private observed_seq counter (W4) is INDEPENDENT of the store cursor this reconciler allocates from, so enumerating would risk REISSUING/ALIASING a seq and silently swallowing a distinct obligation. #908 tracks the store-side allocator unification that makes co-feeding safe. OPERATIONAL GUARD: do not run a detector and the reconciler against one store until #908." + elif [ "$allow_enumerate" -ne 1 ]; then + enum_mode=0 + enum_refuse_reason="enumeration writes observed_seq via an allocator INDEPENDENT of the detector's private counter (W4); until #908 unifies them, co-feeding one store aliases seqs even under strictly-sequential calls. Set --allow-enumerate (or WAKE_RECONCILE_ALLOW_ENUMERATE=1) to assert this store is reconciler-SOLE-FEEDER (no detector co-feeds it)." + fi + fi + + # Seq allocation base: the store's authoritative observed_seq (§2.4). Used ONLY + # on the sole-feeder path the gate above has proven safe. + local obs_base offset + obs_base="$("$STORE_SH" cursors 2>/dev/null | sed -n 's/^observed_seq=//p')" + case "$obs_base" in '' | *[!0-9]*) obs_base=0 ;; esac + offset=0 + local refuse_announced=0 refused=0 + + local pairs kind id failed=0 unaccounted=0 enumerated=0 + pairs="$(printf '%s' "$json" | jq -r '[ .watches[].sources[] | "\(.kind)\t\(.id)" ] | unique | .[]')" + if [ -z "$pairs" ]; then + echo "reconcile.sh: watch-list declares no sources under watches[].sources[]" >&2 + exit 2 + fi + + echo "# full reconcile (declared+configured source state vs observed_seq/inbox)" + while IFS=$'\t' read -r kind id; do + [ -n "$kind" ] || continue + local coll def + if ! coll="$(_coll_for_kind "$kind")"; then + echo "reconcile.sh: FAIL LOUD — unknown source kind '$kind' in watch-list" >&2 + failed=1 + continue + fi + def="$(printf '%s' "$json" | jq -c --arg c "$coll" --arg id "$id" '(.[$c] // []) | map(select(.id == $id)) | .[0] // empty')" + if [ -z "$def" ]; then + echo "reconcile.sh: FAIL LOUD (G3 parity) — watch references '$kind/$id' but no such entry is declared in '$coll'" >&2 + failed=1 + continue + fi + + local curhash key ledger accounted + if ! curhash="$(_observe_source "$kind" "$id" "$def")"; then + failed=1 + continue + fi + key="$(_recon_key "$kind" "$id")" + ledger="$RECON_DIR/seen-$key" + + accounted=0 + # Reflected in the durable inbox (delivered, awaiting consume)? + if _inbox_has "$kind" "$id" "$curhash"; then + accounted=1 + # Or already reconciled to this exact state before (durable ledger)? + elif [ -f "$ledger" ] && [ "$(tr -d '[:space:]' <"$ledger")" = "$curhash" ]; then + accounted=1 + fi + + if [ "$accounted" -eq 1 ]; then + printf ' %s/%s: ACCOUNTED\n' "$kind" "$id" + continue + fi + + # UNACCOUNTED: a source state not reflected in observed_seq/inbox => a gap. + unaccounted=$((unaccounted + 1)) + printf ' %s/%s: UNACCOUNTED (current state not reflected in observed_seq/inbox)\n' "$kind" "$id" >&2 + + if [ "$enumerate" -eq 1 ] && [ "$enum_mode" -eq 0 ]; then + # REFUSE to enumerate (fail-closed): the obligation is loudly FLAGGED, NOT + # silently swallowed. Announce the dual-allocator reason once. + if [ "$refuse_announced" -eq 0 ]; then + echo "reconcile.sh: FAIL LOUD (dual-allocator hazard, #908) — $enum_refuse_reason" >&2 + refuse_announced=1 + fi + refused=1 + printf ' -> REFUSED enumeration (FLAGGED, not swallowed; see dual-allocator hazard / #908)\n' >&2 + elif [ "$enum_mode" -eq 1 ]; then + # ENUMERATE the pre-existing/startup obligation into the durable store. + # This path runs ONLY when the safety gate proved the reconciler is the + # SOLE feeder (no detector co-feeding), so its store-cursor allocation + # cannot be aliased. Class = non-coalescible fail-safe `actionable` (§2.3) + # so distinct pre-existing obligations never collapse together. + offset=$((offset + 1)) + local seq locators + seq=$((obs_base + offset)) + locators="$(jq -cn --arg kind "$kind" --arg id "$id" --arg hash "$curhash" \ + --argjson sdef "$def" \ + '{kind:$kind, id:$id, observed_hash:$hash, reconciled:true} + + ( $sdef | {repo, path, anchor, remote, branches} | with_entries(select(.value != null)) )')" + if "$STORE_SH" enqueue --seq "$seq" --class actionable --locators "$locators" >/dev/null 2>&1; then + printf '%s' "$curhash" | _atomic_write "$ledger" + enumerated=$((enumerated + 1)) + printf ' -> enumerated into store as observed_seq=%s (class=actionable)\n' "$seq" + else + echo "reconcile.sh: enumerate FAILED for '$kind/$id' seq $seq" >&2 + failed=1 + fi + fi + done <&2 + return 1 + fi + if [ "$unaccounted" -ne 0 ]; then + if [ "$refused" -eq 1 ]; then + echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) REFUSED enumeration (dual-allocator hazard, #908). They are flagged, NOT swallowed. Resolve co-feeding / assert sole-feeder, then re-run." >&2 + elif [ "$enumerate" -eq 1 ]; then + echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) found and enumerated. Re-run to confirm 0." >&2 + else + echo "reconcile.sh: FLAG — $unaccounted unaccounted source-state(s) (audit mode; nothing enumerated)." >&2 + fi + return 1 + fi + echo "reconcile.sh: reconcile CLEAN — 0 unaccounted (every declared source state is reflected in observed_seq/inbox)." + return 0 +} + +cmd_check() { + # The full §4/G3 gate, side-effect-free. + local rc=0 + cmd_inventory || rc=1 + cmd_reconcile --no-enumerate || rc=1 + if [ "$rc" -eq 0 ]; then + echo "reconcile.sh: G3 GATE PASS — inventory complete AND 0 unaccounted." + else + echo "reconcile.sh: G3 GATE FAIL — inventory incomplete OR unaccounted state present." >&2 + fi + return "$rc" +} + +main() { + [ $# -ge 1 ] || { + usage + exit 2 + } + local cmd="$1" + shift + case "$cmd" in + inventory) cmd_inventory "$@" ;; + reconcile) cmd_reconcile "$@" ;; + check) cmd_check "$@" ;; + -h | --help | help) usage ;; + *) + echo "reconcile.sh: unknown command '$cmd'" >&2 + usage + exit 2 + ;; + esac +} + +main "$@" diff --git a/packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh b/packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh new file mode 100755 index 00000000..e3766326 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/test-wake-fn-oracle.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# test-wake-fn-oracle.sh — RED-FIRST invariant harness for W5 (EPIC #892): +# the FN-oracle / synthetic-canary (fn-oracle.sh, A6). +# +# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if +# that invariant regresses: +# O1 healthy pipeline -> synthetic-canary FN-rate = 0 (PASS, exit 0) (§4 vector) +# O2 a DISABLED / dropping detector (perfect no-op rate) -> FN-DETECTED — +# the off-domain false-negative-blindspot killer: a detector that silently +# drops a KNOWN injected delta FAILS the oracle even at 0 wakes/day (§4/A8) +# O3 reached-CONSUMED-but-too-slow -> FN-DETECTED (the "within its per-class +# SLO" clause; a late delivery is still a false negative) (§4) +# O4 the verdict is OFF-DOMAIN: it is rendered from the terminal store cursor +# (consumed_seq), independent of any detector self-report (§4/A8) +# O5 no invented SLO: --slo-seconds is REQUIRED (fail-loud usage guard) (design law) +# +# Uses the oracle's OWN isolated state namespace + a pluggable drive command +# (WAKE_ORACLE_DETECTOR_CMD) so a disabled detector can be exercised. No live +# network, no operator queue touched. +# +# SC2030/SC2031 are DELIBERATELY disabled: each test runs in its own ( ) subshell +# and re-exports the per-test env, so environments are isolated by design (the +# same idiom as test-wake-detector.sh). +# shellcheck disable=SC2030,SC2031 +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ORACLE="$SCRIPT_DIR/fn-oracle.sh" +DET="$SCRIPT_DIR/detector.sh" + +command -v jq >/dev/null 2>&1 || { + echo "SKIP: jq not available" >&2 + exit 0 +} + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +# Failures recorded to a FILE (subshell-safe — a var counter would silently +# swallow failures across the per-test subshells; mirrors the W2/W4 harnesses). +FAILFILE="$TMP_ROOT/failures" +: >"$FAILFILE" +pass=0 +fail_msg() { + echo " FAIL: $*" >&2 + echo "x" >>"$FAILFILE" +} +ok() { pass=$((pass + 1)); } + +fresh_home() { + local d="$TMP_ROOT/$1" + rm -rf "$d" + mkdir -p "$d" + printf '%s' "$d" +} + +echo "== O1: healthy pipeline -> synthetic-canary FN-rate = 0 ==" +( + WAKE_ORACLE_HOME="$(fresh_home o1)" + export WAKE_ORACLE_HOME + unset WAKE_ORACLE_DETECTOR_CMD + out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "O1: a healthy pipeline must exit 0 (got $rc) [$out]" + echo "$out" | grep -q 'FN-RATE = 0/3' || fail_msg "O1: FN-rate must be 0/3 on a healthy pipeline [$out]" + echo "$out" | grep -q 'VERDICT = PASS' || fail_msg "O1: verdict must be PASS on a healthy pipeline [$out]" +) && ok + +echo "== O2: DISABLED detector (perfect no-op) -> FN-DETECTED (blindspot killer) ==" +( + WAKE_ORACLE_HOME="$(fresh_home o2)" + export WAKE_ORACLE_HOME + # A fully-disabled detector: it observes nothing, enqueues nothing — a + # "perfect" 0-wake rate that would look ideal on the wakes/day metric alone. + export WAKE_ORACLE_DETECTOR_CMD="true" + out="$("$ORACLE" run --slo-seconds 120 --count 3 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "O2: a detector that drops a KNOWN delta MUST fail the oracle (non-zero exit), even at a perfect no-op rate" + echo "$out" | grep -q 'FN-RATE = 3/3' || fail_msg "O2: every dropped canary must count as a false-negative (FN-RATE 3/3) [$out]" + echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O2: verdict must be FN-DETECTED for a dropping detector [$out]" + echo "$out" | grep -qi 'never OBSERVED' || fail_msg "O2: the failure must name the dropped (never-observed) delta [$out]" +) && ok + +echo "== O3: reached CONSUMED but too slow -> FN-DETECTED (within-SLO clause) ==" +( + WAKE_ORACLE_HOME="$(fresh_home o3)" + export WAKE_ORACLE_HOME + # The REAL detector delivers the canary, but a delay pushes event->CONSUMED + # past a tight per-class SLO. A late delivery is still a false negative. + export WAKE_ORACLE_DETECTOR_CMD="sleep 2 && '$DET' poll-once" + out="$("$ORACLE" run --slo-seconds 1 --count 1 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "O3: a canary that reaches CONSUMED past its SLO must be a false-negative (non-zero exit)" + echo "$out" | grep -qi 'per-class SLO' || fail_msg "O3: the failure must attribute to the per-class SLO, not a drop [$out]" + # It must NOT be the 'never observed' branch: the delta WAS observed/delivered, + # just too slowly. This proves O3 exercises the SLO clause specifically. + if echo "$out" | grep -qi 'never OBSERVED'; then + fail_msg "O3: an SLO breach must NOT be misreported as a dropped delta [$out]" + fi +) && ok + +echo "== O4: verdict is OFF-DOMAIN (from terminal consumed_seq, not detector self-report) ==" +( + WAKE_ORACLE_HOME="$(fresh_home o4)" + export WAKE_ORACLE_HOME + # A drive that LIES 'success' (exit 0) but enqueues nothing. If the oracle + # trusted the drive's exit code it would pass; because it judges only the + # terminal store state, it correctly reports FN-DETECTED. + export WAKE_ORACLE_DETECTOR_CMD="exit 0" + out="$("$ORACLE" run --slo-seconds 120 --count 1 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "O4: a drive that exits 0 but delivers nothing must still be FN-DETECTED (verdict is off-domain)" + echo "$out" | grep -q 'VERDICT = FN-DETECTED' || fail_msg "O4: off-domain verdict must not be fooled by a clean exit code [$out]" +) && ok + +echo "== O5: no invented SLO -> --slo-seconds is REQUIRED (fail-loud) ==" +( + WAKE_ORACLE_HOME="$(fresh_home o5)" + export WAKE_ORACLE_HOME + unset WAKE_ORACLE_DETECTOR_CMD WAKE_ORACLE_SLO_SECONDS + err="$("$ORACLE" run --count 1 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "O5: run without an SLO must fail loud (no invented default)" + echo "$err" | grep -qi 'slo' || fail_msg "O5: the usage error must name the missing SLO [$err]" +) && ok + +echo +if [ -s "$FAILFILE" ]; then + echo "wake fn-oracle harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 + exit 1 +fi +echo "wake fn-oracle harness: all invariants passed ($pass groups)" diff --git a/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh new file mode 100755 index 00000000..77d248a0 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/test-wake-reconcile.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# test-wake-reconcile.sh — RED-FIRST invariant harness for W5 (EPIC #892): +# the source-parity reconciler (reconcile.sh, A7). +# +# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if +# that invariant regresses: +# R1 complete inventory -> PASS (exit 0) (§4/G3-i) +# R2 an OMITTED source (declared but unwatched) -> FLAG; the §4 vector +# cannot pass VACUOUSLY, and it is NOT silently green (§4/G3-i) +# R3 a required_sources omission -> FLAG (explicit lane-dependency form) (§4/G3-i) +# R4 a dangling watch reference -> FLAG (§4/G3-i) +# R5 an UNACCOUNTED source-state -> FLAGGED (found + non-zero exit) (§4/G3-ii) +# R6 pre-existing state at startup -> ENUMERATED into the durable store; +# a follow-up reconcile then reports 0 unaccounted (W4-division) +# R7 a source already reflected in the inbox -> ACCOUNTED (no double- +# enumeration of detector-delivered state) (§4/G3-ii) +# R8 a source adapter error -> FAIL LOUD (G2a parity; never 'no state') (§4/G2a) +# +# Uses FAKE/STUB sources only (no live network). Isolated per test. +# shellcheck disable=SC2030,SC2031 +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RECON="$SCRIPT_DIR/reconcile.sh" +STORE="$SCRIPT_DIR/store.sh" +DET="$SCRIPT_DIR/detector.sh" + +command -v jq >/dev/null 2>&1 || { + echo "SKIP: jq not available" >&2 + exit 0 +} + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +FAILFILE="$TMP_ROOT/failures" +: >"$FAILFILE" +pass=0 +fail_msg() { + echo " FAIL: $*" >&2 + echo "x" >>"$FAILFILE" +} +ok() { pass=$((pass + 1)); } + +fresh_state() { + local d="$TMP_ROOT/$1" + rm -rf "$d" + mkdir -p "$d" + printf '%s' "$d" +} +depth() { "$STORE" cursors | sed -n 's/pending_depth=//p'; } + +# make_stub DIR — a source adapter reading $DIR/_ with optional .rc. +make_stub() { + local dir="$1" + cat >"$dir/adapter.sh" < PASS ==" +( + WAKE_STATE_HOME="$(fresh_state r1)" + export WAKE_STATE_HOME + unset WAKE_AGENT + wl="$TMP_ROOT/r1.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" }, { "id": "r2" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r2" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + out="$("$RECON" inventory 2>&1)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "R1: a complete inventory must PASS (exit 0) [$out]" + echo "$out" | grep -qi 'COMPLETE' || fail_msg "R1: a complete inventory must report COMPLETE [$out]" +) && ok + +echo "== R2: OMITTED source -> FLAG (vacuous-pass prevented, not silently green) ==" +( + WAKE_STATE_HOME="$(fresh_state r2)" + export WAKE_STATE_HOME + unset WAKE_AGENT + # r2 is DECLARED (operationally depended-on) but no watch covers it. + wl="$TMP_ROOT/r2.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" }, { "id": "r2" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + out="$("$RECON" inventory 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R2: an omitted (declared-but-unwatched) source must FLAG (non-zero), not silently pass" + echo "$out" | grep -qi 'r2' || fail_msg "R2: the flag must name the omitted source r2 [$out]" + echo "$out" | grep -qi 'vacuous' || fail_msg "R2: the flag must state the vacuous-pass is prevented [$out]" +) && ok + +echo "== R3: required_sources omission -> FLAG ==" +( + WAKE_STATE_HOME="$(fresh_state r3)" + export WAKE_STATE_HOME + unset WAKE_AGENT + # The lane explicitly declares it depends on r1 AND r2, but only watches r1. + wl="$TMP_ROOT/r3.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" }, { "id": "r2" } ], + "watches": [ { "lane": "L", "required_sources": [ "r1", "r2" ], "sources": [ { "kind": "repo", "id": "r1" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + out="$("$RECON" inventory 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R3: a required_sources omission must FLAG (non-zero)" + echo "$out" | grep -qi "requires source 'r2'" || fail_msg "R3: the flag must name the required-but-omitted source r2 [$out]" +) && ok + +echo "== R4: dangling watch reference -> FLAG ==" +( + WAKE_STATE_HOME="$(fresh_state r4)" + export WAKE_STATE_HOME + unset WAKE_AGENT + wl="$TMP_ROOT/r4.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r9" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + out="$("$RECON" inventory 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R4: a dangling reference must FLAG (non-zero)" + echo "$out" | grep -qi 'dangling' || fail_msg "R4: the flag must state it is dangling [$out]" +) && ok + +echo "== R5/R6: pre-existing state -> UNACCOUNTED flag + enumerated into store; re-run 0 ==" +( + WAKE_STATE_HOME="$(fresh_state r56)" + export WAKE_STATE_HOME + unset WAKE_AGENT + stub="$TMP_ROOT/r56stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + # TWO pre-existing sources with content already present at startup. + printf 'PRE-EXISTING-1\n' >"$stub/repo_r1" + printf 'PRE-EXISTING-2\n' >"$stub/repo_r2" + wl="$TMP_ROOT/r56.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" }, { "id": "r2" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" }, { "kind": "repo", "id": "r2" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + # Sole-feeder mode: no detector co-feeds this store (detector.sh is never run + # here), so enumeration is collision-safe once the operator asserts it. + export WAKE_RECONCILE_ALLOW_ENUMERATE=1 + [ "$(depth)" = "0" ] || fail_msg "R5: store must start empty" + out="$("$RECON" reconcile 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R5: pre-existing unaccounted state must FLAG (non-zero) on first reconcile" + echo "$out" | grep -qi 'UNACCOUNTED=2' || fail_msg "R5: both pre-existing sources must be UNACCOUNTED [$out]" + # R6: enumerated into the durable store — one entry per pre-existing source. + [ "$(depth)" = "2" ] || fail_msg "R6: pre-existing state must be ENUMERATED into the store (depth 2), got $(depth)" + # A follow-up reconcile now finds everything accounted -> 0 unaccounted. + out2="$("$RECON" reconcile 2>&1)" + rc2=$? + [ "$rc2" -eq 0 ] || fail_msg "R6: a second reconcile must report 0 unaccounted (exit 0) [$out2]" + echo "$out2" | grep -qi 'UNACCOUNTED=0' || fail_msg "R6: second reconcile must be 0 unaccounted [$out2]" + [ "$(depth)" = "2" ] || fail_msg "R6: a clean reconcile must NOT re-enumerate (depth still 2), got $(depth)" +) && ok + +echo "== R7: source already in the inbox -> ACCOUNTED (no double-enumeration) ==" +( + WAKE_STATE_HOME="$(fresh_state r7)" + export WAKE_STATE_HOME + unset WAKE_AGENT + stub="$TMP_ROOT/r7stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + printf 'V0\n' >"$stub/repo_r1" + wl="$TMP_ROOT/r7.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + # Detector baselines (silent) then observes a real delta -> pending inbox entry. + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R7: detector baseline failed" + printf 'V1\n' >"$stub/repo_r1" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R7: detector delta failed" + [ "$(depth)" = "1" ] || fail_msg "R7: detector delta should leave one pending entry, got $(depth)" + # The source's CURRENT state IS the pending inbox entry -> ACCOUNTED, and the + # reconciler must NOT enumerate a duplicate. + out="$("$RECON" reconcile 2>&1)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "R7: state reflected in the inbox must be ACCOUNTED (exit 0) [$out]" + echo "$out" | grep -qi 'UNACCOUNTED=0' || fail_msg "R7: inbox-reflected state must be 0 unaccounted [$out]" + [ "$(depth)" = "1" ] || fail_msg "R7: reconciler must NOT double-enumerate inbox-reflected state (depth still 1), got $(depth)" +) && ok + +echo "== R8: source adapter error -> FAIL LOUD (G2a; never 'no state') ==" +( + WAKE_STATE_HOME="$(fresh_state r8)" + export WAKE_STATE_HOME + unset WAKE_AGENT + stub="$TMP_ROOT/r8stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + printf 'FORBIDDEN\n' >"$stub/repo_r1" + printf '3\n' >"$stub/repo_r1.rc" # adapter exits non-zero (403/partial class) + wl="$TMP_ROOT/r8.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "r1" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "r1" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + out="$("$RECON" reconcile 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R8: a source error must FAIL LOUD (non-zero exit)" + echo "$out" | grep -qi 'FAIL LOUD' || fail_msg "R8: the source error must be loud [$out]" + [ "$(depth)" = "0" ] || fail_msg "R8: a failed observation must NOT enumerate anything, got depth $(depth)" +) && ok + +echo "== R9: serialized dual-allocator collision -> reconciler FAILS LOUD (never silent-alias) ==" +( + WAKE_STATE_HOME="$(fresh_state r9)" + export WAKE_STATE_HOME + unset WAKE_AGENT WAKE_RECONCILE_ALLOW_ENUMERATE + stub="$TMP_ROOT/r9stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + # Pre-existing sources present at startup (the reviewer's alpha/beta repro). + printf 'ALPHA\n' >"$stub/repo_alpha" + printf 'BETA\n' >"$stub/repo_beta" + wl="$TMP_ROOT/r9.json" + cat >"$wl" <<'EOF' +{ "schema_version": 1, + "repos": [ { "id": "alpha" }, { "id": "beta" } ], + "watches": [ { "lane": "L", "sources": [ { "kind": "repo", "id": "alpha" }, { "kind": "repo", "id": "beta" } ] } ] } +EOF + export WAKE_WATCH_LIST="$wl" + # A DETECTOR is co-feeding this store: baseline it (creates the detector's + # private-counter state dir). The detector's private observed_seq allocator is + # now live and INDEPENDENT of the store cursor — the exact co-feeding hazard. + "$DET" poll-once >/dev/null 2>&1 || fail_msg "R9: detector baseline failed" + # The pre-existing state is unaccounted (baseline is silent). Enumerating it + # here would allocate a store-cursor seq the detector's private counter can + # later REISSUE and alias — the serialized collision. The reconciler MUST + # refuse (fail loud), NOT enqueue a collidable seq. + before="$(depth)" + out="$("$RECON" reconcile 2>&1)" + rc=$? + [ "$rc" -ne 0 ] || fail_msg "R9: enumerating into a detector-co-fed store must FAIL LOUD (non-zero), not silently alias" + echo "$out" | grep -qi 'dual-allocator' || fail_msg "R9: the failure must name the dual-allocator hazard [$out]" + echo "$out" | grep -q '908' || fail_msg "R9: the failure must reference the #908 follow-up [$out]" + echo "$out" | grep -qi 'REFUSED' || fail_msg "R9: the obligation must be REFUSED/flagged, not swallowed [$out]" + # No collidable seq was written: depth is unchanged, so a later detector delta + # cannot alias a reconciler-enumerated entry. + [ "$(depth)" = "$before" ] || fail_msg "R9: a refused reconcile must NOT enqueue any (aliasable) seq, depth changed $before->$(depth)" + # (That the sole-feeder path DOES still enumerate — i.e. the guard is not a + # blanket no-op — is proven by R5/R6, which enumerate under --allow-enumerate.) +) && ok + +echo +if [ -s "$FAILFILE" ]; then + echo "wake reconcile harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 + exit 1 +fi +echo "wake reconcile harness: all invariants passed ($pass groups)" diff --git a/packages/mosaic/package.json b/packages/mosaic/package.json index 714ff399..767ff66d 100644 --- a/packages/mosaic/package.json +++ b/packages/mosaic/package.json @@ -25,7 +25,7 @@ "lint": "eslint src", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests && pnpm run test:framework-shell", - "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh" + "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh" }, "dependencies": { "@mosaicstack/brain": "workspace:*",