#!/usr/bin/env bash # preimage.sh — A11 of the wake canon (#958): durable PROVENANCE for the # OPERATOR-SIDE preimage definition. # # THE GAP THIS CLOSES (#958): every observed_hash in the lane store is # sha256(adapter stdout) — the operator's source adapter (plus the watch-list # and any operator-declared preimage files) IS the preimage definition for # every hash the pipeline ever records. Those files live OUTSIDE the versioned # wake component, so a byte change to them was attributable only through agent # transcripts: provenance thinner than any hash it feeds. A changed preimage # also re-baselines EVERY source at once, which previously surfaced only as N # per-source deltas / UNACCOUNTED lines with no first-class cause. # # WHAT THIS TOOL DOES: # - Derives the PREIMAGE SET generically (no operator paths baked in): # * the resolved file behind WAKE_DETECTOR_SOURCE_CMD (first word), # * the WAKE_WATCH_LIST file, # * operator-declared extras via WAKE_PREIMAGE_EXTRA (colon-separated — # e.g. detector.env, sink/feed scripts). # - Records each file's (sha256, size, mtime, ts) as an APPEND-ONLY ledger # row in /preimage/preimage-ledger.jsonl and stores the full bytes # CONTENT-ADDRESSED at /preimage/objects/ — so a change is # attributable (prior bytes + change time) from durable state alone, with # no transcript required. Acceptance (a) of #958. # - On a CHANGE (not first-seen), `check --enqueue` enqueues ONE first-class # class=actionable entry per changed file via the store's single allocator # (#908), carrying a §2.1 hard locator (`path`), BEFORE the per-source # deltas it explains are observed — so the digest shows the cause line, # not just N re-baselined sources. Acceptance (c). # # CREDENTIAL HARD GATE — acceptance (b): the mechanism must be UNABLE to # capture credential material even by operator error. Enforced two ways, both # fail-closed toward NOT capturing bytes (the hash/size/mtime row is still # written — change-time attribution survives, content capture does not; # a sha256 discloses nothing about the bytes): # 1. PATH deny: the mosaic credential store locations # (/credentials.json, /tools/_lib/ # credentials.json, anything under /credentials/, and any # file literally named credentials.json) are refused by resolved realpath # before a single byte is read into the object store. # 2. CONTENT deny: a candidate whose bytes match the well-known secret-token # shapes (same conservative set digest.sh redacts: PAT/Slack/AKIA/JWT/PEM) # is refused — refusal, not redaction: a redacted preimage would be a # false witness, and secret bytes must never land in the object store. # Plus a size cap (WAKE_PREIMAGE_MAX_BYTES, default 1 MiB) so a stray extra # pointing at a large binary cannot turn provenance into data hoovering. # # FAIL-LOUD DISCIPLINE (the D2/#955 class, both layers): an infrastructure # failure of THIS tool (unresolvable adapter, unreadable/corrupt ledger, # failed durable write, failed enqueue) exits NON-ZERO and is never read as # "no change". A corrupt ledger REFUSES to compare (and to re-baseline): # silently restarting history would erase the very attribution this exists # to provide. # # Operator-agnostic (framework firewall): all state via XDG/env; the deny # list names only framework-defined credential locations, no operator hosts/ # names/secrets. This tool never prints file CONTENT to any stream. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=./_wake-common.sh disable=SC1091 . "$SCRIPT_DIR/_wake-common.sh" STATE_DIR="$(wake_state_dir)" STORE_SH="$SCRIPT_DIR/store.sh" PRE_DIR="$STATE_DIR/preimage" LEDGER="$PRE_DIR/preimage-ledger.jsonl" OBJECTS="$PRE_DIR/objects" _need_jq() { command -v jq >/dev/null 2>&1 || { echo "preimage.sh: jq is required" >&2 exit 3 } } # _hash_stdin — sha256 of stdin, first field only (portable, same fallback # chain as detector.sh). _hash_stdin() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}' elif command -v shasum >/dev/null 2>&1; then shasum -a 256 | awk '{print $1}' elif command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 | awk '{print $NF}' else echo "preimage.sh: no sha256 tool (sha256sum/shasum/openssl) available" >&2 exit 3 fi } usage() { cat >&2 <<'EOF' Usage: preimage.sh Commands: check [--enqueue] Compare every preimage-set file against the ledger head. A changed file gets a new ledger row + captured bytes; with --enqueue each change (not first-seen) also enqueues ONE first-class class=actionable store entry (hard locator: path). First-seen files baseline SILENTLY (row, no enqueue — same idiom as the detector's first-seen). Exit 0 whether or not changes were found; non-zero ONLY on infrastructure failure (never read as "no change"). record Alias of `check` without enqueue (baseline/refresh). set Print the derived preimage set, one resolved path per line (diagnostic). status Print the ledger head row for every tracked path. Environment: WAKE_DETECTOR_SOURCE_CMD Adapter command; its resolved file joins the set. WAKE_WATCH_LIST Watch-list path; joins the set when set. WAKE_PREIMAGE_EXTRA Colon-separated additional operator preimage files (e.g. detector.env, sink scripts). A listed path that does not exist is tracked as ABSENT (deletion of a preimage file is a change, not an error). WAKE_PREIMAGE_MAX_BYTES Byte-capture cap (default 1048576). Larger files: hash/size/mtime recorded, bytes refused. WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh). EOF } # --- preimage-set derivation (generic; no operator paths baked in) ---------- # _resolve_cmd_file CMD — resolve the FIRST WORD of an adapter command string # to a real file. Non-zero (loud) if it cannot be resolved: an adapter the # detector will invoke but provenance cannot see is an infrastructure failure, # not a smaller set. _resolve_cmd_file() { local cmd="$1" word path # shellcheck disable=SC2086 set -- $cmd word="${1:-}" [ -n "$word" ] || return 1 if [ -f "$word" ]; then path="$word" else path="$(command -v -- "$word" 2>/dev/null)" || return 1 [ -f "$path" ] || return 1 fi # realpath so the ledger keys on the actual file, not a symlink alias. if command -v realpath >/dev/null 2>&1; then realpath -- "$path" 2>/dev/null || printf '%s' "$path" else printf '%s' "$path" fi } # _preimage_set — print the derived set, one path per line. Paths from # WAKE_PREIMAGE_EXTRA are printed EVEN IF ABSENT (deletion is a tracked # state); the adapter and watch-list must resolve (loud failure otherwise — # see _resolve_cmd_file rationale). _preimage_set() { local failed=0 if [ -n "${WAKE_DETECTOR_SOURCE_CMD:-}" ]; then local af if af="$(_resolve_cmd_file "$WAKE_DETECTOR_SOURCE_CMD")"; then printf '%s\n' "$af" else echo "preimage.sh: FAIL LOUD — WAKE_DETECTOR_SOURCE_CMD ('$WAKE_DETECTOR_SOURCE_CMD') does not resolve to a file; the preimage definition cannot be observed (NOT treated as 'no change')." >&2 failed=1 fi fi if [ -n "${WAKE_WATCH_LIST:-}" ]; then if [ -f "$WAKE_WATCH_LIST" ]; then if command -v realpath >/dev/null 2>&1; then realpath -- "$WAKE_WATCH_LIST" 2>/dev/null || printf '%s' "$WAKE_WATCH_LIST" else printf '%s' "$WAKE_WATCH_LIST" fi printf '\n' else echo "preimage.sh: FAIL LOUD — WAKE_WATCH_LIST ('$WAKE_WATCH_LIST') is not a file; the preimage definition cannot be observed." >&2 failed=1 fi fi if [ -n "${WAKE_PREIMAGE_EXTRA:-}" ]; then local IFS=':' p for p in $WAKE_PREIMAGE_EXTRA; do [ -n "$p" ] || continue # Extras are tracked even when absent (ABSENT is a state). Resolve the # realpath only when the file exists; otherwise track the literal path. if [ -e "$p" ] && command -v realpath >/dev/null 2>&1; then realpath -- "$p" 2>/dev/null || printf '%s' "$p" printf '\n' else printf '%s\n' "$p" fi done fi return "$failed" } # --- credential hard gate (acceptance (b)) ---------------------------------- # _deny_path PATH — 0 (deny) iff PATH is a known credential-store location. # Framework-defined locations only (firewall): the mosaic credential store, # the tools/_lib credential file the framework-manifest itself carves out of # tools/**, the credentials/ operator subtree, and any file literally named # credentials.json. _deny_path() { local p="$1" home="${MOSAIC_HOME:-$HOME/.config/mosaic}" case "$p" in "$home/credentials.json") return 0 ;; "$home/tools/_lib/credentials.json") return 0 ;; "$home/credentials/"*) return 0 ;; esac case "$(basename -- "$p")" in credentials.json) return 0 ;; esac return 1 } # _deny_content FILE — 0 (deny) iff FILE's bytes match a well-known secret- # token shape. Same conservative shape set digest.sh redacts (PAT / Slack / # AKIA / JWT / PEM) — conservative on purpose: a 40-hex git sha never matches. _deny_content() { LC_ALL=C grep -Eq \ -e 'gh[pousr]_[A-Za-z0-9]{16,}' \ -e 'github_pat_[A-Za-z0-9_]{16,}' \ -e 'xox[baprs]-[A-Za-z0-9-]{10,}' \ -e 'AKIA[0-9A-Z]{16}' \ -e 'eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}' \ -e '-----BEGIN[A-Z ]*PRIVATE KEY-----' \ -- "$1" 2>/dev/null } # --- ledger primitives ------------------------------------------------------ # _ledger_head PATH — print the LAST ledger row for PATH (empty if none). # Non-zero (loud) if the ledger exists but is unparseable: a corrupt ledger # must REFUSE to compare, never silently re-baseline (that would erase the # attribution history this tool exists to keep). _ledger_head() { local path="$1" [ -f "$LEDGER" ] || return 0 if [ -s "$LEDGER" ] && ! jq -cn 'inputs' <"$LEDGER" >/dev/null 2>&1; then echo "preimage.sh: FAIL LOUD — preimage ledger $LEDGER is unparseable; REFUSING to compare or re-baseline over corrupt history. Investigate/restore the ledger." >&2 return 1 fi jq -c --arg p "$path" 'select(.path == $p)' "$LEDGER" 2>/dev/null | tail -n 1 } # _ledger_append ROW_JSON — append one row atomically (read + append + rename; # rows are small and the writer is serialized by the preimage lock). _ledger_append() { local row="$1" { [ -f "$LEDGER" ] && cat "$LEDGER" printf '%s\n' "$row" } | _atomic_write "$LEDGER" } # --- the check itself ------------------------------------------------------- # _check_one PATH ENQUEUE — compare PATH to its ledger head; on change record # (+bytes unless denied) and optionally enqueue. Prints nothing on no-change. # Returns: 0 ok (changed or not), 1 infrastructure failure. _check_one() { local path="$1" enqueue="$2" local cur_sha size mtime denied="" refused="" if [ -f "$path" ]; then cur_sha="$(_hash_stdin <"$path")" || return 1 [ -n "$cur_sha" ] || { echo "preimage.sh: FAIL LOUD — could not hash $path" >&2 return 1 } size="$(wc -c <"$path" | tr -d '[:space:]')" # Portable mtime (GNU stat -c / BSD stat -f). mtime="$(stat -c %Y -- "$path" 2>/dev/null || stat -f %m -- "$path" 2>/dev/null || echo 0)" local cap="${WAKE_PREIMAGE_MAX_BYTES:-1048576}" case "$cap" in '' | *[!0-9]*) cap=1048576 ;; esac # Deny order matters: path first (a known credential store is refused # without a content probe — the hash read above is deliberate: a sha256 # discloses nothing about the bytes), size cap second (never content-grep # an oversized file), content shape last. if _deny_path "$path"; then denied="credential-store path (deny list)" elif [ "$size" -gt "$cap" ]; then denied="exceeds WAKE_PREIMAGE_MAX_BYTES ($size > $cap)" elif _deny_content "$path"; then denied="secret-shaped content" fi else # ABSENT is a state (deletion of a preimage file is a change to record). cur_sha="ABSENT" size=0 mtime=0 fi local head prev_sha="" head="$(_ledger_head "$path")" || return 1 [ -n "$head" ] && prev_sha="$(jq -r '.sha256 // ""' <<<"$head")" if [ "$cur_sha" = "$prev_sha" ]; then return 0 # unchanged — no row, no enqueue (idempotent) fi # --- capture bytes (content-addressed) unless the hard gate refuses ------- local captured=true if [ "$cur_sha" != "ABSENT" ]; then if [ -n "$denied" ]; then captured=false refused="$denied" echo "preimage.sh: REFUSED byte capture for $path ($denied) — hash/size/mtime recorded, content NOT stored (acceptance (b), #958)." >&2 else mkdir -p "$OBJECTS" || return 1 if [ ! -f "$OBJECTS/$cur_sha" ]; then if ! _atomic_write "$OBJECTS/$cur_sha" <"$path"; then echo "preimage.sh: FAIL LOUD — durable object write failed for $path ($OBJECTS/$cur_sha); NO ledger row recorded (a row whose bytes were never durably kept would be a false witness)." >&2 return 1 fi fi fi fi local ts row ts="$(date +%s)" row="$(jq -cn \ --arg path "$path" \ --arg sha "$cur_sha" \ --arg prev "$prev_sha" \ --argjson size "$size" \ --argjson mtime "$mtime" \ --argjson ts "$ts" \ --argjson captured "$captured" \ --arg refused "$refused" \ '{ts:$ts, path:$path, sha256:$sha, size:$size, mtime:$mtime, captured:$captured} + (if $prev != "" then {prev:$prev} else {} end) + (if $refused != "" then {refused:$refused} else {} end)')" || return 1 if ! _ledger_append "$row"; then echo "preimage.sh: FAIL LOUD — durable ledger append failed ($LEDGER)" >&2 return 1 fi if [ -z "$head" ]; then # First-seen: baseline SILENTLY (row only, no enqueue) — the same idiom as # the detector's first-seen source baseline. return 0 fi echo "preimage.sh: preimage definition CHANGED — $path ($prev_sha -> $cur_sha); prior bytes: $OBJECTS/$prev_sha" >&2 if [ "$enqueue" = "1" ]; then # First-class cause line (acceptance (c)): ONE class=actionable entry per # changed preimage file, allocated by the store's single allocator (#908). # `path` is a §2.1 hard locator, so the digest renders it — never # quarantined. Callers run this BEFORE polling sources, so this seq is # LOWER than the per-source deltas the change explains. local locators locators="$(jq -cn \ --arg path "$path" \ --arg sha "$cur_sha" \ --arg prev "$prev_sha" \ '{kind:"preimage", id:$path, path:$path, observed_hash:$sha, prev_hash:$prev, preimage:true, reason:"preimage-definition-changed"}')" if ! "$STORE_SH" enqueue --class actionable --locators "$locators" --emit-ts "$(date +%s)" >/dev/null; then echo "preimage.sh: FAIL LOUD — store enqueue of the preimage-change entry FAILED for $path (the change IS recorded in the ledger; the first-class wake line is NOT — treat as infrastructure failure)." >&2 return 1 fi fi return 0 } cmd_check() { local enqueue=0 while [ $# -gt 0 ]; do case "$1" in --enqueue) enqueue=1 shift ;; *) echo "preimage.sh check: unknown option '$1'" >&2 exit 2 ;; esac done _need_jq local set_list if ! set_list="$(_preimage_set)"; then exit 1 fi if [ -z "$set_list" ]; then # Nothing declared (no adapter/watch-list/extras in env) — an empty set is # a no-op, not an error: standalone invocations outside a wake runtime # must not fail loud for lacking one. return 0 fi mkdir -p "$PRE_DIR" || exit 1 # Serialize concurrent checks (detector tick vs reconcile vs manual) — the # ledger append is read+rewrite, so two writers must not interleave. if ! _wake_lock_acquire "$PRE_DIR/preimage.lock"; then echo "preimage.sh: FAIL LOUD — cannot acquire preimage lock" >&2 exit 1 fi local failed=0 p while IFS= read -r p; do [ -n "$p" ] || continue _check_one "$p" "$enqueue" || failed=1 done <&2 return 0 } # Head row per path (last wins). jq -cs 'group_by(.path) | map(last) | .[]' "$LEDGER" } cmd_set() { _preimage_set || exit 1 } main() { [ $# -ge 1 ] || { usage exit 2 } local cmd="$1" shift case "$cmd" in check) cmd_check "$@" ;; record) cmd_check ;; set) cmd_set "$@" ;; status) cmd_status "$@" ;; -h | --help | help) usage ;; *) echo "preimage.sh: unknown command '$cmd'" >&2 usage exit 2 ;; esac } main "$@"