#!/usr/bin/env bash # preimage.sh — A11 of the wake canon (#958): durable PROVENANCE for the # OPERATOR-SIDE preimage definition. # # THE GAP THIS CLOSES (#958): every observed_hash in the lane store is # sha256(adapter stdout) — the operator's source adapter (plus the watch-list # and any operator-declared preimage files) IS the preimage definition for # every hash the pipeline ever records. Those files live OUTSIDE the versioned # wake component, so a byte change to them was attributable only through agent # transcripts: provenance thinner than any hash it feeds. A changed preimage # also re-baselines EVERY source at once, which previously surfaced only as N # per-source deltas / UNACCOUNTED lines with no first-class cause. # # WHAT THIS TOOL DOES: # - Derives the PREIMAGE SET generically (no operator paths baked in): # * the resolved file behind WAKE_DETECTOR_SOURCE_CMD (first word), # * the WAKE_WATCH_LIST file, # * operator-declared extras via WAKE_PREIMAGE_EXTRA (colon-separated — # e.g. sink/feed scripts). Extras are RECORD-ONLY by default (see the # credential hard gate below); never list key-material files. # - Records each file's (sha256, size, mtime, ts) as an APPEND-ONLY ledger # row in /preimage/preimage-ledger.jsonl and stores the full bytes # CONTENT-ADDRESSED at /preimage/objects/ — so a change is # attributable (prior bytes + change time) from durable state alone, with # no transcript required. Acceptance (a) of #958. # - On a CHANGE (not first-seen), `check --enqueue` enqueues ONE first-class # class=actionable entry per changed file via the store's single allocator # (#908), carrying a §2.1 hard locator (`path`), BEFORE the per-source # deltas it explains are observed — so the digest shows the cause line, # not just N re-baselined sources. Acceptance (c). # # CREDENTIAL HARD GATE — acceptance (b): the mechanism must be UNABLE to # capture credential material even by operator error. Layered, every layer # fail-closed toward NOT capturing bytes (the hash/size/mtime row is still # written — change-time attribution survives, content capture does not; # a sha256 discloses nothing about the bytes): # 1. POLARITY (#964 review, case D): byte capture is DENY-BY-DEFAULT for # operator extras. WAKE_PREIMAGE_EXTRA paths are RECORD-ONLY (rows + # first-class change entries, no bytes) unless explicitly listed in # WAKE_PREIMAGE_CAPTURE — a shape list can only refuse the secrets # someone already enumerated, so arbitrary operator-pointed files must # not default to capture. The CORE set (the resolved adapter file, the # watch-list) is capture-eligible: it IS the preimage definition this # tool exists to snapshot, and the deny layers below still apply to it. # 2. PATH deny: the mosaic credential store locations # (/credentials.json, /tools/_lib/ # credentials.json, anything under /credentials/, and any # file literally named credentials.json) are refused. Evaluated on BOTH # the operator-supplied form AND the symlink-resolved form, against BOTH # the unresolved and resolved forms of the mosaic-home anchor — a deny # list written only in unresolved paths cannot match a path that was # resolved before it arrived (#964 review, case C). # 3. CONTENT deny: a candidate whose bytes match the well-known secret-token # shapes (same conservative set digest.sh redacts: PAT/Slack/AKIA/JWT/PEM) # OR a named-assignment secret shape (key/token/secret/passw/hmac/ # credential/bearer = long unbroken value — catches prefix-less # high-entropy keys, e.g. an HMAC key in an env file) is refused — # refusal, not redaction: a redacted preimage would be a false witness, # and secret bytes must never land in the object store. Deliberately # conservative toward REFUSAL: a false match only withholds byte capture, # never tracking. # Plus a size cap (WAKE_PREIMAGE_MAX_BYTES, default 1 MiB) so a stray extra # pointing at a large binary cannot turn provenance into data hoovering. # Deny ALWAYS wins over the WAKE_PREIMAGE_CAPTURE opt-in. # # FAIL-LOUD DISCIPLINE (the D2/#955 class, both layers): an infrastructure # failure of THIS tool (unresolvable adapter, unreadable/corrupt ledger, # failed durable write, failed enqueue) exits NON-ZERO and is never read as # "no change". A corrupt ledger REFUSES to compare (and to re-baseline): # silently restarting history would erase the very attribution this exists # to provide. Likewise (#964 review, B11) an ABSENT/EMPTY ledger while # objects/ still holds prior captures is DELETED HISTORY, not a first # install — it refuses loudly instead of re-baselining, because a silent # restart would absorb the next real change as first-seen. # # Operator-agnostic (framework firewall): all state via XDG/env; the deny # list names only framework-defined credential locations, no operator hosts/ # names/secrets. This tool never prints file CONTENT to any stream. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=./_wake-common.sh disable=SC1091 . "$SCRIPT_DIR/_wake-common.sh" STATE_DIR="$(wake_state_dir)" STORE_SH="$SCRIPT_DIR/store.sh" PRE_DIR="$STATE_DIR/preimage" LEDGER="$PRE_DIR/preimage-ledger.jsonl" OBJECTS="$PRE_DIR/objects" _need_jq() { command -v jq >/dev/null 2>&1 || { echo "preimage.sh: jq is required" >&2 exit 3 } } # _hash_stdin — sha256 of stdin, first field only (portable, same fallback # chain as detector.sh). _hash_stdin() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}' elif command -v shasum >/dev/null 2>&1; then shasum -a 256 | awk '{print $1}' elif command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 | awk '{print $NF}' else echo "preimage.sh: no sha256 tool (sha256sum/shasum/openssl) available" >&2 exit 3 fi } usage() { cat >&2 <<'EOF' Usage: preimage.sh Commands: check [--enqueue] Compare every preimage-set file against the ledger head. A changed file gets a new ledger row (+ captured bytes when capture-eligible and not denied — see Environment); with --enqueue each change (not first-seen) also enqueues ONE first-class class=actionable store entry (hard locator: path). First-seen files baseline SILENTLY (row, no enqueue — same idiom as the detector's first-seen). Exit 0 whether or not changes were found; non-zero ONLY on infrastructure failure (never read as "no change"). record Alias of `check` without enqueue (baseline/refresh). set Print the derived preimage set, one resolved path per line (diagnostic). status Print the ledger head row for every tracked path. Environment: WAKE_DETECTOR_SOURCE_CMD Adapter command; its resolved file joins the set. WAKE_WATCH_LIST Watch-list path; joins the set when set. WAKE_PREIMAGE_EXTRA Colon-separated additional operator preimage files (e.g. sink/feed scripts). Extras are RECORD-ONLY by default: changes get a hash/size/mtime row and a first-class change entry, but their BYTES are never captured unless the path is also listed in WAKE_PREIMAGE_CAPTURE. A listed path that does not exist is tracked as ABSENT (deletion of a preimage file is a change, not an error). WAKE_PREIMAGE_CAPTURE Colon-separated allowlist of extras whose bytes MAY be captured. The credential deny rules ALWAYS win over this list. NEVER list files that can hold key material (env files with keys/HMACs, credential stores): the deny gate is a backstop, not a license. WAKE_PREIMAGE_MAX_BYTES Byte-capture cap (default 1048576). Larger files: hash/size/mtime recorded, bytes refused. WAKE_STATE_HOME/WAKE_AGENT store namespace (see store.sh). EOF } # --- preimage-set derivation (generic; no operator paths baked in) ---------- # _realpath_or_self PATH — resolved form when resolvable, the input otherwise. _realpath_or_self() { local p="$1" if command -v realpath >/dev/null 2>&1; then realpath -- "$p" 2>/dev/null || printf '%s' "$p" else printf '%s' "$p" fi } # _resolve_cmd_file CMD — resolve the FIRST WORD of an adapter command string # to a real file; prints `rawresolved`. Non-zero (loud) if it cannot be # resolved: an adapter the detector will invoke but provenance cannot see is # an infrastructure failure, not a smaller set. BOTH forms are kept: the deny # gate must see the pre-resolution form too (#964 review, case C). _resolve_cmd_file() { local cmd="$1" word path # shellcheck disable=SC2086 set -- $cmd word="${1:-}" [ -n "$word" ] || return 1 if [ -f "$word" ]; then path="$word" else path="$(command -v -- "$word" 2>/dev/null)" || return 1 [ -f "$path" ] || return 1 fi # realpath so the ledger keys on the actual file, not a symlink alias. printf '%s\t%s' "$path" "$(_realpath_or_self "$path")" } # _preimage_set — print the derived set, one member per line as # `originrawresolved` (origin: core|extra). BOTH path forms travel # with every member so the deny gate and the capture allowlist are evaluated # on the SAME candidate in BOTH its forms — resolving before denying is the # #964 case-C ordering defect. Paths from WAKE_PREIMAGE_EXTRA are printed # EVEN IF ABSENT (deletion is a tracked state); the adapter and watch-list # must resolve (loud failure otherwise — see _resolve_cmd_file rationale). _preimage_set() { local failed=0 if [ -n "${WAKE_DETECTOR_SOURCE_CMD:-}" ]; then local af if af="$(_resolve_cmd_file "$WAKE_DETECTOR_SOURCE_CMD")"; then printf 'core\t%s\n' "$af" else echo "preimage.sh: FAIL LOUD — WAKE_DETECTOR_SOURCE_CMD ('$WAKE_DETECTOR_SOURCE_CMD') does not resolve to a file; the preimage definition cannot be observed (NOT treated as 'no change')." >&2 failed=1 fi fi if [ -n "${WAKE_WATCH_LIST:-}" ]; then if [ -f "$WAKE_WATCH_LIST" ]; then printf 'core\t%s\t%s\n' "$WAKE_WATCH_LIST" "$(_realpath_or_self "$WAKE_WATCH_LIST")" else echo "preimage.sh: FAIL LOUD — WAKE_WATCH_LIST ('$WAKE_WATCH_LIST') is not a file; the preimage definition cannot be observed." >&2 failed=1 fi fi if [ -n "${WAKE_PREIMAGE_EXTRA:-}" ]; then local IFS=':' p for p in $WAKE_PREIMAGE_EXTRA; do [ -n "$p" ] || continue # Extras are tracked even when absent (ABSENT is a state). Resolve the # realpath only when the file exists; otherwise track the literal path. if [ -e "$p" ]; then printf 'extra\t%s\t%s\n' "$p" "$(_realpath_or_self "$p")" else printf 'extra\t%s\t%s\n' "$p" "$p" fi done fi return "$failed" } # --- credential hard gate (acceptance (b)) ---------------------------------- # _deny_path RAW RESOLVED — 0 (deny) iff EITHER form of the candidate names a # known credential-store location. Framework-defined locations only # (firewall): the mosaic credential store, the tools/_lib credential file the # framework-manifest itself carves out of tools/**, the credentials/ operator # subtree, and any file literally named credentials.json. # # Evaluated on BOTH the operator-supplied (raw) form and the symlink-resolved # form, against BOTH the unresolved and resolved forms of the mosaic-home # anchor: a deny list written only in unresolved paths cannot match a path # that was resolved before it arrived (#964 review, case C — a symlink whose # target was renamed slipped every path rule because the candidate had # already been realpath'd but the anchors never were). _deny_path() { local raw="$1" resolved="$2" local home="${MOSAIC_HOME:-$HOME/.config/mosaic}" rhome p a rhome="$(_realpath_or_self "$home")" for p in "$raw" "$resolved"; do [ -n "$p" ] || continue for a in "$home" "$rhome"; do case "$p" in "$a/credentials.json") return 0 ;; "$a/tools/_lib/credentials.json") return 0 ;; "$a/credentials/"*) return 0 ;; esac done case "$(basename -- "$p")" in credentials.json) return 0 ;; esac done return 1 } # _deny_content FILE — 0 (deny) iff FILE's bytes look secret-shaped. # Two probes: (a) the well-known vendor-token shapes digest.sh redacts # (PAT / Slack / AKIA / JWT / PEM — conservative: a 40-hex git sha never # matches); (b) a named-assignment shape (key/token/secret/passw/hmac/ # credential/bearer = long unbroken value) that catches prefix-less # high-entropy keys, e.g. an HMAC key in an env file (#964 review, case D). # # (b) is defense-in-depth for the OPT-IN path only, NOT the thing that keeps # case D safe — polarity does that (extras are record-only unless allowlisted; # a shape list can only refuse the secrets someone already enumerated). It is # deliberately conservative toward REFUSAL: a false match (a checksum in a # config, a path that happens to be long and unbroken) only withholds byte # capture — the hash/size/mtime row and change entry still land. A variable # REFERENCE (token="$GITEA_TOKEN") never matches: `$` is not in the value # class — only literal secret values do. # # FAILS TOWARD REFUSAL: if a probe itself errors (unreadable file, grep # failure — rc >= 2), the answer is DENY, not capture. An error exit is not a # negative result. _deny_content() { local rc LC_ALL=C grep -Eq \ -e 'gh[pousr]_[A-Za-z0-9]{16,}' \ -e 'github_pat_[A-Za-z0-9_]{16,}' \ -e 'xox[baprs]-[A-Za-z0-9-]{10,}' \ -e 'AKIA[0-9A-Z]{16}' \ -e 'eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}' \ -e '-----BEGIN[A-Z ]*PRIVATE KEY-----' \ -- "$1" 2>/dev/null rc=$? [ "$rc" -eq 1 ] || return 0 LC_ALL=C grep -Eiq \ -e "[A-Za-z0-9_]*(key|token|secret|passw|hmac|credential|bearer)[A-Za-z0-9_]*[[:space:]]*[=:][[:space:]]*[\"']?[A-Za-z0-9+/=_-]{16,}" \ -- "$1" 2>/dev/null rc=$? [ "$rc" -eq 1 ] || return 0 return 1 } # _capture_allowed ORIGIN RAW RESOLVED — 0 iff byte capture may even be # ATTEMPTED for this member (the deny gate still runs after, and wins). # POLARITY (#964 review, case D): core members (the adapter file, the # watch-list) are capture-eligible — they ARE the preimage definition this # tool exists to snapshot (acceptance (a)). Extras are RECORD-ONLY unless # listed in WAKE_PREIMAGE_CAPTURE. # # The allowlist is matched with the SAME both-forms discipline as the deny # list (entry raw/resolved vs candidate raw/resolved): two lists keyed on # different strings would let a symlink alias slip between them. "Deny wins # over opt-in" is a precedence rule, not a guarantee both lists see the same # path — the guarantee comes from _check_one running _deny_path on both forms # FIRST, independent of anything matched here. _capture_allowed() { local origin="$1" raw="$2" resolved="$3" [ "$origin" = "core" ] && return 0 [ -n "${WAKE_PREIMAGE_CAPTURE:-}" ] || return 1 local IFS=':' e er for e in $WAKE_PREIMAGE_CAPTURE; do [ -n "$e" ] || continue er="$e" [ -e "$e" ] && er="$(_realpath_or_self "$e")" if [ "$e" = "$raw" ] || [ "$e" = "$resolved" ] || [ "$er" = "$raw" ] || [ "$er" = "$resolved" ]; then return 0 fi done return 1 } # --- ledger primitives ------------------------------------------------------ # _ledger_head PATH — print the LAST ledger row for PATH (empty if none). # Non-zero (loud) if the ledger exists but is unparseable: a corrupt ledger # must REFUSE to compare, never silently re-baseline (that would erase the # attribution history this tool exists to keep). _ledger_head() { local path="$1" [ -f "$LEDGER" ] || return 0 if [ -s "$LEDGER" ] && ! jq -cn 'inputs' <"$LEDGER" >/dev/null 2>&1; then echo "preimage.sh: FAIL LOUD — preimage ledger $LEDGER is unparseable; REFUSING to compare or re-baseline over corrupt history. Investigate/restore the ledger." >&2 return 1 fi jq -c --arg p "$path" 'select(.path == $p)' "$LEDGER" 2>/dev/null | tail -n 1 } # _ledger_append ROW_JSON — append one row atomically (read + append + rename; # rows are small and the writer is serialized by the preimage lock). _ledger_append() { local row="$1" { [ -f "$LEDGER" ] && cat "$LEDGER" printf '%s\n' "$row" } | _atomic_write "$LEDGER" } # --- the check itself ------------------------------------------------------- # _check_one ORIGIN RAW PATH ENQUEUE — compare PATH (the resolved form; the # ledger keys on it) to its ledger head; on change record (+bytes only if # capture-eligible and not denied) and optionally enqueue. Prints nothing on # no-change. Returns: 0 ok (changed or not), 1 infrastructure failure. _check_one() { local origin="$1" raw="$2" path="$3" enqueue="$4" local cur_sha size mtime denied="" refused="" if [ -f "$path" ]; then cur_sha="$(_hash_stdin <"$path")" || return 1 [ -n "$cur_sha" ] || { echo "preimage.sh: FAIL LOUD — could not hash $path" >&2 return 1 } size="$(wc -c <"$path" | tr -d '[:space:]')" # Portable mtime (GNU stat -c / BSD stat -f). mtime="$(stat -c %Y -- "$path" 2>/dev/null || stat -f %m -- "$path" 2>/dev/null || echo 0)" local cap="${WAKE_PREIMAGE_MAX_BYTES:-1048576}" case "$cap" in '' | *[!0-9]*) cap=1048576 ;; esac # Gate order matters: path deny FIRST, on BOTH forms, INDEPENDENT of the # capture allowlist — so an allowlisted symlink whose target is denied # refuses no matter what string the allowlist matched (a known credential # store is refused without a content probe; the hash read above is # deliberate: a sha256 discloses nothing about the bytes). Then polarity # (record-only extras never reach the content probe — case D must be safe # WITHOUT the shape list), then size cap (never content-grep an oversized # file), content shape last. if _deny_path "$raw" "$path"; then denied="credential-store path (deny list)" elif ! _capture_allowed "$origin" "$raw" "$path"; then denied="extra is record-only by default (byte capture requires WAKE_PREIMAGE_CAPTURE; deny rules still win)" elif [ "$size" -gt "$cap" ]; then denied="exceeds WAKE_PREIMAGE_MAX_BYTES ($size > $cap)" elif _deny_content "$path"; then denied="secret-shaped content" fi else # ABSENT is a state (deletion of a preimage file is a change to record). cur_sha="ABSENT" size=0 mtime=0 fi local head prev_sha="" head="$(_ledger_head "$path")" || return 1 [ -n "$head" ] && prev_sha="$(jq -r '.sha256 // ""' <<<"$head")" if [ "$cur_sha" = "$prev_sha" ]; then return 0 # unchanged — no row, no enqueue (idempotent) fi # --- capture bytes (content-addressed) unless the hard gate refuses ------- local captured=true if [ "$cur_sha" != "ABSENT" ]; then if [ -n "$denied" ]; then captured=false refused="$denied" echo "preimage.sh: byte capture WITHHELD for $path ($denied) — hash/size/mtime recorded, content NOT stored (#958)." >&2 else mkdir -p "$OBJECTS" || return 1 if [ ! -f "$OBJECTS/$cur_sha" ]; then if ! _atomic_write "$OBJECTS/$cur_sha" <"$path"; then echo "preimage.sh: FAIL LOUD — durable object write failed for $path ($OBJECTS/$cur_sha); NO ledger row recorded (a row whose bytes were never durably kept would be a false witness)." >&2 return 1 fi fi fi fi local ts row ts="$(date +%s)" row="$(jq -cn \ --arg path "$path" \ --arg sha "$cur_sha" \ --arg prev "$prev_sha" \ --argjson size "$size" \ --argjson mtime "$mtime" \ --argjson ts "$ts" \ --argjson captured "$captured" \ --arg refused "$refused" \ '{ts:$ts, path:$path, sha256:$sha, size:$size, mtime:$mtime, captured:$captured} + (if $prev != "" then {prev:$prev} else {} end) + (if $refused != "" then {refused:$refused} else {} end)')" || return 1 if ! _ledger_append "$row"; then echo "preimage.sh: FAIL LOUD — durable ledger append failed ($LEDGER)" >&2 return 1 fi if [ -z "$head" ]; then # First-seen: baseline SILENTLY (row only, no enqueue) — the same idiom as # the detector's first-seen source baseline. return 0 fi echo "preimage.sh: preimage definition CHANGED — $path ($prev_sha -> $cur_sha); prior bytes: $OBJECTS/$prev_sha" >&2 if [ "$enqueue" = "1" ]; then # First-class cause line (acceptance (c)): ONE class=actionable entry per # changed preimage file, allocated by the store's single allocator (#908). # `path` is a §2.1 hard locator, so the digest renders it — never # quarantined. Callers run this BEFORE polling sources, so this seq is # LOWER than the per-source deltas the change explains. local locators locators="$(jq -cn \ --arg path "$path" \ --arg sha "$cur_sha" \ --arg prev "$prev_sha" \ '{kind:"preimage", id:$path, path:$path, observed_hash:$sha, prev_hash:$prev, preimage:true, reason:"preimage-definition-changed"}')" if ! "$STORE_SH" enqueue --class actionable --locators "$locators" --emit-ts "$(date +%s)" >/dev/null; then echo "preimage.sh: FAIL LOUD — store enqueue of the preimage-change entry FAILED for $path (the change IS recorded in the ledger; the first-class wake line is NOT — treat as infrastructure failure)." >&2 return 1 fi fi return 0 } cmd_check() { local enqueue=0 while [ $# -gt 0 ]; do case "$1" in --enqueue) enqueue=1 shift ;; *) echo "preimage.sh check: unknown option '$1'" >&2 exit 2 ;; esac done _need_jq local set_list if ! set_list="$(_preimage_set)"; then exit 1 fi if [ -z "$set_list" ]; then # Nothing declared (no adapter/watch-list/extras in env) — an empty set is # a no-op, not an error: standalone invocations outside a wake runtime # must not fail loud for lacking one. return 0 fi mkdir -p "$PRE_DIR" || exit 1 # Serialize concurrent checks (detector tick vs reconcile vs manual) — the # ledger append is read+rewrite, so two writers must not interleave. if ! _wake_lock_acquire "$PRE_DIR/preimage.lock"; then echo "preimage.sh: FAIL LOUD — cannot acquire preimage lock" >&2 exit 1 fi # #964 review (B11): an ABSENT/EMPTY ledger while objects/ still holds # prior captures is DELETED HISTORY, not a first install — the asymmetry is # locally detectable and is the whole detection. Re-baselining here would # silently absorb the next real change as first-seen (the exact erasure # this tool exists to prevent). ABSENT is not CORRUPT: it must not take the # first-install path either. if [ ! -s "$LEDGER" ] && [ -d "$OBJECTS" ] && [ -n "$(ls -A -- "$OBJECTS" 2>/dev/null)" ]; then echo "preimage.sh: FAIL LOUD — preimage ledger $LEDGER is ABSENT/EMPTY but $OBJECTS still holds prior objects: history was deleted; REFUSING to re-baseline over it (a silent restart would absorb the next real change as first-seen). Restore the ledger, or move objects/ aside explicitly after investigation." >&2 _wake_lock_release exit 1 fi local failed=0 origin raw resolved while IFS=$'\t' read -r origin raw resolved; do [ -n "$resolved" ] || continue _check_one "$origin" "$raw" "$resolved" "$enqueue" || failed=1 done <&2 return 0 } # Head row per path (last wins). jq -cs 'group_by(.path) | map(last) | .[]' "$LEDGER" } cmd_set() { local s s="$(_preimage_set)" || exit 1 [ -n "$s" ] && printf '%s\n' "$s" | cut -f3 } main() { [ $# -ge 1 ] || { usage exit 2 } local cmd="$1" shift case "$cmd" in check) cmd_check "$@" ;; record) cmd_check ;; set) cmd_set "$@" ;; status) cmd_status "$@" ;; -h | --help | help) usage ;; *) echo "preimage.sh: unknown command '$cmd'" >&2 usage exit 2 ;; esac } main "$@"