stack/packages/mosaic/framework/tools/wake/preimage.sh

#!/usr/bin/env bash
# preimage.sh — A11 of the wake canon (#958): durable PROVENANCE for the
# OPERATOR-SIDE preimage definition.
#
# THE GAP THIS CLOSES (#958): every observed_hash in the lane store is
# sha256(adapter stdout) — the operator's source adapter (plus the watch-list
# and any operator-declared preimage files) IS the preimage definition for
# every hash the pipeline ever records. Those files live OUTSIDE the versioned
# wake component, so a byte change to them was attributable only through agent
# transcripts: provenance thinner than any hash it feeds. A changed preimage
# also re-baselines EVERY source at once, which previously surfaced only as N
# per-source deltas / UNACCOUNTED lines with no first-class cause.
#
# WHAT THIS TOOL DOES:
#   - Derives the PREIMAGE SET generically (no operator paths baked in):
#       * the resolved file behind WAKE_DETECTOR_SOURCE_CMD (first word),
#       * the WAKE_WATCH_LIST file,
#       * operator-declared extras via WAKE_PREIMAGE_EXTRA (colon-separated —
#         e.g. detector.env, sink/feed scripts).
#   - Records each file's (sha256, size, mtime, ts) as an APPEND-ONLY ledger
#     row in <state>/preimage/preimage-ledger.jsonl and stores the full bytes
#     CONTENT-ADDRESSED at <state>/preimage/objects/<sha256> — so a change is
#     attributable (prior bytes + change time) from durable state alone, with
#     no transcript required. Acceptance (a) of #958.
#   - On a CHANGE (not first-seen), `check --enqueue` enqueues ONE first-class
#     class=actionable entry per changed file via the store's single allocator
#     (#908), carrying a §2.1 hard locator (`path`), BEFORE the per-source
#     deltas it explains are observed — so the digest shows the cause line,
#     not just N re-baselined sources. Acceptance (c).
#
# CREDENTIAL HARD GATE — acceptance (b): the mechanism must be UNABLE to
# capture credential material even by operator error. Enforced two ways, both
# fail-closed toward NOT capturing bytes (the hash/size/mtime row is still
# written — change-time attribution survives, content capture does not;
# a sha256 discloses nothing about the bytes):
#   1. PATH deny: the mosaic credential store locations
#      (<mosaic-home>/credentials.json, <mosaic-home>/tools/_lib/
#      credentials.json, anything under <mosaic-home>/credentials/, and any
#      file literally named credentials.json) are refused by resolved realpath
#      before a single byte is read into the object store.
#   2. CONTENT deny: a candidate whose bytes match the well-known secret-token
#      shapes (same conservative set digest.sh redacts: PAT/Slack/AKIA/JWT/PEM)
#      is refused — refusal, not redaction: a redacted preimage would be a
#      false witness, and secret bytes must never land in the object store.
#   Plus a size cap (WAKE_PREIMAGE_MAX_BYTES, default 1 MiB) so a stray extra
#   pointing at a large binary cannot turn provenance into data hoovering.
#
# FAIL-LOUD DISCIPLINE (the D2/#955 class, both layers): an infrastructure
# failure of THIS tool (unresolvable adapter, unreadable/corrupt ledger,
# failed durable write, failed enqueue) exits NON-ZERO and is never read as
# "no change". A corrupt ledger REFUSES to compare (and to re-baseline):
# silently restarting history would erase the very attribution this exists
# to provide.
#
# Operator-agnostic (framework firewall): all state via XDG/env; the deny
# list names only framework-defined credential locations, no operator hosts/
# names/secrets. This tool never prints file CONTENT to any stream.
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./_wake-common.sh disable=SC1091
. "$SCRIPT_DIR/_wake-common.sh"

STATE_DIR="$(wake_state_dir)"
STORE_SH="$SCRIPT_DIR/store.sh"
PRE_DIR="$STATE_DIR/preimage"
LEDGER="$PRE_DIR/preimage-ledger.jsonl"
OBJECTS="$PRE_DIR/objects"

_need_jq() {
  command -v jq >/dev/null 2>&1 || {
    echo "preimage.sh: jq is required" >&2
    exit 3
  }
}

# _hash_stdin — sha256 of stdin, first field only (portable, same fallback
# chain as detector.sh).
_hash_stdin() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum | awk '{print $1}'
  elif command -v shasum >/dev/null 2>&1; then
    shasum -a 256 | awk '{print $1}'
  elif command -v openssl >/dev/null 2>&1; then
    openssl dgst -sha256 | awk '{print $NF}'
  else
    echo "preimage.sh: no sha256 tool (sha256sum/shasum/openssl) available" >&2
    exit 3
  fi
}

usage() {
  cat >&2 <<'EOF'
Usage: preimage.sh <command>

Commands:
  check [--enqueue]   Compare every preimage-set file against the ledger head.
                      A changed file gets a new ledger row + captured bytes;
                      with --enqueue each change (not first-seen) also enqueues
                      ONE first-class class=actionable store entry (hard
                      locator: path). First-seen files baseline SILENTLY (row,
                      no enqueue — same idiom as the detector's first-seen).
                      Exit 0 whether or not changes were found; non-zero ONLY
                      on infrastructure failure (never read as "no change").
  record              Alias of `check` without enqueue (baseline/refresh).
  set                 Print the derived preimage set, one resolved path per
                      line (diagnostic).
  status              Print the ledger head row for every tracked path.

Environment:
  WAKE_DETECTOR_SOURCE_CMD  Adapter command; its resolved file joins the set.
  WAKE_WATCH_LIST           Watch-list path; joins the set when set.
  WAKE_PREIMAGE_EXTRA       Colon-separated additional operator preimage files
                            (e.g. detector.env, sink scripts). A listed path
                            that does not exist is tracked as ABSENT (deletion
                            of a preimage file is a change, not an error).
  WAKE_PREIMAGE_MAX_BYTES   Byte-capture cap (default 1048576). Larger files:
                            hash/size/mtime recorded, bytes refused.
  WAKE_STATE_HOME/WAKE_AGENT  store namespace (see store.sh).
EOF
}

# --- preimage-set derivation (generic; no operator paths baked in) ----------

# _resolve_cmd_file CMD — resolve the FIRST WORD of an adapter command string
# to a real file. Non-zero (loud) if it cannot be resolved: an adapter the
# detector will invoke but provenance cannot see is an infrastructure failure,
# not a smaller set.
_resolve_cmd_file() {
  local cmd="$1" word path
  # shellcheck disable=SC2086
  set -- $cmd
  word="${1:-}"
  [ -n "$word" ] || return 1
  if [ -f "$word" ]; then
    path="$word"
  else
    path="$(command -v -- "$word" 2>/dev/null)" || return 1
    [ -f "$path" ] || return 1
  fi
  # realpath so the ledger keys on the actual file, not a symlink alias.
  if command -v realpath >/dev/null 2>&1; then
    realpath -- "$path" 2>/dev/null || printf '%s' "$path"
  else
    printf '%s' "$path"
  fi
}

# _preimage_set — print the derived set, one path per line. Paths from
# WAKE_PREIMAGE_EXTRA are printed EVEN IF ABSENT (deletion is a tracked
# state); the adapter and watch-list must resolve (loud failure otherwise —
# see _resolve_cmd_file rationale).
_preimage_set() {
  local failed=0
  if [ -n "${WAKE_DETECTOR_SOURCE_CMD:-}" ]; then
    local af
    if af="$(_resolve_cmd_file "$WAKE_DETECTOR_SOURCE_CMD")"; then
      printf '%s\n' "$af"
    else
      echo "preimage.sh: FAIL LOUD — WAKE_DETECTOR_SOURCE_CMD ('$WAKE_DETECTOR_SOURCE_CMD') does not resolve to a file; the preimage definition cannot be observed (NOT treated as 'no change')." >&2
      failed=1
    fi
  fi
  if [ -n "${WAKE_WATCH_LIST:-}" ]; then
    if [ -f "$WAKE_WATCH_LIST" ]; then
      if command -v realpath >/dev/null 2>&1; then
        realpath -- "$WAKE_WATCH_LIST" 2>/dev/null || printf '%s' "$WAKE_WATCH_LIST"
      else
        printf '%s' "$WAKE_WATCH_LIST"
      fi
      printf '\n'
    else
      echo "preimage.sh: FAIL LOUD — WAKE_WATCH_LIST ('$WAKE_WATCH_LIST') is not a file; the preimage definition cannot be observed." >&2
      failed=1
    fi
  fi
  if [ -n "${WAKE_PREIMAGE_EXTRA:-}" ]; then
    local IFS=':' p
    for p in $WAKE_PREIMAGE_EXTRA; do
      [ -n "$p" ] || continue
      # Extras are tracked even when absent (ABSENT is a state). Resolve the
      # realpath only when the file exists; otherwise track the literal path.
      if [ -e "$p" ] && command -v realpath >/dev/null 2>&1; then
        realpath -- "$p" 2>/dev/null || printf '%s' "$p"
        printf '\n'
      else
        printf '%s\n' "$p"
      fi
    done
  fi
  return "$failed"
}

# --- credential hard gate (acceptance (b)) ----------------------------------

# _deny_path PATH — 0 (deny) iff PATH is a known credential-store location.
# Framework-defined locations only (firewall): the mosaic credential store,
# the tools/_lib credential file the framework-manifest itself carves out of
# tools/**, the credentials/ operator subtree, and any file literally named
# credentials.json.
_deny_path() {
  local p="$1" home="${MOSAIC_HOME:-$HOME/.config/mosaic}"
  case "$p" in
  "$home/credentials.json") return 0 ;;
  "$home/tools/_lib/credentials.json") return 0 ;;
  "$home/credentials/"*) return 0 ;;
  esac
  case "$(basename -- "$p")" in
  credentials.json) return 0 ;;
  esac
  return 1
}

# _deny_content FILE — 0 (deny) iff FILE's bytes match a well-known secret-
# token shape. Same conservative shape set digest.sh redacts (PAT / Slack /
# AKIA / JWT / PEM) — conservative on purpose: a 40-hex git sha never matches.
_deny_content() {
  LC_ALL=C grep -Eq \
    -e 'gh[pousr]_[A-Za-z0-9]{16,}' \
    -e 'github_pat_[A-Za-z0-9_]{16,}' \
    -e 'xox[baprs]-[A-Za-z0-9-]{10,}' \
    -e 'AKIA[0-9A-Z]{16}' \
    -e 'eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}' \
    -e '-----BEGIN[A-Z ]*PRIVATE KEY-----' \
    -- "$1" 2>/dev/null
}

# --- ledger primitives ------------------------------------------------------

# _ledger_head PATH — print the LAST ledger row for PATH (empty if none).
# Non-zero (loud) if the ledger exists but is unparseable: a corrupt ledger
# must REFUSE to compare, never silently re-baseline (that would erase the
# attribution history this tool exists to keep).
_ledger_head() {
  local path="$1"
  [ -f "$LEDGER" ] || return 0
  if [ -s "$LEDGER" ] && ! jq -cn 'inputs' <"$LEDGER" >/dev/null 2>&1; then
    echo "preimage.sh: FAIL LOUD — preimage ledger $LEDGER is unparseable; REFUSING to compare or re-baseline over corrupt history. Investigate/restore the ledger." >&2
    return 1
  fi
  jq -c --arg p "$path" 'select(.path == $p)' "$LEDGER" 2>/dev/null | tail -n 1
}

# _ledger_append ROW_JSON — append one row atomically (read + append + rename;
# rows are small and the writer is serialized by the preimage lock).
_ledger_append() {
  local row="$1"
  {
    [ -f "$LEDGER" ] && cat "$LEDGER"
    printf '%s\n' "$row"
  } | _atomic_write "$LEDGER"
}

# --- the check itself -------------------------------------------------------

# _check_one PATH ENQUEUE — compare PATH to its ledger head; on change record
# (+bytes unless denied) and optionally enqueue. Prints nothing on no-change.
# Returns: 0 ok (changed or not), 1 infrastructure failure.
_check_one() {
  local path="$1" enqueue="$2"
  local cur_sha size mtime denied="" refused=""

  if [ -f "$path" ]; then
    cur_sha="$(_hash_stdin <"$path")" || return 1
    [ -n "$cur_sha" ] || {
      echo "preimage.sh: FAIL LOUD — could not hash $path" >&2
      return 1
    }
    size="$(wc -c <"$path" | tr -d '[:space:]')"
    # Portable mtime (GNU stat -c / BSD stat -f).
    mtime="$(stat -c %Y -- "$path" 2>/dev/null || stat -f %m -- "$path" 2>/dev/null || echo 0)"
    local cap="${WAKE_PREIMAGE_MAX_BYTES:-1048576}"
    case "$cap" in '' | *[!0-9]*) cap=1048576 ;; esac
    # Deny order matters: path first (a known credential store is refused
    # without a content probe — the hash read above is deliberate: a sha256
    # discloses nothing about the bytes), size cap second (never content-grep
    # an oversized file), content shape last.
    if _deny_path "$path"; then
      denied="credential-store path (deny list)"
    elif [ "$size" -gt "$cap" ]; then
      denied="exceeds WAKE_PREIMAGE_MAX_BYTES ($size > $cap)"
    elif _deny_content "$path"; then
      denied="secret-shaped content"
    fi
  else
    # ABSENT is a state (deletion of a preimage file is a change to record).
    cur_sha="ABSENT"
    size=0
    mtime=0
  fi

  local head prev_sha=""
  head="$(_ledger_head "$path")" || return 1
  [ -n "$head" ] && prev_sha="$(jq -r '.sha256 // ""' <<<"$head")"

  if [ "$cur_sha" = "$prev_sha" ]; then
    return 0 # unchanged — no row, no enqueue (idempotent)
  fi

  # --- capture bytes (content-addressed) unless the hard gate refuses -------
  local captured=true
  if [ "$cur_sha" != "ABSENT" ]; then
    if [ -n "$denied" ]; then
      captured=false
      refused="$denied"
      echo "preimage.sh: REFUSED byte capture for $path ($denied) — hash/size/mtime recorded, content NOT stored (acceptance (b), #958)." >&2
    else
      mkdir -p "$OBJECTS" || return 1
      if [ ! -f "$OBJECTS/$cur_sha" ]; then
        if ! _atomic_write "$OBJECTS/$cur_sha" <"$path"; then
          echo "preimage.sh: FAIL LOUD — durable object write failed for $path ($OBJECTS/$cur_sha); NO ledger row recorded (a row whose bytes were never durably kept would be a false witness)." >&2
          return 1
        fi
      fi
    fi
  fi

  local ts row
  ts="$(date +%s)"
  row="$(jq -cn \
    --arg path "$path" \
    --arg sha "$cur_sha" \
    --arg prev "$prev_sha" \
    --argjson size "$size" \
    --argjson mtime "$mtime" \
    --argjson ts "$ts" \
    --argjson captured "$captured" \
    --arg refused "$refused" \
    '{ts:$ts, path:$path, sha256:$sha, size:$size, mtime:$mtime, captured:$captured}
       + (if $prev != "" then {prev:$prev} else {} end)
       + (if $refused != "" then {refused:$refused} else {} end)')" || return 1
  if ! _ledger_append "$row"; then
    echo "preimage.sh: FAIL LOUD — durable ledger append failed ($LEDGER)" >&2
    return 1
  fi

  if [ -z "$head" ]; then
    # First-seen: baseline SILENTLY (row only, no enqueue) — the same idiom as
    # the detector's first-seen source baseline.
    return 0
  fi

  echo "preimage.sh: preimage definition CHANGED — $path ($prev_sha -> $cur_sha); prior bytes: $OBJECTS/$prev_sha" >&2

  if [ "$enqueue" = "1" ]; then
    # First-class cause line (acceptance (c)): ONE class=actionable entry per
    # changed preimage file, allocated by the store's single allocator (#908).
    # `path` is a §2.1 hard locator, so the digest renders it — never
    # quarantined. Callers run this BEFORE polling sources, so this seq is
    # LOWER than the per-source deltas the change explains.
    local locators
    locators="$(jq -cn \
      --arg path "$path" \
      --arg sha "$cur_sha" \
      --arg prev "$prev_sha" \
      '{kind:"preimage", id:$path, path:$path, observed_hash:$sha, prev_hash:$prev,
        preimage:true, reason:"preimage-definition-changed"}')"
    if ! "$STORE_SH" enqueue --class actionable --locators "$locators" --emit-ts "$(date +%s)" >/dev/null; then
      echo "preimage.sh: FAIL LOUD — store enqueue of the preimage-change entry FAILED for $path (the change IS recorded in the ledger; the first-class wake line is NOT — treat as infrastructure failure)." >&2
      return 1
    fi
  fi
  return 0
}

cmd_check() {
  local enqueue=0
  while [ $# -gt 0 ]; do
    case "$1" in
    --enqueue)
      enqueue=1
      shift
      ;;
    *)
      echo "preimage.sh check: unknown option '$1'" >&2
      exit 2
      ;;
    esac
  done
  _need_jq

  local set_list
  if ! set_list="$(_preimage_set)"; then
    exit 1
  fi
  if [ -z "$set_list" ]; then
    # Nothing declared (no adapter/watch-list/extras in env) — an empty set is
    # a no-op, not an error: standalone invocations outside a wake runtime
    # must not fail loud for lacking one.
    return 0
  fi

  mkdir -p "$PRE_DIR" || exit 1
  # Serialize concurrent checks (detector tick vs reconcile vs manual) — the
  # ledger append is read+rewrite, so two writers must not interleave.
  if ! _wake_lock_acquire "$PRE_DIR/preimage.lock"; then
    echo "preimage.sh: FAIL LOUD — cannot acquire preimage lock" >&2
    exit 1
  fi
  local failed=0 p
  while IFS= read -r p; do
    [ -n "$p" ] || continue
    _check_one "$p" "$enqueue" || failed=1
  done <<EOF
$set_list
EOF
  _wake_lock_release
  [ "$failed" -eq 0 ] || exit 1
}

cmd_status() {
  _need_jq
  [ -f "$LEDGER" ] || {
    echo "preimage.sh: no ledger yet ($LEDGER)" >&2
    return 0
  }
  # Head row per path (last wins).
  jq -cs 'group_by(.path) | map(last) | .[]' "$LEDGER"
}

cmd_set() {
  _preimage_set || exit 1
}

main() {
  [ $# -ge 1 ] || {
    usage
    exit 2
  }
  local cmd="$1"
  shift
  case "$cmd" in
  check) cmd_check "$@" ;;
  record) cmd_check ;;
  set) cmd_set "$@" ;;
  status) cmd_status "$@" ;;
  -h | --help | help) usage ;;
  *)
    echo "preimage.sh: unknown command '$cmd'" >&2
    usage
    exit 2
    ;;
  esac
}

main "$@"