feat(mosaic): manifest-owned upgrade guard so updates never wipe operator config (#791)

Invert the framework updater from a denylist ("framework owns everything unless
preserved") to an explicit allow-list manifest ("operator owns everything unless
framework"). A path the manifest never anticipated resolves to operator-owned by
the fail-safe default, so it is structurally unreachable by any write or prune.

Root cause (#791): `mosaic update` re-seeds via `install.sh` keep-mode, whose
`rsync -a --delete` + hand-maintained PRESERVE_PATHS denylist wiped operator
paths the denylist forgot (agents/*.conf, policy/*.md, *.local.md, harvester
SOP, tools/_lib/credentials.json, unanticipated fleet files).

- framework-manifest.txt: single SSOT ([framework]/[operator], deny-wins,
  UNKNOWN=>operator fail-safe), read by BOTH installers.
- src/framework/manifest.ts: pure resolver (parse/matchGlob/resolveOwnership/
  frameworkSubtreeRoots/planPrune) — the testable seam.
- tools/_lib/manifest.sh: bash resolver (compiled globs, fork-free hot path),
  sourced by install.sh; parity-tested against the TS resolver.
- install.sh keep mode is now manifest-driven (no --delete): overlay-copy
  framework files, scoped-prune only retired framework files inside shipped
  subtrees. Operator + unknown paths are never written or deleted.
- file-ops.syncDirectory gains an isOperatorOwned guard; file-adapter derives it
  from the shared manifest, replacing the drifted hardcoded preservePaths.

Tests (TDD, red->green):
- HARD GATE test-upgrade-manifest-guard.sh: 10 operator sentinels (incl. an
  unanticipated one) survive a keep-mode reseed byte-identical + mtime-unchanged;
  retired framework file pruned; secret value absent from output. RED 31 fail on
  the old installer -> GREEN 48 pass. Wired merge-blocking into CI.
- manifest-parity.spec.ts (§6.1): bash<->TS agree on 34 paths + subtree roots.
- manifest.spec.ts: 18 tests incl. planPrune property test + shipped-tree
  completeness (§6.2).
- test-install-migration.sh F6 flipped: an unanticipated operator fleet file now
  MUST survive keep-mode reseed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

packages/mosaic/framework/tools/_lib/manifest.sh

@@ -0,0 +1,215 @@
+#!/usr/bin/env bash
+# Shared bash reader for framework-manifest.txt (#791).
+#
+# This is the bash half of the SSOT ownership resolver; the TypeScript half is
+# packages/mosaic/src/framework/manifest.ts. BOTH read the same
+# framework-manifest.txt and MUST resolve identical ownership for any path — the
+# parity test (manifest-parity.spec.ts) invokes this file's `resolve` CLI and
+# compares it against the TS resolver, so the two can never drift (the #631
+# two-copies failure class this closes).
+#
+# Ownership resolution (deny-wins / fail-safe):
+#   1. operator glob matches  -> operator
+#   2. else framework glob    -> framework
+#   3. else                   -> operator   (UNKNOWN defaults to operator, #791)
+#
+# Globs are compiled once at load into exact-prefix checks or POSIX EREs, so the
+# hot resolver (manifest_is_framework) forks no subprocesses — the installer
+# calls it once per file across the whole tree.
+#
+# Usage as a library (source it, then):
+#   manifest_load [manifest-file]      # populates + compiles the manifest
+#   manifest_is_framework <rel-path>   # rc 0 = framework-owned, rc 1 = operator
+#   manifest_resolve <rel-path>        # echoes: framework | operator
+#   manifest_subtree_roots             # echoes shipped framework `dir/**` roots
+#
+# Usage as a CLI (parity harness):
+#   bash manifest.sh resolve <rel-path>
+#   bash manifest.sh subtree-roots
+#   bash manifest.sh classify          # reads paths on stdin -> "<own>\t<path>"
+
+MANIFEST_FRAMEWORK=()
+MANIFEST_OPERATOR=()
+
+# Compiled forms (parallel arrays). _*_KIND[i] is "exact" or "re".
+_MF_KIND=(); _MF_EXACT=(); _MF_RE=()
+_MO_KIND=(); _MO_EXACT=(); _MO_RE=()
+_MF_ROOTS=()
+
+_manifest_default_root() { cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd; }
+
+# Normalize a path/glob: backslashes -> slashes, strip leading ./ and /, strip
+# trailing / (mirrors normalizeRel in manifest.ts).
+_manifest_norm() {
+  local p="$1"
+  p="${p//\\//}"
+  p="${p#./}"
+  while [[ "$p" == /* ]]; do p="${p#/}"; done
+  while [[ "$p" == */ ]]; do p="${p%/}"; done
+  printf '%s' "$p"
+}
+
+# Translate a normalized glob into a POSIX ERE body (mirrors globToRegExpBody).
+_manifest_glob_to_ere() {
+  local pattern; pattern="$(_manifest_norm "$1")"
+  local out="" c n i len=${#pattern} trailing
+  for (( i = 0; i < len; i++ )); do
+    c="${pattern:i:1}"
+    if [[ "$c" == "*" ]]; then
+      n="${pattern:i+1:1}"
+      if [[ "$n" == "*" ]]; then
+        i=$((i + 1))
+        trailing=0
+        if [[ "${pattern:i+1:1}" == "/" ]]; then i=$((i + 1)); trailing=1; fi
+        if [[ "$out" == */ ]]; then
+          out="${out%/}(/.*)?"
+        elif [[ "$trailing" -eq 1 ]]; then
+          out="$out(.*/)?"
+        else
+          out="$out.*"
+        fi
+      else
+        out="$out[^/]*"
+      fi
+    else
+      case "$c" in
+        .|+|\?|^|\$|\{|\}|\(|\)|\||\[|\]|\\) out="$out\\$c" ;;
+        *) out="$out$c" ;;
+      esac
+    fi
+  done
+  printf '%s' "$out"
+}
+
+# Compile one raw glob into (kind, exact, re) appended to the given section.
+# $1 = raw glob, $2 = section letter (F|O).
+_manifest_compile_one() {
+  local norm; norm="$(_manifest_norm "$1")"
+  [[ -n "$norm" ]] || return 0
+  if [[ "$norm" == *"*"* ]]; then
+    local re="^$(_manifest_glob_to_ere "$norm")\$"
+    if [[ "$2" == F ]]; then
+      _MF_KIND+=(re); _MF_EXACT+=(""); _MF_RE+=("$re")
+    else
+      _MO_KIND+=(re); _MO_EXACT+=(""); _MO_RE+=("$re")
+    fi
+  else
+    if [[ "$2" == F ]]; then
+      _MF_KIND+=(exact); _MF_EXACT+=("$norm"); _MF_RE+=("")
+    else
+      _MO_KIND+=(exact); _MO_EXACT+=("$norm"); _MO_RE+=("")
+    fi
+  fi
+  [[ "$2" == F && "$norm" == */"**" ]] && _MF_ROOTS+=("${norm%/**}")
+  return 0
+}
+
+_manifest_compile() {
+  _MF_KIND=(); _MF_EXACT=(); _MF_RE=(); _MF_ROOTS=()
+  _MO_KIND=(); _MO_EXACT=(); _MO_RE=()
+  local g
+  for g in "${MANIFEST_FRAMEWORK[@]:-}"; do [[ -n "$g" ]] && _manifest_compile_one "$g" F; done
+  for g in "${MANIFEST_OPERATOR[@]:-}"; do [[ -n "$g" ]] && _manifest_compile_one "$g" O; done
+}
+
+# Load + compile the manifest. Rejects a malformed file the same way
+# parseManifest() does (entry before a section header / unknown header).
+manifest_load() {
+  local file="${1:-}"
+  [[ -n "$file" ]] || file="$(_manifest_default_root)/framework-manifest.txt"
+  MANIFEST_FRAMEWORK=()
+  MANIFEST_OPERATOR=()
+  local section="" line
+  while IFS= read -r line || [[ -n "$line" ]]; do
+    line="${line#"${line%%[![:space:]]*}"}" # ltrim
+    line="${line%"${line##*[![:space:]]}"}" # rtrim
+    [[ -z "$line" || "${line:0:1}" == "#" ]] && continue
+    case "$line" in
+      "[framework]") section=framework; continue ;;
+      "[operator]") section=operator; continue ;;
+      "["*) echo "manifest: unknown section header: $line" >&2; return 1 ;;
+    esac
+    if [[ -z "$section" ]]; then
+      echo "manifest: entry before any [section] header: $line" >&2
+      return 1
+    fi
+    if [[ "$section" == framework ]]; then
+      MANIFEST_FRAMEWORK+=("$line")
+    else
+      MANIFEST_OPERATOR+=("$line")
+    fi
+  done < "$file"
+  _manifest_compile
+}
+
+# Fork-free: does $1 (a mosaic-home-relative path) match an operator glob?
+_mo_matches() {
+  local path="$1" i n=${#_MO_KIND[@]} re pat
+  for (( i = 0; i < n; i++ )); do
+    if [[ "${_MO_KIND[i]}" == exact ]]; then
+      pat="${_MO_EXACT[i]}"
+      [[ "$path" == "$pat" || "$path" == "$pat/"* ]] && return 0
+    else
+      re="${_MO_RE[i]}"
+      [[ "$path" =~ $re ]] && return 0
+    fi
+  done
+  return 1
+}
+
+# Fork-free: does $1 match a framework glob?
+_mf_matches() {
+  local path="$1" i n=${#_MF_KIND[@]} re pat
+  for (( i = 0; i < n; i++ )); do
+    if [[ "${_MF_KIND[i]}" == exact ]]; then
+      pat="${_MF_EXACT[i]}"
+      [[ "$path" == "$pat" || "$path" == "$pat/"* ]] && return 0
+    else
+      re="${_MF_RE[i]}"
+      [[ "$path" =~ $re ]] && return 0
+    fi
+  done
+  return 1
+}
+
+# The installer hot path — no subshell. rc 0 = framework-owned, rc 1 = operator
+# (deny-wins / fail-safe). Assumes an already-clean POSIX relative path.
+manifest_is_framework() {
+  _mo_matches "$1" && return 1
+  _mf_matches "$1" && return 0
+  return 1
+}
+
+# Echo the ownership of a path: framework | operator. Normalizes first, so it is
+# safe for CLI / test callers passing unnormalized input.
+manifest_resolve() {
+  local path; path="$(_manifest_norm "$1")"
+  if manifest_is_framework "$path"; then echo framework; else echo operator; fi
+}
+
+# Echo each shipped framework subtree root (a `dir/**` entry, without the /**).
+manifest_subtree_roots() {
+  local r
+  for r in "${_MF_ROOTS[@]:-}"; do [[ -n "$r" ]] && printf '%s\n' "$r"; done
+}
+
+# CLI dispatch — only when executed directly, never when sourced.
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+  set -o pipefail
+  manifest_load "${MANIFEST_FILE:-}"
+  cmd="${1:-}"
+  case "$cmd" in
+    resolve) manifest_resolve "${2:?path required}" ;;
+    subtree-roots) manifest_subtree_roots ;;
+    classify)
+      while IFS= read -r p; do
+        [[ -z "$p" ]] && continue
+        printf '%s\t%s\n' "$(manifest_resolve "$p")" "$p"
+      done
+      ;;
+    *)
+      echo "usage: manifest.sh {resolve <path>|subtree-roots|classify}" >&2
+      exit 2
+      ;;
+  esac
+fi

packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh

@@ -61,8 +61,10 @@ MOSAIC_HOME="$T5" MOSAIC_INSTALL_MODE=bogus MOSAIC_SYNC_ONLY=1 bash "$INSTALL" >
 chk "F5 failure: invalid mode rejected (nonzero exit)" "[ $rc -ne 0 ]"
 chk "F5 failure: SOUL + credentials intact" "grep -q orig '$T5/SOUL.md' && grep -q keepme '$T5/credentials/c.json'"
 
-# F6 — keep-mode re-seed (the `mosaic update` path) MUST preserve only the
-# exact user-owned roster paths while refreshing framework-owned schema/examples.
+# F6 — keep-mode re-seed (the `mosaic update` path) MUST preserve ALL user-owned
+# fleet state — including an unanticipated file the manifest never names, which
+# resolves to operator-owned by the #791 fail-safe — while refreshing the
+# framework-owned schema/examples.
 T6=$(mktemp -d); mkdir -p "$T6/fleet/examples" "$T6/fleet/run" "$T6/fleet/agents"
 printf '# persona\n' > "$T6/SOUL.md"   # makes it a recognized existing install (→ keep mode)
 printf 'version: 1\nagents:\n  - name: coder0\n' > "$T6/fleet/roster.yaml"
@@ -75,13 +77,14 @@ printf '{"stale":true}\n' > "$T6/fleet/roster.schema.json"
 E6=$(mktemp -d)
 cp "$T6/fleet/roster.yaml" "$E6/roster-yaml.expected"
 cp "$T6/fleet/roster.json" "$E6/roster-json.expected"
+cp "$T6/fleet/my-fleet.yaml" "$E6/my-fleet.expected"
 cp "$T6/fleet/run/coder0.hb" "$E6/run.expected"
 cp "$T6/fleet/agents/coder0.env" "$E6/agent.expected"
 echo 3 > "$T6/.framework-version"
 run "$T6" keep
 chk "F6 reseed: exact roster.yaml bytes survive keep-mode sync" "cmp -s '$T6/fleet/roster.yaml' '$E6/roster-yaml.expected'"
 chk "F6 reseed: exact roster.json bytes survive keep-mode sync" "cmp -s '$T6/fleet/roster.json' '$E6/roster-json.expected'"
-chk "F6 reseed: unrelated fleet YAML is not preserved" "[ ! -f '$T6/fleet/my-fleet.yaml' ]"
+chk "F6 reseed: unanticipated operator fleet file survives (fail-safe, #791)" "cmp -s '$T6/fleet/my-fleet.yaml' '$E6/my-fleet.expected'"
 chk "F6 reseed: per-agent env bytes survive" "cmp -s '$T6/fleet/agents/coder0.env' '$E6/agent.expected'"
 chk "F6 reseed: heartbeat bytes survive" "cmp -s '$T6/fleet/run/coder0.hb' '$E6/run.expected'"
 chk "F6 reseed: framework examples are refreshed" "grep -q orchestrator '$T6/fleet/examples/general.yaml'"

packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# test-upgrade-manifest-guard.sh — the #791 HARD GATE.
+#
+# Proves that a keep-mode framework upgrade (the `mosaic update` path:
+# install.sh with MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1) touches NO path
+# outside the framework-owned manifest. Every operator-owned sentinel — including
+# a deliberately UNANTICIPATED one the manifest never names — must survive
+# byte-identical with an unchanged mtime (not even rewritten). Framework files
+# must still update, and a retired framework file inside a shipped subtree must
+# still be pruned. No operator secret value may appear in installer output.
+#
+# Runs the whole matrix twice: once with rsync available, once forcing the
+# cp-based fallback (both destructive paths in install.sh must obey the manifest).
+#
+# Usage:  bash test-upgrade-manifest-guard.sh
+set -uo pipefail
+
+FW="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"   # packages/mosaic/framework
+INSTALL="$FW/install.sh"
+
+pass=0; fail=0
+chk() { if eval "$2"; then echo "  ✓ $1"; pass=$((pass + 1)); else echo "  ✗ $1"; fail=$((fail + 1)); fi; }
+
+SECRET='SUPER-SECRET-TOKEN-do-not-log-3f9a'
+
+# Seed a throwaway MOSAIC_HOME with an operator sentinel per ownership class.
+seed_home() {
+  local H="$1"
+  mkdir -p "$H/agents" "$H/policy" "$H/memory" "$H/tools/_lib" \
+           "$H/fleet/agents" "$H/harvester" "$H/unknown-operator-dir" "$H/guides"
+  printf '# persona\n' > "$H/SOUL.md"          # marks a recognized existing install → keep mode
+  printf 'MODEL=opus\n' > "$H/agents/coder0.conf"
+  printf '# operator policy\n' > "$H/policy/custom.md"
+  printf '# soul overlay\n' > "$H/SOUL.local.md"
+  printf '# operator memory\n' > "$H/memory/note.md"
+  printf 'TOKEN=%s\n' "$SECRET" > "$H/tools/_lib/credentials.json"
+  printf 'MOSAIC_AGENT_NAME=coder0\n' > "$H/fleet/agents/coder0.env"
+  printf 'version: 2\nagents:\n  - name: coder0\n' > "$H/fleet/roster.yaml"
+  printf '# harvester SOP\n' > "$H/harvester/sop.md"
+  printf 'operator data the manifest never anticipated\n' > "$H/unknown-operator-dir/x"
+  printf 'version: 1\nagents:\n  - name: mine\n' > "$H/fleet/my-fleet.yaml"
+  # A retired framework file inside a shipped subtree (absent from source) — must be pruned.
+  printf '# retired guide\n' > "$H/guides/RETIRED-OLD-GUIDE.md"
+  echo 3 > "$H/.framework-version"
+}
+
+OPERATOR_SENTINELS=(
+  "agents/coder0.conf"
+  "policy/custom.md"
+  "SOUL.local.md"
+  "memory/note.md"
+  "tools/_lib/credentials.json"
+  "fleet/agents/coder0.env"
+  "fleet/roster.yaml"
+  "harvester/sop.md"
+  "unknown-operator-dir/x"
+  "fleet/my-fleet.yaml"
+)
+
+run_matrix() {
+  local label="$1"; shift          # extra env / PATH override applied to the run
+  local H E OUT rel before_hash after_hash before_mt after_mt
+  H=$(mktemp -d); E=$(mktemp -d); OUT=$(mktemp)
+  seed_home "$H"
+
+  # Snapshot hash + mtime of every operator sentinel before the upgrade.
+  for rel in "${OPERATOR_SENTINELS[@]}"; do
+    sha256sum "$H/$rel" | awk '{print $1}' > "$E/$(echo "$rel" | tr / _).hash"
+    stat -c %Y "$H/$rel" > "$E/$(echo "$rel" | tr / _).mt"
+  done
+
+  # The upgrade under test (keep + sync-only = the `mosaic update` reseed path).
+  MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 "$@" bash "$INSTALL" >"$OUT" 2>&1
+
+  # HARD GATE: every operator sentinel survives byte-identical AND mtime-unchanged.
+  for rel in "${OPERATOR_SENTINELS[@]}"; do
+    before_hash=$(cat "$E/$(echo "$rel" | tr / _).hash")
+    before_mt=$(cat "$E/$(echo "$rel" | tr / _).mt")
+    after_hash=$(sha256sum "$H/$rel" 2>/dev/null | awk '{print $1}')
+    after_mt=$(stat -c %Y "$H/$rel" 2>/dev/null || echo MISSING)
+    chk "[$label] operator sentinel survives byte-identical: $rel" \
+      "[ -n '$after_hash' ] && [ '$before_hash' = '$after_hash' ]"
+    chk "[$label] operator sentinel not rewritten (mtime unchanged): $rel" \
+      "[ '$before_mt' = '$after_mt' ]"
+  done
+
+  # Positive controls: the framework tree still updates, retired file pruned.
+  chk "[$label] framework file present after upgrade (guides synced)" \
+    "[ -f '$H/guides/E2E-DELIVERY.md' ]"
+  chk "[$label] retired framework file inside a subtree is pruned" \
+    "[ ! -f '$H/guides/RETIRED-OLD-GUIDE.md' ]"
+  chk "[$label] manifest itself is installed" "[ -f '$H/framework-manifest.txt' ]"
+
+  # Secret-safety: the operator secret value never appears in installer output.
+  chk "[$label] operator secret value absent from installer stdout/stderr" \
+    "! grep -q '$SECRET' '$OUT'"
+
+  rm -rf "$H" "$E" "$OUT"
+}
+
+echo "#791 upgrade manifest guard (HARD GATE):"
+
+# 1) rsync path (if available on this host).
+if command -v rsync >/dev/null 2>&1; then
+  run_matrix "rsync"
+else
+  echo "  · rsync not installed — skipping rsync-path matrix"
+fi
+
+# 2) cp fallback path — hide rsync behind a scratch PATH so install.sh takes the
+#    find/rm+cp branch. Provide the coreutils the fallback needs.
+FBIN=$(mktemp -d)
+for t in bash cp find mktemp rm mkdir chmod cmp sed grep cat dirname basename stat sha256sum awk tr; do
+  p=$(command -v "$t" 2>/dev/null) && ln -s "$p" "$FBIN/$t"
+done
+run_matrix "cp-fallback" env "PATH=$FBIN"
+rm -rf "$FBIN"
+
+echo
+echo "RESULT: $pass passed, $fail failed"
+[ "$fail" -eq 0 ]