feat(mosaic): durable pre-update snapshot + verify net + restore CLI (#791 PR2)
Some checks failed
ci/woodpecker/pr/ci Pipeline failed

Third, independent recovery layer for keep-mode upgrades (Part of #791). PR1's
manifest keeps the sync out of operator paths and the ephemeral /tmp snapshot
rolls back a mid-write crash; this adds the case neither covers — a "successful"
upgrade that a manifest/logic bug silently let touch an operator file.

- install.sh make_durable_snapshot(): before any mutation, copy the operator-owned
  surface that exists to $XDG_STATE_HOME/mosaic/backups/pre-update-<UTC-ts>/
  (0700 dirs / 0600 files, umask 077 scoped + restored). Fail-open; retention
  MOSAIC_BACKUP_RETENTION (default 5).
- install.sh verify_operator_surface(): post-sync, restore any operator file the
  upgrade diverged/removed and warn loudly (a divergence = manifest bug). Honors
  intentional migration removals; refuses to restore through a symlink (CWE-59);
  guards mkdir under set -e.
- mosaic restore (TS): --list (default dry-run), --from <ts> confirmation-gated
  restore. Strict timestamp selector (CWE-22), O_NOFOLLOW 0600 writes (CWE-59),
  containment + managed-dir checks via secure-file.ts. Counts/paths only — never
  emits file contents (secrev).
- Gates: typecheck/lint/format ✓; mosaic vitest 1252; restore.spec 30;
  bash gates durable-snapshot 41 / manifest-guard 193 / rollback 28 / migration 21.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Hermes Agent
2026-07-16 19:04:14 -05:00
parent 32a0ffba13
commit 8bee1b65ea
8 changed files with 1403 additions and 4 deletions

View File

@@ -0,0 +1,292 @@
#!/usr/bin/env bash
# test-upgrade-durable-snapshot.sh — the #791 PR2 regression gate.
#
# PR1 gave keep-mode upgrades two protections: the manifest (a keep-sync only
# ever writes framework-owned paths — operator config is structurally untouched)
# and an EPHEMERAL /tmp snapshot that rolls the whole target back if the sync
# CRASHES mid-write. PR2 adds a third, independent layer for the case neither
# covers: a "successful" upgrade that a manifest/logic bug silently let touch an
# operator file. That layer is a DURABLE, operator-scoped pre-update snapshot:
#
# Part 1 (scope): before any mutation, the installer copies exactly the
# operator-owned files that exist into a retained backup
# under $XDG_STATE_HOME/mosaic/backups/pre-update-<ts>/ —
# framework files are NOT captured.
# Part 2 (perms): the backup root, snapshot dir and every nested dir are
# 0700; every backed-up file is 0600 (never world-readable,
# even though operator config may hold secrets).
# Part 3 (no leak): a secret seeded into credentials.json is copied into the
# snapshot (proving coverage) but its value never appears
# on stdout/stderr — the snapshot reports counts/paths only.
# Part 4 (retention): only the newest MOSAIC_BACKUP_RETENTION snapshots survive;
# older ones are pruned.
# Part 5 (verify net): if the upgrade DID modify an operator file (injected here
# with a cp shim that scribbles on SOUL.md while a framework
# file is copied), the post-sync verify restores that file
# from the durable snapshot and warns loudly. The control —
# the same installer with the verify call stripped — leaves
# the corruption in place, proving the net is load-bearing.
#
# Usage: bash test-upgrade-durable-snapshot.sh
set -uo pipefail
FW="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" # packages/mosaic/framework
INSTALL="$FW/install.sh"
ORIG_PATH="$PATH"
FRAMEWORK_VERSION="$(grep -m1 '^FRAMEWORK_VERSION=' "$INSTALL" | cut -d= -f2)"
# Control installers must live INSIDE $FW: install.sh derives SOURCE_DIR from its
# own path and sources tools/_lib/manifest.sh relative to it, so a copy anywhere
# else aborts before the sync. Each control is a shipped installer with one guard
# line stripped (keyed off a `# <MARKER>` anchor), proving that guard load-bearing.
# All controls share the .install-*.tmp.sh glob so one trap sweeps them on exit.
VERIFYCTRL="$FW/.install-verifynet-control.tmp.sh"
rm -f "$FW"/.install-*.tmp.sh
trap 'rm -f "$FW"/.install-*.tmp.sh' EXIT
# mk_control <marker-regex> <name> — echo a control installer path ($FW-local) that
# is $INSTALL with every line matching /<marker-regex>/ deleted.
mk_control() {
local path="$FW/.install-$2.tmp.sh"
sed "/$1/d" "$INSTALL" > "$path"
printf '%s' "$path"
}
pass=0; fail=0
chk() { if eval "$2"; then echo "$1"; pass=$((pass + 1)); else echo "$1"; fail=$((fail + 1)); fi; }
SECRET='SUPER-SECRET-TOKEN-do-not-log-pr2'
SOUL_ORIG='# persona'
# A framework file the sync copies (source ships it; the seeded target omits it,
# so the bytes differ and cp is attempted). The Part-5 shim keys off this path.
POISON_REL='guides/E2E-DELIVERY.md'
# Seed a recognized keep-mode install holding four operator-owned files across
# the identity file, an operator subtree, memory, and the credentials carve-out.
seed_home() {
local H="$1"
mkdir -p "$H/agents" "$H/tools/_lib" "$H/memory"
printf '%s\n' "$SOUL_ORIG" > "$H/SOUL.md" # recognized install → keep mode
printf 'MODEL=opus\n' > "$H/agents/coder0.conf"
printf '# operator memory\n' > "$H/memory/note.md"
printf 'TOKEN=%s\n' "$SECRET" > "$H/tools/_lib/credentials.json"
echo 3 > "$H/.framework-version"
# Deliberately NO guides/E2E-DELIVERY.md so the sync copies it (framework file,
# bytes differ) — that copy is where the Part-5 corruption shim fires.
}
# A pre-v2 (legacy) keep-mode install: SOUL.md marks it recognized, and a bin/
# tree with NO .framework-version makes installed_framework_version() report 1, so
# the v1→v2 migration (which deletes bin/) runs. bin/ is unknown⇒operator, so the
# durable snapshot captures it — the verify net must NOT heal the intended removal.
seed_home_v1() {
local H="$1"
mkdir -p "$H/agents" "$H/tools/_lib" "$H/memory" "$H/bin"
printf '%s\n' "$SOUL_ORIG" > "$H/SOUL.md"
printf 'TOKEN=%s\n' "$SECRET" > "$H/tools/_lib/credentials.json"
printf '#!/bin/sh\necho legacy\n' > "$H/bin/tool.sh"; chmod +x "$H/bin/tool.sh"
# Deliberately NO .framework-version and NO guides/E2E-DELIVERY.md (see seed_home).
}
# A cp shim that, while the framework POISON file is being copied during sync,
# swaps the operator credentials file for a symlink pointing at an attacker-
# readable file OUTSIDE the target — simulating post-snapshot tampering (CWE-59).
# The durable snapshot already holds the real credentials (it is taken before any
# sync), so the verify net must restore a REAL file in place WITHOUT following the
# link (which would write the snapshot's secret out through it). $EXFIL_TARGET is
# expanded at shim-write time from the caller's environment.
make_symlink_leaf_shim() {
local dir="$1" home="$2"
cat > "$dir/cp" <<SHIM
#!/usr/bin/env bash
dest="\${@: -1}"
case "\$dest" in
*/$POISON_REL)
rm -f "$home/tools/_lib/credentials.json"
ln -s "$EXFIL_TARGET" "$home/tools/_lib/credentials.json"
;;
esac
exec env PATH="$ORIG_PATH" cp "\$@"
SHIM
chmod +x "$dir/cp"
}
# A cp shim that, while the framework POISON file is being copied during sync,
# also appends garbage to the operator SOUL.md — simulating a manifest bug that
# writes outside the framework lane. The framework copy itself still succeeds
# (real cp runs), so the sync completes 0 and the post-sync verify is what must
# catch and undo the operator-file damage. The snapshot's own cp only ever
# targets operator files (never guides/…), so it is never corrupted by this shim.
make_corrupt_shim() {
local dir="$1" home="$2"
cat > "$dir/cp" <<SHIM
#!/usr/bin/env bash
dest="\${@: -1}"
case "\$dest" in
*/$POISON_REL) printf 'CORRUPTION-mid-sync\n' >> "$home/SOUL.md" 2>/dev/null || true ;;
esac
exec env PATH="$ORIG_PATH" cp "\$@"
SHIM
chmod +x "$dir/cp"
}
# Run one keep-mode, sync-only upgrade with $XDG_STATE_HOME redirected to a
# throwaway dir (so the real ~/.local/state is never touched). Optional args:
# $2 shim-maker (default none), $3 MOSAIC_BACKUP_RETENTION (default unset).
# Echoes: "<exit>\t<out>\t<state-dir>\t<home>".
# $4 seeder (default seed_home) — swap in seed_home_v1 for the migration case.
run_snap() {
local installer="$1" shim_maker="${2:-}" retention="${3:-}" seeder="${4:-seed_home}" H STATE OUT SHIM rc pathpre
H=$(mktemp -d); STATE=$(mktemp -d); OUT=$(mktemp); pathpre="$ORIG_PATH"
"$seeder" "$H"
if [[ -n "$shim_maker" ]]; then
SHIM=$(mktemp -d); "$shim_maker" "$SHIM" "$H"; pathpre="$SHIM:$ORIG_PATH"
fi
set +e
env PATH="$pathpre" XDG_STATE_HOME="$STATE" \
${retention:+MOSAIC_BACKUP_RETENTION="$retention"} \
MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 \
bash "$installer" >"$OUT" 2>&1
rc=$?
set -e 2>/dev/null || true
[[ -n "$shim_maker" ]] && rm -rf "$SHIM"
printf '%s\t%s\t%s\t%s\n' "$rc" "$OUT" "$STATE" "$H"
}
# Resolve the single pre-update-* snapshot dir under a state dir (newest if many).
snap_dir() {
find "$1/mosaic/backups" -maxdepth 1 -type d -name 'pre-update-*' 2>/dev/null \
| LC_ALL=C sort -r | head -1
}
echo "── Part 1/2/3: durable snapshot scope, perms, no-leak ──────────────────"
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$INSTALL")
SNAP="$(snap_dir "$STATE")"
chk "upgrade succeeds" "[ '$rc' -eq 0 ]"
chk "exactly one pre-update snapshot created" "[ \$(find '$STATE/mosaic/backups' -maxdepth 1 -type d -name 'pre-update-*' | wc -l) -eq 1 ]"
chk "snapshot: SOUL.md captured" "[ -f '$SNAP/SOUL.md' ]"
chk "snapshot: operator subtree captured" "[ -f '$SNAP/agents/coder0.conf' ]"
chk "snapshot: memory captured" "[ -f '$SNAP/memory/note.md' ]"
chk "snapshot: credentials carve-out captured" "[ -f '$SNAP/tools/_lib/credentials.json' ]"
chk "snapshot: SOUL.md bytes preserved" "[ \"\$(cat '$SNAP/SOUL.md')\" = '$SOUL_ORIG' ]"
chk "snapshot: framework file NOT captured" "[ ! -e '$SNAP/CONSTITUTION.md' ] && [ ! -e '$SNAP/$POISON_REL' ]"
# Part 2 — permissions (0700 dirs, 0600 files); never world-readable.
chk "perms: backup root is 0700" "[ \$(stat -c '%a' '$STATE/mosaic/backups') -eq 700 ]"
chk "perms: snapshot dir is 0700" "[ \$(stat -c '%a' '$SNAP') -eq 700 ]"
chk "perms: nested dir is 0700" "[ \$(stat -c '%a' '$SNAP/agents') -eq 700 ]"
chk "perms: credentials backup is 0600" "[ \$(stat -c '%a' '$SNAP/tools/_lib/credentials.json') -eq 600 ]"
chk "perms: SOUL.md backup is 0600" "[ \$(stat -c '%a' '$SNAP/SOUL.md') -eq 600 ]"
# Part 3 — the secret is backed up but never emitted to stdout/stderr.
chk "no-leak: secret IS in the backup file" "grep -q '$SECRET' '$SNAP/tools/_lib/credentials.json'"
chk "no-leak: secret NOT on stdout/stderr" "! grep -q '$SECRET' '$OUT'"
rm -rf "$STATE" "$H"; rm -f "$OUT"
echo "── Part 4: retention prune (MOSAIC_BACKUP_RETENTION) ───────────────────"
# Pre-seed four dated snapshots, then take one real snapshot with retention=2:
# only the two newest (the fresh real one + the newest pre-seeded) must survive.
IFS=$'\t' read -r rc OUT STATE H < <(
H=$(mktemp -d); STATE=$(mktemp -d); OUT=$(mktemp)
seed_home "$H"
mkdir -p "$STATE/mosaic/backups"
for ts in 20200101T000000Z 20210101T000000Z 20220101T000000Z 20230101T000000Z; do
mkdir -p "$STATE/mosaic/backups/pre-update-$ts"
done
set +e
env PATH="$ORIG_PATH" XDG_STATE_HOME="$STATE" MOSAIC_BACKUP_RETENTION=2 \
MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 \
bash "$INSTALL" >"$OUT" 2>&1
rc=$?
set -e 2>/dev/null || true
printf '%s\t%s\t%s\t%s\n' "$rc" "$OUT" "$STATE" "$H"
)
chk "retention: upgrade succeeds" "[ '$rc' -eq 0 ]"
chk "retention: pruned to exactly 2 snapshots" "[ \$(find '$STATE/mosaic/backups' -maxdepth 1 -type d -name 'pre-update-*' | wc -l) -eq 2 ]"
chk "retention: newest pre-seeded survives" "[ -d '$STATE/mosaic/backups/pre-update-20230101T000000Z' ]"
chk "retention: oldest pre-seeded pruned" "[ ! -d '$STATE/mosaic/backups/pre-update-20200101T000000Z' ]"
rm -rf "$STATE" "$H"; rm -f "$OUT"
echo "── Part 5: post-sync verify restores an operator file (+ control) ──────"
# Shipped installer: the cp shim corrupts SOUL.md mid-sync; verify must restore it.
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$INSTALL" make_corrupt_shim)
chk "verify: upgrade still succeeds" "[ '$rc' -eq 0 ]"
chk "verify: SOUL.md restored to original" "[ \"\$(cat '$H/SOUL.md')\" = '$SOUL_ORIG' ]"
chk "verify: no corruption remains in SOUL.md" "! grep -q 'CORRUPTION-mid-sync' '$H/SOUL.md'"
chk "verify: loud restore warning emitted" "grep -qi 'restored from the pre-update snapshot' '$OUT'"
chk "verify: secret still not leaked" "! grep -q '$SECRET' '$OUT'"
rm -rf "$STATE" "$H"; rm -f "$OUT"
# Control: strip the verify call → the corruption must SURVIVE (net is load-bearing).
sed '/# VERIFY-NET/d' "$INSTALL" > "$VERIFYCTRL"
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$VERIFYCTRL" make_corrupt_shim)
chk "control: SOUL.md corruption survives" "grep -q 'CORRUPTION-mid-sync' '$H/SOUL.md'"
chk "control: no restore warning emitted" "! grep -qi 'restored from the pre-update snapshot' '$OUT'"
rm -rf "$STATE" "$H"; rm -f "$OUT"
echo "── Part 6: verify net honors an intentional migration removal (+ control) ─"
# BLOCKER regression: on a pre-v2 install, bin/ is operator-classified so the durable
# snapshot captures it — but the v1→v2 migration deletes bin/ ON PURPOSE. The verify
# net must SKIP that removal (is_migration_removed), or it heals bin/ back and the
# migration is silently undone forever once the version is stamped.
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$INSTALL" "" "" seed_home_v1)
chk "migration: upgrade succeeds" "[ '$rc' -eq 0 ]"
chk "migration: legacy bin/ stays removed" "[ ! -e '$H/bin' ]"
chk "migration: operator SOUL.md untouched" "[ \"\$(cat '$H/SOUL.md')\" = '$SOUL_ORIG' ]"
chk "migration: version stamped to $FRAMEWORK_VERSION" "[ \"\$(cat '$H/.framework-version')\" = '$FRAMEWORK_VERSION' ]"
rm -rf "$STATE" "$H"; rm -f "$OUT"
# Control: strip the MIGRATION-SKIP-GUARD → the verify net restores bin/ from the
# snapshot, silently undoing the migration (proves the guard is load-bearing).
MIGCTRL="$(mk_control 'MIGRATION-SKIP-GUARD' migration-control)"
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$MIGCTRL" "" "" seed_home_v1)
chk "control: bin/ wrongly restored by verify" "[ -e '$H/bin/tool.sh' ]"
rm -rf "$STATE" "$H"; rm -f "$OUT"
echo "── Part 7: verify net never restores a secret through a symlink (+ control) ─"
# HIGH (CWE-59) regression: an attacker who swaps an operator file for a symlink
# AFTER the durable snapshot must not cause the verify net's restore to write the
# snapshot's secret out THROUGH that link. The shipped net drops a symlinked leaf and
# writes a real file in its place, leaving the external target untouched.
EXFIL_DIR=$(mktemp -d); EXFIL_TARGET="$EXFIL_DIR/stolen"
printf 'ATTACKER-PLACEHOLDER\n' > "$EXFIL_TARGET"
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$INSTALL" make_symlink_leaf_shim)
chk "symlink-leaf: upgrade succeeds" "[ '$rc' -eq 0 ]"
chk "symlink-leaf: secret NOT written through link" "! grep -q '$SECRET' '$EXFIL_TARGET'"
chk "symlink-leaf: credentials.json is a real file" "[ -f '$H/tools/_lib/credentials.json' ] && [ ! -L '$H/tools/_lib/credentials.json' ]"
chk "symlink-leaf: credentials.json restored intact" "grep -q '$SECRET' '$H/tools/_lib/credentials.json'"
chk "symlink-leaf: secret not leaked to stdout/stderr" "! grep -q '$SECRET' '$OUT'"
rm -rf "$STATE" "$H" "$EXFIL_DIR"; rm -f "$OUT"
# Control: strip the SYMLINK-LEAF-GUARD → cp follows the swapped-in link and writes
# the snapshot secret out through it (proves the guard is load-bearing).
EXFIL_DIR=$(mktemp -d); EXFIL_TARGET="$EXFIL_DIR/stolen"
printf 'ATTACKER-PLACEHOLDER\n' > "$EXFIL_TARGET"
LEAFCTRL="$(mk_control 'SYMLINK-LEAF-GUARD' symlinkleaf-control)"
IFS=$'\t' read -r rc OUT STATE H < <(run_snap "$LEAFCTRL" make_symlink_leaf_shim)
chk "control: secret leaked through the symlink" "grep -q '$SECRET' '$EXFIL_TARGET'"
rm -rf "$STATE" "$H" "$EXFIL_DIR"; rm -f "$OUT"
echo "── Part 8: snapshot umask 077 does not leak into synced files (+ control) ──"
# SHOULD-FIX regression: umask 077 is process-global. Scoped to the snapshot it keeps
# backups 0600; leaked past it, every later cp/mkdir inherits 0600/0700. A freshly-
# synced framework file must be 0644 (per the ambient 022 umask) while the backup of
# a secret stays 0600.
IFS=$'\t' read -r rc OUT STATE H < <(umask 022; run_snap "$INSTALL")
SNAP="$(snap_dir "$STATE")"
chk "umask: upgrade succeeds" "[ '$rc' -eq 0 ]"
chk "umask: synced framework file is 0644" "[ \$(stat -c '%a' '$H/$POISON_REL') -eq 644 ]"
chk "umask: backup of a secret stays 0600" "[ \$(stat -c '%a' '$SNAP/tools/_lib/credentials.json') -eq 600 ]"
rm -rf "$STATE" "$H"; rm -f "$OUT"
# Control: strip the UMASK-RESTORE-NORMAL line → umask 077 leaks past the snapshot,
# so the newly-synced framework file is created 0600 (proves the restore matters).
UMASKCTRL="$(mk_control 'UMASK-RESTORE-NORMAL' umask-control)"
IFS=$'\t' read -r rc OUT STATE H < <(umask 022; run_snap "$UMASKCTRL")
chk "control: leaked umask makes synced file 0600" "[ \$(stat -c '%a' '$H/$POISON_REL') -eq 600 ]"
rm -rf "$STATE" "$H"; rm -f "$OUT"
echo ""
echo "RESULT: $pass passed, $fail failed"
[ "$fail" -eq 0 ]

View File

@@ -29,6 +29,13 @@ INSTALL="$FW/install.sh"
pass=0; fail=0
chk() { if eval "$2"; then echo "$1"; pass=$((pass + 1)); else echo "$1"; fail=$((fail + 1)); fi; }
# Redirect the #791 PR2 durable pre-update snapshot ($XDG_STATE_HOME/mosaic/backups)
# into a throwaway so a keep-mode upgrade under test never writes into the real
# ~/.local/state. This test asserts operator-surface fidelity, not backup content.
export XDG_STATE_HOME
XDG_STATE_HOME="$(mktemp -d)"
trap 'rm -rf "$XDG_STATE_HOME"' EXIT
SECRET='SUPER-SECRET-TOKEN-do-not-log-3f9a'
# Seed a throwaway MOSAIC_HOME with an operator sentinel per ownership class.
@@ -206,7 +213,7 @@ fi
# does not silently depend on rsync being installed. (Provide the coreutils the
# installer needs on the stripped PATH.)
FBIN=$(mktemp -d)
for t in bash cp find mktemp rm mkdir chmod cmp sed grep cat dirname basename stat sha256sum awk tr; do
for t in bash cp find mktemp rm mkdir chmod cmp sed grep cat dirname basename stat sha256sum awk tr date sort; do
p=$(command -v "$t" 2>/dev/null) && ln -s "$p" "$FBIN/$t"
done
run_matrix "rsync-absent" env "PATH=$FBIN"