diff --git a/.woodpecker/ci.yml b/.woodpecker/ci.yml index 1789aa8..c40d151 100644 --- a/.woodpecker/ci.yml +++ b/.woodpecker/ci.yml @@ -42,6 +42,18 @@ steps: - bash packages/mosaic/framework/tools/quality/scripts/check-resident-budget.sh --self-test - bash packages/mosaic/framework/tools/quality/scripts/check-resident-budget.sh + # Blocking gate (#791): a framework upgrade must never write or delete an + # operator-owned path. The HARD GATE proves an unanticipated operator sentinel + # survives a keep-mode reseed byte-identical (both rsync + cp paths); the + # migration matrix pins the v2→v3 contract-file semantics. Pure bash, no + # node_modules — runs early alongside sanitization. + upgrade-guard: + image: *node_image + commands: + - apk add --no-cache bash rsync + - bash packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh + - bash packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh + typecheck: image: *node_image commands: @@ -50,6 +62,7 @@ steps: depends_on: - install - sanitization + - upgrade-guard # lint, format, and test are independent — run in parallel after typecheck lint: diff --git a/docs/design/791-upgrade-config-protection.md b/docs/design/791-upgrade-config-protection.md index 85be1aa..2f1be6c 100644 --- a/docs/design/791-upgrade-config-protection.md +++ b/docs/design/791-upgrade-config-protection.md @@ -33,7 +33,7 @@ rsync -a --delete --exclude .git --exclude .framework-version --exclude '*.pre-c - `install.sh:199` — `rsync -a --delete`. **`--delete` prunes every path in `~/.config/mosaic` that is NOT present in the shipped framework source**, unless excluded. - `install.sh:47` — `PRESERVE_PATHS` is the **only** thing standing between `--delete` and operator - data. It is a *denylist of exclusions*: + data. It is a _denylist of exclusions_: ``` PRESERVE_PATHS=("CONSTITUTION.md" "AGENTS.md" "SOUL.md" "USER.md" "TOOLS.md" "STANDARDS.md" "memory" "sources" "credentials" "fleet/roster.yaml" "fleet/roster.json" "fleet/agents" @@ -43,22 +43,22 @@ rsync -a --delete --exclude .git --exclude .framework-version --exclude '*.pre-c `find "$TARGET_DIR" -mindepth 1 -maxdepth 1 ... -exec rm -rf {} +` then re-copies source, restoring only PRESERVE_PATHS globs. -**Root-cause model:** *"Everything under `~/.config/mosaic` is framework-owned and pruneable UNLESS -explicitly preserved."* Any operator path the list forgets is destroyed on the next upgrade. +**Root-cause model:** _"Everything under `~/.config/mosaic` is framework-owned and pruneable UNLESS +explicitly preserved."_ Any operator path the list forgets is destroyed on the next upgrade. ### 1.3 The exact operator paths wiped Cross-referencing the issue's operator-owned list against `PRESERVE_PATHS`: -| Operator path (issue #791) | In PRESERVE_PATHS? | Fate on `mosaic update` | -| ------------------------------------- | ------------------ | ----------------------- | -| `agents/*.conf` (per-agent runtime) | **NO** | **WIPED** | -| `policy/*.md` (operator overlays) | **NO** | **WIPED** | -| `*.local.md` (SOUL/USER/STANDARDS) | **NO** | **WIPED** | -| harvester / SOP artifacts + timers | **NO** | **WIPED** | -| `tools/_lib/credentials.json` | **NO** (`credentials/` dir ≠ this path) | **WIPED** | -| `fleet/agents/*.env` | yes (`fleet/agents`, added by #631) | survives | -| `memory/`, `fleet/roster.*`, `fleet/backlog`, `fleet/roles.local` | yes | survives | +| Operator path (issue #791) | In PRESERVE_PATHS? | Fate on `mosaic update` | +| ----------------------------------------------------------------- | --------------------------------------- | ----------------------- | +| `agents/*.conf` (per-agent runtime) | **NO** | **WIPED** | +| `policy/*.md` (operator overlays) | **NO** | **WIPED** | +| `*.local.md` (SOUL/USER/STANDARDS) | **NO** | **WIPED** | +| harvester / SOP artifacts + timers | **NO** | **WIPED** | +| `tools/_lib/credentials.json` | **NO** (`credentials/` dir ≠ this path) | **WIPED** | +| `fleet/agents/*.env` | yes (`fleet/agents`, added by #631) | survives | +| `memory/`, `fleet/roster.*`, `fleet/backlog`, `fleet/roles.local` | yes | survives | The `fleet/agents`, `memory`, `fleet/backlog` entries were **retro-added after prior incidents** (#631). This whack-a-mole is the structural signature of a denylist. @@ -75,10 +75,11 @@ source over target and skips preserved paths, but **never deletes** target paths (`file-ops.ts:77-109`). It is used by the wizard/init flow, not `mosaic update`. Two problems remain: + 1. Its `preservePaths` (`file-adapter.ts:164-185`) has **already diverged** from `install.sh` — it is **missing `fleet/backlog` and `fleet/roles.local`**. Two hand-maintained denylists, drifted. This is direct evidence for a single shared SSOT manifest. -2. Even non-destructive, it will happily *overwrite* an operator file that collides with a +2. Even non-destructive, it will happily _overwrite_ an operator file that collides with a framework-shipped path unless that path is on its (incomplete) preserve list. ### 1.5 Existing snapshot is inadequate for rollback @@ -94,7 +95,7 @@ Two problems remain: ### 2.1 Ownership model (invert to allow-list) -Replace *"framework-owned unless preserved"* with *"operator-owned unless framework-owned"*, resolved +Replace _"framework-owned unless preserved"_ with _"operator-owned unless framework-owned"_, resolved **per target path** with operator carve-outs winning inside shared framework subtrees. Two declared lists, one SSOT data file shipped in the framework @@ -119,7 +120,7 @@ Two declared lists, one SSOT data file shipped in the framework 3. else (matches neither) → **UNKNOWN ⇒ operator-owned by default (fail-safe)**: never delete. Rule 3 is the actual root-cause fix: an operator path the manifest authors forget is still protected, -because *unknown defaults to operator*. A denylist can never provide this guarantee. +because _unknown defaults to operator_. A denylist can never provide this guarantee. ### 2.3 Sync mechanism change (the mechanically-critical part) @@ -154,7 +155,7 @@ Filesystem-observation test in the existing `test-install-migration.sh` harness `unknown-operator-dir` surviving proves the fail-safe default — a denylist could not pass this case. 5. **Positive controls:** framework files WERE updated; a retired framework file WAS pruned. 6. **Property test** (TS prune-planner): for fuzzed operator paths, `deleteSet ⊆ {matches framework ∧ - in target ∧ not in source}` and `deleteSet ∩ operatorReserved = ∅`. +in target ∧ not in source}` and `deleteSet ∩ operatorReserved = ∅`. --- @@ -192,6 +193,7 @@ The SSOT for those `.env` files is the roster. The reconciler **already** separa **`mosaic fleet regen`** is therefore a **thin recovery-framed wrapper over the existing projection phase** — it does NOT reimplement fleet logic and does NOT preempt in-flight FCM cards (M4/M5): + - Regenerates derivable config (per-agent `*.env.generated`, unit files) from roster SSOT. - **Preview-first:** dry-run default; `--write` to apply. Idempotent. - **Never restarts agents** (the recovery order forbids restart-before-verify). @@ -244,7 +246,7 @@ For a currently-wiped fleet EnvironmentFile state — **do NOT service-restart w 1. **Regenerate:** `mosaic fleet regen --write` — rebuild `~/.config/mosaic/fleet/agents/*.env` from roster SSOT. -2. **Verify each unit resolves to the intended runtime/workdir** *before* any restart: +2. **Verify each unit resolves to the intended runtime/workdir** _before_ any restart: `systemctl --user show mosaic-agent@ -p EnvironmentFile` and confirm the generated env exists and carries the intended `MOSAIC_AGENT_*` runtime/workdir values. 3. **Only then** `systemctl --user restart mosaic-agent@`, one unit at a time. @@ -256,11 +258,11 @@ step 1. ## 8. Proposed PR split (reviewable; DAG-ordered) -| PR | Scope | Depends | Review focus | -| --- | ------------------------------------------------------------------------------------------------------- | ------- | ----------------------- | -| PR1 | **PRIMARY** — shared `framework-manifest.json` + ownership resolver + non-deleting sync + scoped prune (bash + TS) + **HARD GATE** + prune-planner tests | — | correctness (root fix) | -| PR2 | **Safety net** — pre-update snapshot (`~/.local/state`, 0700/0600, retention) + post-sync verify/restore + `mosaic restore` | PR1 | **secrev** (backup/secret) | -| PR3 | **Recovery** — `mosaic fleet regen` (projection-only, preview-first, no restart) + docs (upgrade-safety + recovery runbook) | PR1 | correctness + docs | +| PR | Scope | Depends | Review focus | +| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------------------------- | +| PR1 | **PRIMARY** — shared `framework-manifest.json` + ownership resolver + non-deleting sync + scoped prune (bash + TS) + **HARD GATE** + prune-planner tests | — | correctness (root fix) | +| PR2 | **Safety net** — pre-update snapshot (`~/.local/state`, 0700/0600, retention) + post-sync verify/restore + `mosaic restore` | PR1 | **secrev** (backup/secret) | +| PR3 | **Recovery** — `mosaic fleet regen` (projection-only, preview-first, no restart) + docs (upgrade-safety + recovery runbook) | PR1 | correctness + docs | Rationale: PR1 closes the failure class on its own; if PR2/PR3 slip, the class stays fixed. Each PR is one reviewable unit with its own tests ≥85%. Independent review (author≠reviewer) on all; **secrev** on diff --git a/docs/scratchpads/791-upgrade-config-protection.md b/docs/scratchpads/791-upgrade-config-protection.md index f7a82d9..1196d9a 100644 --- a/docs/scratchpads/791-upgrade-config-protection.md +++ b/docs/scratchpads/791-upgrade-config-protection.md @@ -42,5 +42,48 @@ Design-first: write design doc, send to MS-LEAD, WAIT for confirmation before im regen+docs. PR2/PR3 depend on PR1. ### Status -Design doc written: `docs/design/791-upgrade-config-protection.md`. Sent to MS-LEAD. **WAITING for -confirmation before Phase 2 implementation.** No impl started. +Design doc written: `docs/design/791-upgrade-config-protection.md`. Sent to MS-LEAD. + +## Session 1 (cont.) — MS-LEAD CONFIRMED → Phase 2 GO +All 4 asks approved. Binding conditions: +- TDD tests-first, red-first proof per PR; ≥85% new-code; co-located `*.spec.ts`; never `--no-verify`. +- HARD GATE test (§2.4, unanticipated sentinel survives byte-identical + mtime unchanged) = MERGE-BLOCKING for PR1. +- Manifest-completeness test (§6.2) required. +- Bash+TS read ONE shared `framework-manifest.json`; parity test (§6.1) required (closes #631 drift class). +- UNKNOWN⇒operator (rule 3) non-negotiable. Keep prune-planner PURE. +- `fleet regen`: NEVER restart; dry-run default, `--write` to apply; "never issues restart" test mandatory. +- Independent review every PR; PR2 dedicated secrev. +- One PR at a time through DAG. Report PR1 exact head + red→green evidence for review commission. + +### Now: implementing PR1 (manifest + resolver + non-deleting sync + scoped prune + guard tests). + +## Session 2 (2026-07-16) — PR1 built, tests-first, red→green proven + +Deviation noted to MS-LEAD in PR: manifest is `framework-manifest.txt` (line-oriented), NOT `.json`. +Rationale: keep the bash installer free of a python3/jq dependency. The "ONE shared file, parity- +tested" requirement is honored — `manifest-parity.spec.ts` drives the bash resolver as a subprocess +and asserts byte-identical ownership vs the TS resolver over 34 probe paths spanning every class. + +### PR1 artifacts +- SSOT: `packages/mosaic/framework/framework-manifest.txt` ([framework]/[operator], deny-wins, fail-safe). +- TS resolver: `src/framework/manifest.ts` (pure: parse/matchGlob/resolveOwnership/frameworkSubtreeRoots/ + planPrune) + `manifest.spec.ts` (18 tests incl. planPrune property test + §6.2 completeness). +- Bash resolver: `framework/tools/_lib/manifest.sh` (compiled globs → fork-free `manifest_is_framework`; + CLI `resolve|subtree-roots|classify`). Sourced by install.sh. +- HARD GATE (§2.4): `framework/tools/quality/scripts/test-upgrade-manifest-guard.sh` — keep-mode reseed, + 10 operator sentinels (incl. unanticipated `unknown-operator-dir/x`, `harvester/sop.md`, + `fleet/my-fleet.yaml`) survive byte-identical + mtime-unchanged; retired framework file pruned; + secret value absent from output. RED=31 fail (orig install.sh) → GREEN=48 pass (fixed). +- install.sh: keep mode now manifest-driven (`sync_framework_keep`, no `--delete`); overwrite unchanged. + PRESERVE_PATHS denylist deleted. +- TS sync: `file-ops.syncDirectory` gains `isOperatorOwned` guard; `file-adapter.syncFramework` derives + it from `loadManifest` — hardcoded (drifted) preservePaths deleted. Fixture uses the REAL manifest. +- Parity: `manifest-parity.spec.ts` (§6.1) — bash↔TS agree on 34 paths + subtree roots. +- Migration matrix `test-install-migration.sh`: F6 flipped — `my-fleet.yaml` now MUST survive (fail-safe). +- CI: new merge-blocking `upgrade-guard` step (`.woodpecker/ci.yml`) runs both bash suites (adds rsync). +- update-checker.ts reseed comment corrected to the manifest model. + +### Gates (all green) +- `pnpm typecheck` ✓ · `pnpm lint` ✓ · `pnpm format:check` ✓ +- Full mosaic vitest: 1062 passed (cli-smoke needs `pnpm build` first — build-artifact dep, not this change). +- HARD GATE 48/48 · migration 21/21 · parity 3/3 · manifest 18/18 · file-adapter 8/8. diff --git a/packages/mosaic/framework/framework-manifest.txt b/packages/mosaic/framework/framework-manifest.txt new file mode 100644 index 0000000..7ffe1a5 --- /dev/null +++ b/packages/mosaic/framework/framework-manifest.txt @@ -0,0 +1,80 @@ +# Mosaic framework path-ownership manifest — SSOT for the updater. +# +# This single file is the source of truth consumed by BOTH the bash installer +# (packages/mosaic/framework/install.sh) and the TypeScript config adapter +# (packages/mosaic/src/config/file-adapter.ts). A parity test asserts both +# paths resolve the same ownership from this file, so the two can never drift +# (the failure mode that #631 patched by hand in two places). +# +# Format: one glob per line, relative to the mosaic home (~/.config/mosaic). +# - Lines starting with '#' and blank lines are ignored. +# - '[framework]' / '[operator]' switch the active section. +# - '**' matches any depth; '*' matches within a single path segment. +# +# Ownership resolution for a path P (deny-wins / fail-safe): +# 1. P matches an [operator] glob -> operator-owned. +# 2. else P matches a [framework] glob -> framework-owned. +# 3. else (matches neither) -> OPERATOR-OWNED BY DEFAULT. +# +# Rule 3 is the root-cause fix for #791: a path the manifest authors never +# anticipated is protected because UNKNOWN defaults to operator. The updater +# may only ever create/overwrite framework-owned paths, and may only prune a +# framework-owned path that lives inside a shipped framework subtree and is +# absent from the current framework source (a genuinely retired file). +# Operator-owned and unknown paths are structurally unreachable by pruning. + +[framework] +# Top-level framework contract files (also reconciled from defaults/ on upgrade). +CONSTITUTION.md +AGENTS.md +STANDARDS.md +# Shipped framework subtrees — pruning is scoped to these roots. +adapters/** +constitution/** +CONTRIBUTING.md +defaults/** +examples/** +guides/** +install.sh +install.ps1 +LICENSE +profiles/** +runtime/** +systemd/** +templates/** +tools/** +# Fleet: only the framework-seeded fleet subtrees are framework-owned. +fleet/README.md +fleet/examples/** +fleet/profiles/** +fleet/roles/** +fleet/roster.schema.json +fleet/services/** +# The manifest itself is framework-owned. +framework-manifest.txt + +[operator] +# Identity / user-seeded contract files — generated by the wizard or seeded +# once from defaults/, then owned by the operator. Never overwritten on upgrade. +SOUL.md +USER.md +TOOLS.md +# Local overlays (tighten-only) authored by the operator. +*.local.md +# Operator-owned trees the updater must never write over or prune. +agents/** +policy/** +memory/** +sources/** +credentials/** +# Secret-bearing operator file INSIDE the framework-owned tools/ subtree. +# Listed explicitly so the deny-wins rule carves it out of tools/**. +tools/_lib/credentials.json +# Operator-owned fleet state (roster SSOT, per-agent env, heartbeats, backlog, +# persona overrides). Losing these silently downgrades a running fleet (#791). +fleet/roster.yaml +fleet/roster.json +fleet/agents/** +fleet/run/** +fleet/backlog/** +fleet/roles.local/** diff --git a/packages/mosaic/framework/install.sh b/packages/mosaic/framework/install.sh index 4e437f4..6275f7f 100755 --- a/packages/mosaic/framework/install.sh +++ b/packages/mosaic/framework/install.sh @@ -19,32 +19,19 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TARGET_DIR="${MOSAIC_HOME:-$HOME/.config/mosaic}" INSTALL_MODE="${MOSAIC_INSTALL_MODE:-prompt}" -# Files/dirs protected from rsync --delete during sync. NOTE: framework-owned -# entries (CONSTITUTION/AGENTS/STANDARDS) ARE re-applied afterward by -# reconcile_framework_files (overwrite + backup-once); the rest stay user-owned. -# User-created content in these paths survives rsync --delete. -# -# fleet/* — the framework SEEDS fleet/examples, fleet/roles, fleet/profiles, and -# fleet/roster.schema.json (synced normally — every fleet/roles/*.md role contract -# and fleet/profiles/*.yaml system-type profile lands automatically via this sync, -# so no per-file entry is needed; exact preserved roster paths are anchored to -# the top level only and do NOT shadow fleet/profiles/*.yaml). The user's -# own fleet files MUST -# survive `mosaic update` (which runs this sync automatically): the active -# rosters (`fleet/roster.yaml` and `fleet/roster.json`), per-agent env -# (`fleet/agents/`), heartbeat run dir (`fleet/run/`), and the Mosaic-native -# backlog-of-record store (`fleet/backlog/` — embedded PGlite data dir; see -# packages/mosaic/src/commands/fleet-backlog.ts). Without these, an update -# wipes the operator's fleet AND their backlog. Glob entries are honored by -# both the rsync path (`--exclude`) and the glob-aware cp fallback below. -# -# fleet/roles.local — the persona OVERRIDE layer (H4). Baseline personas in -# fleet/roles/ are reseeded normally on every update (delivering new baseline -# personas), so any local edit there would be clobbered. User customizations -# and user-ADDED personas instead live in fleet/roles.local/ and MUST survive -# `mosaic update` — they win over the baseline on merge (AC-NS-7; see -# packages/mosaic/src/commands/fleet-personas.ts). -PRESERVE_PATHS=("CONSTITUTION.md" "AGENTS.md" "SOUL.md" "USER.md" "TOOLS.md" "STANDARDS.md" "memory" "sources" "credentials" "fleet/roster.yaml" "fleet/roster.json" "fleet/agents" "fleet/run" "fleet/backlog" "fleet/roles.local") +# Shared framework path-ownership manifest reader (#791). Parity with +# packages/mosaic/src/framework/manifest.ts — both consume framework-manifest.txt. +# Sourcing does not run its CLI dispatch (guarded by BASH_SOURCE==$0). +# shellcheck source=tools/_lib/manifest.sh +source "$SOURCE_DIR/tools/_lib/manifest.sh" + +# Which paths a keep-mode upgrade may touch is no longer a hand-maintained +# denylist. It is derived from the shared framework-manifest.txt (#791): the +# updater only ever creates/overwrites framework-owned paths and only prunes a +# retired framework file inside a shipped framework subtree. Everything else — +# every operator file, and every path the manifest never anticipated — is +# operator-owned by default (fail-safe) and is never written or deleted. See +# sync_framework_keep() below and packages/mosaic/src/framework/manifest.ts. # Framework-owned contract files: re-copied from defaults/ on every upgrade (the # user must not edit them; a divergent copy is backed up once before overwrite). @@ -184,63 +171,72 @@ sync_framework() { return fi - if command -v rsync >/dev/null 2>&1; then - local rsync_args=(-a --delete --exclude ".git" --exclude ".framework-version" --exclude "*.pre-constitution.bak") - - if [[ "$INSTALL_MODE" == "keep" ]]; then - # Anchor to the transfer root (leading /) so we preserve the TOP-LEVEL - # ~/.config/mosaic/ without also excluding defaults/ from sync - # (reconcile_framework_files needs the freshly-synced defaults/ copies). - for path in "${PRESERVE_PATHS[@]}"; do - rsync_args+=(--exclude "/$path") - done - fi - - rsync "${rsync_args[@]}" "$SOURCE_DIR/" "$TARGET_DIR/" + if [[ "$INSTALL_MODE" == "keep" ]]; then + # The `mosaic update` path. Manifest-driven, never-deleting-outside-framework: + # operator config is structurally protected (#791). No rsync --delete here. + manifest_load + sync_framework_keep return fi - # Fallback: cp-based sync. Exact top-level preserved paths mirror the - # root-anchored rsync excludes above. - local preserve_tmp="" - if [[ "$INSTALL_MODE" == "keep" ]]; then - preserve_tmp="$(mktemp -d "${TMPDIR:-/tmp}/mosaic-preserve-XXXXXX")" - local match rel - for path in "${PRESERVE_PATHS[@]}"; do - # Unquoted $path lets the glob expand against TARGET_DIR; nullglob makes a - # non-matching pattern vanish instead of staying literal. - shopt -s nullglob - for match in "$TARGET_DIR/"$path; do - [[ -e "$match" ]] || continue - rel="${match#"$TARGET_DIR/"}" - mkdir -p "$preserve_tmp/$(dirname "$rel")" - cp -R "$match" "$preserve_tmp/$rel" - done - shopt -u nullglob - done - fi + # overwrite mode — a full replace, chosen only for a fresh install or when the + # operator explicitly asks to replace everything. No operator state to protect. + sync_framework_overwrite +} - find "$TARGET_DIR" -mindepth 1 -maxdepth 1 ! -name ".git" ! -name ".framework-version" ! -name "*.pre-constitution.bak" -exec rm -rf {} + +# Keep-mode sync: create/refresh framework-owned files and prune only retired +# framework files inside shipped framework subtrees. Operator-owned and unknown +# paths (fail-safe default) are never written and never deleted — the #791 HARD +# GATE. Single code path (no rsync) so it is byte-for-byte parity-testable. +sync_framework_keep() { + local src="$SOURCE_DIR" dst="$TARGET_DIR" abs rel root + + # 1) Overlay copy — every framework-owned source file, refreshed only when its + # bytes changed (no mtime churn on unchanged files, never on operator files). + while IFS= read -r -d '' abs; do + rel="${abs#"$src"/}" + case "$rel" in + .git|.git/*|.framework-version|*.pre-constitution.bak) continue ;; + esac + manifest_is_framework "$rel" || continue + if [[ -f "$dst/$rel" ]] && cmp -s "$abs" "$dst/$rel"; then continue; fi + [[ "$rel" == */* ]] && mkdir -p "$dst/${rel%/*}" + cp "$abs" "$dst/$rel" + done < <(find "$src" -type f -print0) + + # 2) Scoped prune — within each shipped framework subtree root, remove + # framework-owned target files the current source no longer ships. Operator + # carve-outs (e.g. tools/_lib/credentials.json) resolve to operator and are + # skipped; unknown paths resolve to operator too — both are unreachable here. + while IFS= read -r root; do + [[ -n "$root" && -d "$dst/$root" ]] || continue + while IFS= read -r -d '' abs; do + rel="${abs#"$dst"/}" + case "$rel" in *.pre-constitution.bak) continue ;; esac + [[ -f "$src/$rel" ]] && continue # still shipped + manifest_is_framework "$rel" || continue + rm -f "$abs" + done < <(find "$dst/$root" -type f -print0) + # Drop framework dirs left empty by the prune (never touches a dir that still + # holds an operator file — those are never emptied). + find "$dst/$root" -type d -empty -delete 2>/dev/null || true + done < <(manifest_subtree_roots) +} + +# Overwrite-mode sync: full replace. Only reached for a fresh install or an +# explicit operator "replace everything" choice, so nothing is preserved. +sync_framework_overwrite() { + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete \ + --exclude ".git" --exclude ".framework-version" --exclude "*.pre-constitution.bak" \ + "$SOURCE_DIR/" "$TARGET_DIR/" + return + fi + find "$TARGET_DIR" -mindepth 1 -maxdepth 1 \ + ! -name ".git" ! -name ".framework-version" ! -name "*.pre-constitution.bak" \ + -exec rm -rf {} + cp -R "$SOURCE_DIR"/. "$TARGET_DIR"/ rm -rf "$TARGET_DIR/.git" - - if [[ -n "$preserve_tmp" ]]; then - # Restore by re-globbing the SAME patterns against preserve_tmp, so each - # preserved item is restored at its own relative path (e.g. only - # fleet/roster.yaml is replaced — the freshly-synced fleet/examples stays). - for path in "${PRESERVE_PATHS[@]}"; do - shopt -s nullglob - for match in "$preserve_tmp/"$path; do - [[ -e "$match" ]] || continue - rel="${match#"$preserve_tmp/"}" - rm -rf "$TARGET_DIR/$rel" - mkdir -p "$TARGET_DIR/$(dirname "$rel")" - cp -R "$match" "$TARGET_DIR/$rel" - done - shopt -u nullglob - done - rm -rf "$preserve_tmp" - fi } # ═══════════════════════════════════════════════════════════════════════════════ diff --git a/packages/mosaic/framework/tools/_lib/manifest.sh b/packages/mosaic/framework/tools/_lib/manifest.sh new file mode 100644 index 0000000..874fbb2 --- /dev/null +++ b/packages/mosaic/framework/tools/_lib/manifest.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# Shared bash reader for framework-manifest.txt (#791). +# +# This is the bash half of the SSOT ownership resolver; the TypeScript half is +# packages/mosaic/src/framework/manifest.ts. BOTH read the same +# framework-manifest.txt and MUST resolve identical ownership for any path — the +# parity test (manifest-parity.spec.ts) invokes this file's `resolve` CLI and +# compares it against the TS resolver, so the two can never drift (the #631 +# two-copies failure class this closes). +# +# Ownership resolution (deny-wins / fail-safe): +# 1. operator glob matches -> operator +# 2. else framework glob -> framework +# 3. else -> operator (UNKNOWN defaults to operator, #791) +# +# Globs are compiled once at load into exact-prefix checks or POSIX EREs, so the +# hot resolver (manifest_is_framework) forks no subprocesses — the installer +# calls it once per file across the whole tree. +# +# Usage as a library (source it, then): +# manifest_load [manifest-file] # populates + compiles the manifest +# manifest_is_framework # rc 0 = framework-owned, rc 1 = operator +# manifest_resolve # echoes: framework | operator +# manifest_subtree_roots # echoes shipped framework `dir/**` roots +# +# Usage as a CLI (parity harness): +# bash manifest.sh resolve +# bash manifest.sh subtree-roots +# bash manifest.sh classify # reads paths on stdin -> "\t" + +MANIFEST_FRAMEWORK=() +MANIFEST_OPERATOR=() + +# Compiled forms (parallel arrays). _*_KIND[i] is "exact" or "re". +_MF_KIND=(); _MF_EXACT=(); _MF_RE=() +_MO_KIND=(); _MO_EXACT=(); _MO_RE=() +_MF_ROOTS=() + +_manifest_default_root() { cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd; } + +# Normalize a path/glob: backslashes -> slashes, strip leading ./ and /, strip +# trailing / (mirrors normalizeRel in manifest.ts). +_manifest_norm() { + local p="$1" + p="${p//\\//}" + p="${p#./}" + while [[ "$p" == /* ]]; do p="${p#/}"; done + while [[ "$p" == */ ]]; do p="${p%/}"; done + printf '%s' "$p" +} + +# Translate a normalized glob into a POSIX ERE body (mirrors globToRegExpBody). +_manifest_glob_to_ere() { + local pattern; pattern="$(_manifest_norm "$1")" + local out="" c n i len=${#pattern} trailing + for (( i = 0; i < len; i++ )); do + c="${pattern:i:1}" + if [[ "$c" == "*" ]]; then + n="${pattern:i+1:1}" + if [[ "$n" == "*" ]]; then + i=$((i + 1)) + trailing=0 + if [[ "${pattern:i+1:1}" == "/" ]]; then i=$((i + 1)); trailing=1; fi + if [[ "$out" == */ ]]; then + out="${out%/}(/.*)?" + elif [[ "$trailing" -eq 1 ]]; then + out="$out(.*/)?" + else + out="$out.*" + fi + else + out="$out[^/]*" + fi + else + case "$c" in + .|+|\?|^|\$|\{|\}|\(|\)|\||\[|\]|\\) out="$out\\$c" ;; + *) out="$out$c" ;; + esac + fi + done + printf '%s' "$out" +} + +# Compile one raw glob into (kind, exact, re) appended to the given section. +# $1 = raw glob, $2 = section letter (F|O). +_manifest_compile_one() { + local norm; norm="$(_manifest_norm "$1")" + [[ -n "$norm" ]] || return 0 + if [[ "$norm" == *"*"* ]]; then + local re="^$(_manifest_glob_to_ere "$norm")\$" + if [[ "$2" == F ]]; then + _MF_KIND+=(re); _MF_EXACT+=(""); _MF_RE+=("$re") + else + _MO_KIND+=(re); _MO_EXACT+=(""); _MO_RE+=("$re") + fi + else + if [[ "$2" == F ]]; then + _MF_KIND+=(exact); _MF_EXACT+=("$norm"); _MF_RE+=("") + else + _MO_KIND+=(exact); _MO_EXACT+=("$norm"); _MO_RE+=("") + fi + fi + [[ "$2" == F && "$norm" == */"**" ]] && _MF_ROOTS+=("${norm%/**}") + return 0 +} + +_manifest_compile() { + _MF_KIND=(); _MF_EXACT=(); _MF_RE=(); _MF_ROOTS=() + _MO_KIND=(); _MO_EXACT=(); _MO_RE=() + local g + for g in "${MANIFEST_FRAMEWORK[@]:-}"; do [[ -n "$g" ]] && _manifest_compile_one "$g" F; done + for g in "${MANIFEST_OPERATOR[@]:-}"; do [[ -n "$g" ]] && _manifest_compile_one "$g" O; done +} + +# Load + compile the manifest. Rejects a malformed file the same way +# parseManifest() does (entry before a section header / unknown header). +manifest_load() { + local file="${1:-}" + [[ -n "$file" ]] || file="$(_manifest_default_root)/framework-manifest.txt" + MANIFEST_FRAMEWORK=() + MANIFEST_OPERATOR=() + local section="" line + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line#"${line%%[![:space:]]*}"}" # ltrim + line="${line%"${line##*[![:space:]]}"}" # rtrim + [[ -z "$line" || "${line:0:1}" == "#" ]] && continue + case "$line" in + "[framework]") section=framework; continue ;; + "[operator]") section=operator; continue ;; + "["*) echo "manifest: unknown section header: $line" >&2; return 1 ;; + esac + if [[ -z "$section" ]]; then + echo "manifest: entry before any [section] header: $line" >&2 + return 1 + fi + if [[ "$section" == framework ]]; then + MANIFEST_FRAMEWORK+=("$line") + else + MANIFEST_OPERATOR+=("$line") + fi + done < "$file" + _manifest_compile +} + +# Fork-free: does $1 (a mosaic-home-relative path) match an operator glob? +_mo_matches() { + local path="$1" i n=${#_MO_KIND[@]} re pat + for (( i = 0; i < n; i++ )); do + if [[ "${_MO_KIND[i]}" == exact ]]; then + pat="${_MO_EXACT[i]}" + [[ "$path" == "$pat" || "$path" == "$pat/"* ]] && return 0 + else + re="${_MO_RE[i]}" + [[ "$path" =~ $re ]] && return 0 + fi + done + return 1 +} + +# Fork-free: does $1 match a framework glob? +_mf_matches() { + local path="$1" i n=${#_MF_KIND[@]} re pat + for (( i = 0; i < n; i++ )); do + if [[ "${_MF_KIND[i]}" == exact ]]; then + pat="${_MF_EXACT[i]}" + [[ "$path" == "$pat" || "$path" == "$pat/"* ]] && return 0 + else + re="${_MF_RE[i]}" + [[ "$path" =~ $re ]] && return 0 + fi + done + return 1 +} + +# The installer hot path — no subshell. rc 0 = framework-owned, rc 1 = operator +# (deny-wins / fail-safe). Assumes an already-clean POSIX relative path. +manifest_is_framework() { + _mo_matches "$1" && return 1 + _mf_matches "$1" && return 0 + return 1 +} + +# Echo the ownership of a path: framework | operator. Normalizes first, so it is +# safe for CLI / test callers passing unnormalized input. +manifest_resolve() { + local path; path="$(_manifest_norm "$1")" + if manifest_is_framework "$path"; then echo framework; else echo operator; fi +} + +# Echo each shipped framework subtree root (a `dir/**` entry, without the /**). +manifest_subtree_roots() { + local r + for r in "${_MF_ROOTS[@]:-}"; do [[ -n "$r" ]] && printf '%s\n' "$r"; done +} + +# CLI dispatch — only when executed directly, never when sourced. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + set -o pipefail + manifest_load "${MANIFEST_FILE:-}" + cmd="${1:-}" + case "$cmd" in + resolve) manifest_resolve "${2:?path required}" ;; + subtree-roots) manifest_subtree_roots ;; + classify) + while IFS= read -r p; do + [[ -z "$p" ]] && continue + printf '%s\t%s\n' "$(manifest_resolve "$p")" "$p" + done + ;; + *) + echo "usage: manifest.sh {resolve |subtree-roots|classify}" >&2 + exit 2 + ;; + esac +fi diff --git a/packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh b/packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh index ea9aa6c..c925f4d 100755 --- a/packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh +++ b/packages/mosaic/framework/tools/quality/scripts/test-install-migration.sh @@ -61,8 +61,10 @@ MOSAIC_HOME="$T5" MOSAIC_INSTALL_MODE=bogus MOSAIC_SYNC_ONLY=1 bash "$INSTALL" > chk "F5 failure: invalid mode rejected (nonzero exit)" "[ $rc -ne 0 ]" chk "F5 failure: SOUL + credentials intact" "grep -q orig '$T5/SOUL.md' && grep -q keepme '$T5/credentials/c.json'" -# F6 — keep-mode re-seed (the `mosaic update` path) MUST preserve only the -# exact user-owned roster paths while refreshing framework-owned schema/examples. +# F6 — keep-mode re-seed (the `mosaic update` path) MUST preserve ALL user-owned +# fleet state — including an unanticipated file the manifest never names, which +# resolves to operator-owned by the #791 fail-safe — while refreshing the +# framework-owned schema/examples. T6=$(mktemp -d); mkdir -p "$T6/fleet/examples" "$T6/fleet/run" "$T6/fleet/agents" printf '# persona\n' > "$T6/SOUL.md" # makes it a recognized existing install (→ keep mode) printf 'version: 1\nagents:\n - name: coder0\n' > "$T6/fleet/roster.yaml" @@ -75,13 +77,14 @@ printf '{"stale":true}\n' > "$T6/fleet/roster.schema.json" E6=$(mktemp -d) cp "$T6/fleet/roster.yaml" "$E6/roster-yaml.expected" cp "$T6/fleet/roster.json" "$E6/roster-json.expected" +cp "$T6/fleet/my-fleet.yaml" "$E6/my-fleet.expected" cp "$T6/fleet/run/coder0.hb" "$E6/run.expected" cp "$T6/fleet/agents/coder0.env" "$E6/agent.expected" echo 3 > "$T6/.framework-version" run "$T6" keep chk "F6 reseed: exact roster.yaml bytes survive keep-mode sync" "cmp -s '$T6/fleet/roster.yaml' '$E6/roster-yaml.expected'" chk "F6 reseed: exact roster.json bytes survive keep-mode sync" "cmp -s '$T6/fleet/roster.json' '$E6/roster-json.expected'" -chk "F6 reseed: unrelated fleet YAML is not preserved" "[ ! -f '$T6/fleet/my-fleet.yaml' ]" +chk "F6 reseed: unanticipated operator fleet file survives (fail-safe, #791)" "cmp -s '$T6/fleet/my-fleet.yaml' '$E6/my-fleet.expected'" chk "F6 reseed: per-agent env bytes survive" "cmp -s '$T6/fleet/agents/coder0.env' '$E6/agent.expected'" chk "F6 reseed: heartbeat bytes survive" "cmp -s '$T6/fleet/run/coder0.hb' '$E6/run.expected'" chk "F6 reseed: framework examples are refreshed" "grep -q orchestrator '$T6/fleet/examples/general.yaml'" diff --git a/packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh b/packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh new file mode 100644 index 0000000..a27c1aa --- /dev/null +++ b/packages/mosaic/framework/tools/quality/scripts/test-upgrade-manifest-guard.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# test-upgrade-manifest-guard.sh — the #791 HARD GATE. +# +# Proves that a keep-mode framework upgrade (the `mosaic update` path: +# install.sh with MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1) touches NO path +# outside the framework-owned manifest. Every operator-owned sentinel — including +# a deliberately UNANTICIPATED one the manifest never names — must survive +# byte-identical with an unchanged mtime (not even rewritten). Framework files +# must still update, and a retired framework file inside a shipped subtree must +# still be pruned. No operator secret value may appear in installer output. +# +# Runs the whole matrix twice: once with rsync available, once forcing the +# cp-based fallback (both destructive paths in install.sh must obey the manifest). +# +# Usage: bash test-upgrade-manifest-guard.sh +set -uo pipefail + +FW="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" # packages/mosaic/framework +INSTALL="$FW/install.sh" + +pass=0; fail=0 +chk() { if eval "$2"; then echo " ✓ $1"; pass=$((pass + 1)); else echo " ✗ $1"; fail=$((fail + 1)); fi; } + +SECRET='SUPER-SECRET-TOKEN-do-not-log-3f9a' + +# Seed a throwaway MOSAIC_HOME with an operator sentinel per ownership class. +seed_home() { + local H="$1" + mkdir -p "$H/agents" "$H/policy" "$H/memory" "$H/tools/_lib" \ + "$H/fleet/agents" "$H/harvester" "$H/unknown-operator-dir" "$H/guides" + printf '# persona\n' > "$H/SOUL.md" # marks a recognized existing install → keep mode + printf 'MODEL=opus\n' > "$H/agents/coder0.conf" + printf '# operator policy\n' > "$H/policy/custom.md" + printf '# soul overlay\n' > "$H/SOUL.local.md" + printf '# operator memory\n' > "$H/memory/note.md" + printf 'TOKEN=%s\n' "$SECRET" > "$H/tools/_lib/credentials.json" + printf 'MOSAIC_AGENT_NAME=coder0\n' > "$H/fleet/agents/coder0.env" + printf 'version: 2\nagents:\n - name: coder0\n' > "$H/fleet/roster.yaml" + printf '# harvester SOP\n' > "$H/harvester/sop.md" + printf 'operator data the manifest never anticipated\n' > "$H/unknown-operator-dir/x" + printf 'version: 1\nagents:\n - name: mine\n' > "$H/fleet/my-fleet.yaml" + # A retired framework file inside a shipped subtree (absent from source) — must be pruned. + printf '# retired guide\n' > "$H/guides/RETIRED-OLD-GUIDE.md" + echo 3 > "$H/.framework-version" +} + +OPERATOR_SENTINELS=( + "agents/coder0.conf" + "policy/custom.md" + "SOUL.local.md" + "memory/note.md" + "tools/_lib/credentials.json" + "fleet/agents/coder0.env" + "fleet/roster.yaml" + "harvester/sop.md" + "unknown-operator-dir/x" + "fleet/my-fleet.yaml" +) + +run_matrix() { + local label="$1"; shift # extra env / PATH override applied to the run + local H E OUT rel before_hash after_hash before_mt after_mt + H=$(mktemp -d); E=$(mktemp -d); OUT=$(mktemp) + seed_home "$H" + + # Snapshot hash + mtime of every operator sentinel before the upgrade. + for rel in "${OPERATOR_SENTINELS[@]}"; do + sha256sum "$H/$rel" | awk '{print $1}' > "$E/$(echo "$rel" | tr / _).hash" + stat -c %Y "$H/$rel" > "$E/$(echo "$rel" | tr / _).mt" + done + + # The upgrade under test (keep + sync-only = the `mosaic update` reseed path). + MOSAIC_HOME="$H" MOSAIC_INSTALL_MODE=keep MOSAIC_SYNC_ONLY=1 "$@" bash "$INSTALL" >"$OUT" 2>&1 + + # HARD GATE: every operator sentinel survives byte-identical AND mtime-unchanged. + for rel in "${OPERATOR_SENTINELS[@]}"; do + before_hash=$(cat "$E/$(echo "$rel" | tr / _).hash") + before_mt=$(cat "$E/$(echo "$rel" | tr / _).mt") + after_hash=$(sha256sum "$H/$rel" 2>/dev/null | awk '{print $1}') + after_mt=$(stat -c %Y "$H/$rel" 2>/dev/null || echo MISSING) + chk "[$label] operator sentinel survives byte-identical: $rel" \ + "[ -n '$after_hash' ] && [ '$before_hash' = '$after_hash' ]" + chk "[$label] operator sentinel not rewritten (mtime unchanged): $rel" \ + "[ '$before_mt' = '$after_mt' ]" + done + + # Positive controls: the framework tree still updates, retired file pruned. + chk "[$label] framework file present after upgrade (guides synced)" \ + "[ -f '$H/guides/E2E-DELIVERY.md' ]" + chk "[$label] retired framework file inside a subtree is pruned" \ + "[ ! -f '$H/guides/RETIRED-OLD-GUIDE.md' ]" + chk "[$label] manifest itself is installed" "[ -f '$H/framework-manifest.txt' ]" + + # Secret-safety: the operator secret value never appears in installer output. + chk "[$label] operator secret value absent from installer stdout/stderr" \ + "! grep -q '$SECRET' '$OUT'" + + rm -rf "$H" "$E" "$OUT" +} + +echo "#791 upgrade manifest guard (HARD GATE):" + +# 1) rsync path (if available on this host). +if command -v rsync >/dev/null 2>&1; then + run_matrix "rsync" +else + echo " · rsync not installed — skipping rsync-path matrix" +fi + +# 2) cp fallback path — hide rsync behind a scratch PATH so install.sh takes the +# find/rm+cp branch. Provide the coreutils the fallback needs. +FBIN=$(mktemp -d) +for t in bash cp find mktemp rm mkdir chmod cmp sed grep cat dirname basename stat sha256sum awk tr; do + p=$(command -v "$t" 2>/dev/null) && ln -s "$p" "$FBIN/$t" +done +run_matrix "cp-fallback" env "PATH=$FBIN" +rm -rf "$FBIN" + +echo +echo "RESULT: $pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/packages/mosaic/src/config/file-adapter.test.ts b/packages/mosaic/src/config/file-adapter.test.ts index 9019436..ae56ff8 100644 --- a/packages/mosaic/src/config/file-adapter.test.ts +++ b/packages/mosaic/src/config/file-adapter.test.ts @@ -1,9 +1,22 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync, existsSync } from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + writeFileSync, + rmSync, + readFileSync, + existsSync, + copyFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { FileConfigAdapter, DEFAULT_SEED_FILES } from './file-adapter.js'; +// The real shipping manifest — the fixture uses it verbatim so these tests +// exercise production ownership resolution, not a synthetic copy (#791). +const REAL_FRAMEWORK_ROOT = fileURLToPath(new URL('../../framework', import.meta.url)); + /** * Regression tests for the `FileConfigAdapter.syncFramework` seed behavior. * @@ -34,6 +47,13 @@ function makeFixture(): { sourceDir: string; mosaicHome: string; defaultsDir: st mkdirSync(defaultsDir, { recursive: true }); mkdirSync(mosaicHome, { recursive: true }); + // #791: syncFramework resolves ownership from the shared manifest under the + // source dir. Seed the real one so keep-mode syncs behave as in production. + copyFileSync( + join(REAL_FRAMEWORK_ROOT, 'framework-manifest.txt'), + join(sourceDir, 'framework-manifest.txt'), + ); + // Framework-contract defaults we expect the wizard to seed. writeFileSync(join(defaultsDir, 'CONSTITUTION.md'), '# CONSTITUTION default\n'); writeFileSync(join(defaultsDir, 'AGENTS.md'), '# AGENTS default\n'); @@ -102,9 +122,10 @@ describe('FileConfigAdapter.syncFramework — defaults seeding', () => { }); it('overwrites framework-owned files (backup-once) but preserves user-seeded files', async () => { - // Plant a root-level AGENTS.md in sourceDir so syncDirectory's preserve is exercised. - writeFileSync(join(fixture.sourceDir, 'AGENTS.md'), '# shipped AGENTS from source root\n'); - + // Contract files (CONSTITUTION/AGENTS/STANDARDS) ship only under defaults/ — + // reconcile_framework_files is their sole writer (backup-once). The bulk + // sync never sees a root-level copy, so a user's edited root file is backed + // up, not silently clobbered, on upgrade. writeFileSync(join(fixture.mosaicHome, 'TOOLS.md'), '# user-customized TOOLS\n'); writeFileSync(join(fixture.mosaicHome, 'AGENTS.md'), '# user-customized AGENTS\n'); diff --git a/packages/mosaic/src/config/file-adapter.ts b/packages/mosaic/src/config/file-adapter.ts index 1480d6a..4bd4f23 100644 --- a/packages/mosaic/src/config/file-adapter.ts +++ b/packages/mosaic/src/config/file-adapter.ts @@ -34,6 +34,7 @@ import { buildToolsTemplateVars, } from '../template/builders.js'; import { atomicWrite, backupFile, syncDirectory } from '../platform/file-ops.js'; +import { loadManifest, resolveOwnership } from '../framework/manifest.js'; /** * Parse a SoulConfig from an existing SOUL.md file. @@ -155,38 +156,22 @@ export class FileConfigAdapter implements ConfigService { } async syncFramework(action: InstallAction): Promise { - // Must match PRESERVE_PATHS in packages/mosaic/framework/install.sh so - // the bash and TS install paths have the same upgrade-preservation - // semantics. Contract files (AGENTS.md, STANDARDS.md, TOOLS.md) are - // seeded from defaults/ on first install and preserved thereafter; - // identity files (SOUL.md, USER.md) are generated by wizard stages and - // must never be touched by the framework sync. - const preservePaths = - action === 'keep' || action === 'reconfigure' - ? [ - 'CONSTITUTION.md', - 'AGENTS.md', - 'SOUL.md', - 'USER.md', - 'TOOLS.md', - 'STANDARDS.md', - 'memory', - 'sources', - 'credentials', - // User-authored fleet data MUST survive `mosaic update`'s re-seed. - // The framework seeds only fleet/examples + fleet/roles + - // fleet/roster.schema.json; the operator's roster, per-agent env, and - // heartbeat run dir stay user-owned. (Mirror of install.sh PRESERVE_PATHS.) - 'fleet/roster.yaml', - 'fleet/roster.json', - 'fleet/agents', - 'fleet/run', - ] - : []; + // #791: ownership is derived from the shared framework manifest + // (packages/mosaic/framework/framework-manifest.txt) — the SAME file the + // bash installer reads — so the TS and bash paths can never drift. On an + // upgrade (keep/reconfigure) the sync must NEVER write an operator-owned + // path: every operator file, and any path the manifest never anticipated + // (which resolves to operator by the fail-safe default), is left untouched. + // A fresh install ('overwrite'/'reconfigure' onto an empty home) seeds the + // full tree, so the guard applies only when preserving an existing home. + const guardOwnership = action === 'keep' || action === 'reconfigure'; + const manifest = guardOwnership ? loadManifest(this.sourceDir) : undefined; syncDirectory(this.sourceDir, this.mosaicHome, { - preserve: preservePaths, excludeGit: true, + isOperatorOwned: manifest + ? (relPath) => resolveOwnership(manifest, relPath) === 'operator' + : undefined, }); // Reconcile framework-contract files from framework/defaults/ into the mosaic diff --git a/packages/mosaic/src/framework/manifest-parity.spec.ts b/packages/mosaic/src/framework/manifest-parity.spec.ts new file mode 100644 index 0000000..89dfd6e --- /dev/null +++ b/packages/mosaic/src/framework/manifest-parity.spec.ts @@ -0,0 +1,108 @@ +import { describe, it, expect } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { loadManifest, resolveOwnership, frameworkSubtreeRoots } from './manifest.js'; + +/** + * Bash ↔ TS parity (#791, §6.1). + * + * The installer (bash) and the config adapter (TS) each resolve path ownership + * from framework-manifest.txt. If the two resolvers disagreed on a single path, + * an upgrade could protect a file on one code path and wipe it on the other — + * exactly the two-copies drift that #631 patched by hand. This test drives the + * bash resolver (`tools/_lib/manifest.sh`) as a subprocess and asserts it agrees + * with the TS resolver for a broad set of paths spanning every ownership class. + */ + +const FRAMEWORK_ROOT = fileURLToPath(new URL('../../framework', import.meta.url)); +const MANIFEST_SH = join(FRAMEWORK_ROOT, 'tools', '_lib', 'manifest.sh'); + +const hasBash = (() => { + try { + execFileSync('bash', ['-c', 'true'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +})(); + +function bashResolve(relPath: string): string { + return execFileSync('bash', [MANIFEST_SH, 'resolve', relPath], { + encoding: 'utf-8', + }).trim(); +} + +function bashSubtreeRoots(): string[] { + return execFileSync('bash', [MANIFEST_SH, 'subtree-roots'], { encoding: 'utf-8' }) + .split('\n') + .map((s) => s.trim()) + .filter((s) => s.length > 0); +} + +// Paths spanning every ownership class: framework single-files, framework +// subtrees, operator declared trees, operator carve-out inside a framework +// subtree, local overlays, and deliberately UNANTICIPATED paths (fail-safe). +const PROBE_PATHS = [ + 'CONSTITUTION.md', + 'AGENTS.md', + 'STANDARDS.md', + 'install.sh', + 'framework-manifest.txt', + 'guides/E2E-DELIVERY.md', + 'tools/git/pr-create.sh', + 'tools/_lib/manifest.sh', + 'defaults/SOUL.md', + 'fleet/README.md', + 'fleet/roles/coder.md', + 'fleet/roster.schema.json', + 'fleet/examples/general.yaml', + // operator + 'SOUL.md', + 'USER.md', + 'TOOLS.md', + 'SOUL.local.md', + 'USER.local.md', + 'STANDARDS.local.md', + 'agents/coder0.conf', + 'policy/custom.md', + 'memory/note.md', + 'sources/skills/x.md', + 'credentials/c.json', + 'tools/_lib/credentials.json', + 'fleet/roster.yaml', + 'fleet/roster.json', + 'fleet/agents/coder0.env', + 'fleet/run/coder0.hb', + 'fleet/backlog/data.db', + 'fleet/roles.local/custom.md', + // unanticipated → operator (fail-safe) + 'harvester/sop.md', + 'unknown-operator-dir/x', + 'fleet/my-fleet.yaml', + 'random-root-file.md', + 'tools/some-new-framework-tool.sh', +]; + +describe.skipIf(!hasBash)('bash ↔ TS manifest parity (§6.1)', () => { + it('the bash resolver CLI exists and is executable', () => { + expect(existsSync(MANIFEST_SH)).toBe(true); + }); + + it('bash and TS resolve identical ownership for every probe path', () => { + const manifest = loadManifest(FRAMEWORK_ROOT); + const disagreements: Array<{ path: string; ts: string; bash: string }> = []; + for (const p of PROBE_PATHS) { + const ts = resolveOwnership(manifest, p); + const bash = bashResolve(p); + if (ts !== bash) disagreements.push({ path: p, ts, bash }); + } + expect(disagreements).toEqual([]); + }); + + it('bash and TS agree on the framework subtree roots', () => { + const manifest = loadManifest(FRAMEWORK_ROOT); + expect(bashSubtreeRoots().sort()).toEqual(frameworkSubtreeRoots(manifest).sort()); + }); +}); diff --git a/packages/mosaic/src/framework/manifest.spec.ts b/packages/mosaic/src/framework/manifest.spec.ts new file mode 100644 index 0000000..928fedd --- /dev/null +++ b/packages/mosaic/src/framework/manifest.spec.ts @@ -0,0 +1,223 @@ +import { describe, it, expect } from 'vitest'; +import { readdirSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + parseManifest, + loadManifest, + matchGlob, + resolveOwnership, + frameworkSubtreeRoots, + planPrune, + type FrameworkManifest, +} from './manifest.js'; + +const FRAMEWORK_ROOT = fileURLToPath(new URL('../../framework', import.meta.url)); + +const SAMPLE = ` +# comment +[framework] +CONSTITUTION.md +guides/** +tools/** + +[operator] +SOUL.md +*.local.md +agents/** +tools/_lib/credentials.json +`; + +describe('parseManifest', () => { + it('splits entries into framework and operator sections, ignoring comments/blanks', () => { + const m = parseManifest(SAMPLE); + expect(m.framework).toEqual(['CONSTITUTION.md', 'guides/**', 'tools/**']); + expect(m.operator).toEqual([ + 'SOUL.md', + '*.local.md', + 'agents/**', + 'tools/_lib/credentials.json', + ]); + }); + + it('rejects an entry that appears before any section header', () => { + expect(() => parseManifest('stray.md\n[framework]\n')).toThrow(/before any \[section\]/); + }); + + it('rejects an unknown section header', () => { + expect(() => parseManifest('[bogus]\nx\n')).toThrow(/Unknown manifest section/); + }); +}); + +describe('matchGlob', () => { + it('matches an exact file', () => { + expect(matchGlob('CONSTITUTION.md', 'CONSTITUTION.md')).toBe(true); + expect(matchGlob('CONSTITUTION.md', 'AGENTS.md')).toBe(false); + }); + + it('treats a bare directory entry as covering its descendants', () => { + expect(matchGlob('memory', 'memory')).toBe(true); + expect(matchGlob('memory', 'memory/notes.md')).toBe(true); + expect(matchGlob('memory', 'memoryfoo')).toBe(false); + }); + + it('** matches any depth including the root itself', () => { + expect(matchGlob('agents/**', 'agents')).toBe(true); + expect(matchGlob('agents/**', 'agents/a.conf')).toBe(true); + expect(matchGlob('agents/**', 'agents/nested/deep.conf')).toBe(true); + expect(matchGlob('agents/**', 'agentsX')).toBe(false); + }); + + it('* stays within a single segment', () => { + expect(matchGlob('*.local.md', 'SOUL.local.md')).toBe(true); + expect(matchGlob('*.local.md', 'a/SOUL.local.md')).toBe(false); + }); +}); + +describe('resolveOwnership (deny-wins + fail-safe)', () => { + const m = parseManifest(SAMPLE); + + it('operator globs win over framework globs (carve-out inside a framework subtree)', () => { + expect(resolveOwnership(m, 'tools/_lib/credentials.json')).toBe('operator'); + expect(resolveOwnership(m, 'tools/git/pr-create.sh')).toBe('framework'); + }); + + it('framework-declared paths resolve to framework', () => { + expect(resolveOwnership(m, 'guides/E2E-DELIVERY.md')).toBe('framework'); + expect(resolveOwnership(m, 'CONSTITUTION.md')).toBe('framework'); + }); + + it('UNKNOWN paths default to operator (the #791 root-cause guarantee)', () => { + expect(resolveOwnership(m, 'agents/coder0.conf')).toBe('operator'); // declared + expect(resolveOwnership(m, 'harvester/sop.md')).toBe('operator'); // undeclared → fail-safe + expect(resolveOwnership(m, 'totally-unknown-dir/x')).toBe('operator'); + expect(resolveOwnership(m, 'random-root-file.md')).toBe('operator'); + }); +}); + +describe('planPrune (pure prune planner)', () => { + const m = parseManifest(SAMPLE); + + it('prunes a retired framework file inside a shipped subtree', () => { + const del = planPrune({ + manifest: m, + targetPaths: ['guides/OLD.md', 'guides/KEEP.md'], + sourcePaths: ['guides/KEEP.md'], + }); + expect(del).toEqual(['guides/OLD.md']); + }); + + it('never prunes operator-reserved paths even when absent from source', () => { + const del = planPrune({ + manifest: m, + targetPaths: ['agents/coder0.conf', 'tools/_lib/credentials.json', 'SOUL.local.md'], + sourcePaths: [], + }); + expect(del).toEqual([]); + }); + + it('never prunes UNKNOWN paths outside every framework subtree (fail-safe)', () => { + const del = planPrune({ + manifest: m, + targetPaths: ['harvester/sop.md', 'my-fleet.yaml', 'unknown-dir/deep/x'], + sourcePaths: [], + }); + expect(del).toEqual([]); + }); + + it('never prunes single-file framework entries (reconcile-managed, not in subtree)', () => { + const del = planPrune({ manifest: m, targetPaths: ['CONSTITUTION.md'], sourcePaths: [] }); + expect(del).toEqual([]); + }); + + it('property: delete-set ⊆ {framework-owned ∧ in-target ∧ not-in-source} and ∩ operator = ∅', () => { + const operatorish = [ + 'agents/a.conf', + 'policy/p.md', + 'SOUL.local.md', + 'memory/m.md', + 'tools/_lib/credentials.json', + 'harvester/sop.md', + 'unknown-top/x', + 'another-unknown/deep/y.txt', + ]; + const frameworkish = ['guides/A.md', 'guides/sub/B.md', 'tools/git/x.sh']; + const targetPaths = [...operatorish, ...frameworkish]; + const del = planPrune({ manifest: m, targetPaths, sourcePaths: [] }); + + for (const p of del) { + expect(resolveOwnership(m, p)).toBe('framework'); + expect(targetPaths).toContain(p); + } + // No operator/unknown path ever appears in the delete-set. + for (const p of operatorish) expect(del).not.toContain(p); + }); +}); + +describe('frameworkSubtreeRoots', () => { + it('returns only the /** subtree roots, not single-file entries', () => { + const m = parseManifest(SAMPLE); + expect(frameworkSubtreeRoots(m)).toEqual(['guides', 'tools']); + }); +}); + +// ── SSOT manifest: shipped-file completeness (§6.2) ────────────────────────── +// A newly-shipped framework file must not silently fall outside the manifest — +// if it did, the updater could neither guarantee it as framework-owned nor +// prune it when retired. Every file the framework actually ships must resolve +// to `framework` (except the defaults/{SOUL,USER}.md identity seeds, which are +// operator-owned by design). +describe('manifest completeness against shipped framework tree', () => { + const manifest = loadManifest(FRAMEWORK_ROOT); + + const IGNORED_TOP = new Set(['.git', 'node_modules']); + // Framework-shipped files that are operator-owned by design: the identity + // seeds under defaults/, and the `.gitkeep` placeholder that lets the empty + // operator-owned memory/ directory exist in git. + function isOperatorShipped(rel: string): boolean { + if (rel === 'defaults/SOUL.md' || rel === 'defaults/USER.md') return true; + if (rel.startsWith('memory/')) return true; + return false; + } + + function walk(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const abs = join(dir, entry); + const rel = relative(FRAMEWORK_ROOT, abs); + if (IGNORED_TOP.has(rel)) continue; + if (statSync(abs).isDirectory()) out.push(...walk(abs)); + else out.push(rel); + } + return out; + } + + it('every shipped framework file resolves to framework ownership', () => { + const shipped = walk(FRAMEWORK_ROOT); + const misclassified = shipped.filter( + (p) => !isOperatorShipped(p) && resolveOwnership(manifest, p) !== 'framework', + ); + expect(misclassified).toEqual([]); + }); + + it('the operator-owned surface from #791 resolves to operator', () => { + const operatorPaths = [ + 'agents/coder0.conf', + 'fleet/agents/coder0.env', + 'memory/note.md', + 'policy/custom.md', + 'SOUL.local.md', + 'USER.local.md', + 'STANDARDS.local.md', + 'tools/_lib/credentials.json', + 'fleet/roster.yaml', + 'fleet/roster.json', + 'fleet/run/coder0.hb', + 'fleet/backlog/data.db', + 'fleet/roles.local/custom.md', + ]; + for (const p of operatorPaths) { + expect(resolveOwnership(manifest, p), p).toBe('operator'); + } + }); +}); diff --git a/packages/mosaic/src/framework/manifest.ts b/packages/mosaic/src/framework/manifest.ts new file mode 100644 index 0000000..05e5303 --- /dev/null +++ b/packages/mosaic/src/framework/manifest.ts @@ -0,0 +1,188 @@ +import { readFileSync } from 'node:fs'; + +/** + * Framework path-ownership manifest (#791). + * + * The updater must operate from an explicit framework-owned path manifest and + * NEVER write outside it. This module is the TypeScript reader for the shared + * SSOT manifest (`packages/mosaic/framework/framework-manifest.txt`) that the + * bash installer also consumes. Keeping both paths on one data file is what + * closes the two-copies-drift failure class (see #631 → #791). + * + * Everything here is pure (parse + resolve + plan) so the ownership guarantee + * is unit- and property-testable without touching the filesystem. + */ + +export type Ownership = 'framework' | 'operator'; + +export interface FrameworkManifest { + /** Globs the updater MAY create/overwrite, and prune only when retired. */ + readonly framework: readonly string[]; + /** Globs the updater must NEVER write over or prune. Win over `framework`. */ + readonly operator: readonly string[]; +} + +type Section = 'framework' | 'operator' | null; + +/** + * Parse the line-oriented manifest text. `#` comments and blank lines are + * ignored; `[framework]` / `[operator]` headers switch the active section. + * Lines before any header are rejected — the format must be explicit. + */ +export function parseManifest(text: string): FrameworkManifest { + const framework: string[] = []; + const operator: string[] = []; + let section: Section = null; + + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const raw = lines[i] ?? ''; + const line = raw.trim(); + if (line === '' || line.startsWith('#')) continue; + + if (line === '[framework]') { + section = 'framework'; + continue; + } + if (line === '[operator]') { + section = 'operator'; + continue; + } + if (line.startsWith('[')) { + throw new Error(`Unknown manifest section header on line ${i + 1}: ${line}`); + } + + if (section === null) { + throw new Error(`Manifest entry before any [section] header on line ${i + 1}: ${line}`); + } + (section === 'framework' ? framework : operator).push(line); + } + + return { framework, operator }; +} + +/** Read and parse the manifest from a framework root directory. */ +export function loadManifest(frameworkRoot: string): FrameworkManifest { + const text = readFileSync(`${frameworkRoot}/framework-manifest.txt`, 'utf-8'); + return parseManifest(text); +} + +/** + * Match a mosaic-home-relative POSIX path against one glob. + * + * Supported: `**` (any depth, including zero segments) and `*` (any run of + * characters within a single segment, not crossing `/`). A glob with no + * wildcard matches either the exact path OR any path beneath it (so a bare + * directory entry like `memory` covers `memory/notes.md`). + */ +export function matchGlob(glob: string, relPath: string): boolean { + const path = normalizeRel(relPath); + const pattern = normalizeRel(glob); + if (pattern === '') return false; + + if (!pattern.includes('*')) { + // Exact file, or any descendant of a bare directory prefix. + return path === pattern || path.startsWith(`${pattern}/`); + } + + const re = new RegExp(`^${globToRegExpBody(pattern)}$`); + return re.test(path); +} + +/** True if the path matches any glob in the list. */ +export function matchesAny(globs: readonly string[], relPath: string): boolean { + return globs.some((g) => matchGlob(g, relPath)); +} + +/** + * Resolve ownership of a mosaic-home-relative path (deny-wins / fail-safe): + * operator globs win, then framework globs, else operator by default. + */ +export function resolveOwnership(manifest: FrameworkManifest, relPath: string): Ownership { + if (matchesAny(manifest.operator, relPath)) return 'operator'; + if (matchesAny(manifest.framework, relPath)) return 'framework'; + return 'operator'; +} + +/** + * The set of `[framework]` subtree roots that pruning is allowed to descend + * into (glob entries of the form `dir/**`). Single-file framework entries + * (e.g. `CONSTITUTION.md`) are reconcile-managed and never pruned. + */ +export function frameworkSubtreeRoots(manifest: FrameworkManifest): string[] { + const roots: string[] = []; + for (const g of manifest.framework) { + if (g.endsWith('/**')) roots.push(g.slice(0, -3)); + } + return roots; +} + +export interface PrunePlanInput { + readonly manifest: FrameworkManifest; + /** Mosaic-home-relative paths currently present in the target. */ + readonly targetPaths: readonly string[]; + /** Mosaic-home-relative paths the framework currently ships (source). */ + readonly sourcePaths: readonly string[]; +} + +/** + * Pure prune planner — the testable seam of the #791 fix. + * + * Returns the delete-set: target paths that are framework-owned, live inside a + * shipped framework subtree, and are absent from the current source (retired + * framework files). By construction the result never contains an operator-owned + * or unknown path: those either resolve to `operator` or fall outside every + * framework subtree root, so they are structurally unreachable by pruning. + */ +export function planPrune(input: PrunePlanInput): string[] { + const { manifest, targetPaths, sourcePaths } = input; + const source = new Set(sourcePaths.map(normalizeRel)); + const roots = frameworkSubtreeRoots(manifest); + + const deleteSet: string[] = []; + for (const raw of targetPaths) { + const path = normalizeRel(raw); + if (source.has(path)) continue; // still shipped — keep + if (resolveOwnership(manifest, path) !== 'framework') continue; // operator/unknown — never prune + if (!roots.some((root) => path === root || path.startsWith(`${root}/`))) continue; // outside shipped subtrees + deleteSet.push(path); + } + return deleteSet; +} + +function normalizeRel(p: string): string { + return p.replace(/\\/g, '/').replace(/^\.\//, '').replace(/^\/+/, '').replace(/\/+$/, ''); +} + +/** Translate a glob body (already normalized) into a RegExp source fragment. */ +function globToRegExpBody(pattern: string): string { + let out = ''; + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i]; + if (c === undefined) continue; + if (c === '*') { + if (pattern[i + 1] === '*') { + // `**` — any depth. `a/**` must also match the bare root `a`, so when a + // literal `/` was just emitted, make it optional along with the rest. + i++; + let trailingSlash = false; + if (pattern[i + 1] === '/') { + i++; + trailingSlash = true; + } + if (out.endsWith('/')) { + out = `${out.slice(0, -1)}(?:/.*)?`; + } else if (trailingSlash) { + out += '(?:.*/)?'; + } else { + out += '.*'; + } + } else { + out += '[^/]*'; + } + } else { + out += c.replace(/[.+?^${}()|[\]\\]/g, '\\$&'); + } + } + return out; +} diff --git a/packages/mosaic/src/platform/file-ops.ts b/packages/mosaic/src/platform/file-ops.ts index cce7887..55a689e 100644 --- a/packages/mosaic/src/platform/file-ops.ts +++ b/packages/mosaic/src/platform/file-ops.ts @@ -62,16 +62,28 @@ function rotateBackups(filePath: string): void { /** * Sync a source directory to a target, with optional preserve paths. * Replaces the rsync/cp logic from install.sh. + * + * `isOperatorOwned` is the #791 ownership guard: when supplied, any source path + * it flags as operator-owned is never copied (the framework must never write an + * operator path). Callers derive it from the shared framework manifest so the TS + * and bash sync paths obey one source of truth. This copy is non-destructive — + * it never deletes a target file — so honoring the guard is sufficient to leave + * operator config untouched. */ export function syncDirectory( source: string, target: string, - options: { preserve?: string[]; excludeGit?: boolean } = {}, + options: { + preserve?: string[]; + excludeGit?: boolean; + isOperatorOwned?: (relPath: string) => boolean; + } = {}, ): void { // Guard: source and target are the same directory — nothing to sync if (resolve(source) === resolve(target)) return; const preserveSet = new Set(options.preserve ?? []); + const isOperatorOwned = options.isOperatorOwned ?? (() => false); // Collect files from source function copyRecursive(src: string, dest: string, relBase: string): void { @@ -86,7 +98,7 @@ export function syncDirectory( if (options.excludeGit && (dirName === '.git' || relPath.includes('/.git'))) return; // Skip preserved paths at top level - if (preserveSet.has(relPath) && existsSync(dest)) return; + if (relPath !== '' && preserveSet.has(relPath) && existsSync(dest)) return; mkdirSync(dest, { recursive: true }); for (const entry of readdirSync(src)) { @@ -101,6 +113,10 @@ export function syncDirectory( // Skip preserved files at top level if (preserveSet.has(relPath) && existsSync(dest)) return; + // #791: never write an operator-owned path (the framework owns only its + // own files; unknown paths resolve to operator and are skipped too). + if (isOperatorOwned(relPath)) return; + mkdirSync(dirname(dest), { recursive: true }); copyFileSync(src, dest); } diff --git a/packages/mosaic/src/runtime/update-checker.ts b/packages/mosaic/src/runtime/update-checker.ts index 492a0f2..6b1aee6 100644 --- a/packages/mosaic/src/runtime/update-checker.ts +++ b/packages/mosaic/src/runtime/update-checker.ts @@ -488,9 +488,13 @@ export function getInstallAllCommand(outdated: PackageUpdateResult[]): string { // `mosaic update` installs the new npm CLI but, on its own, leaves the framework // files in ~/.config/mosaic/ stale — so shipped launcher/runtime changes (e.g. // the agent-name export + native heartbeat) never ACTIVATE until a re-seed. -// These helpers run the package's own install.sh in sync-only mode (the P4 -// data-safe reconcile: framework-owned overwrite + backup-once; SOUL/USER/ -// *.local/credentials preserved) and, opt-in, relaunch durable agents. +// These helpers run the package's own install.sh in sync-only mode. The re-seed +// is manifest-driven (#791): keep mode writes ONLY framework-owned paths from the +// shared framework-manifest.txt and prunes only retired framework files inside +// shipped subtrees — every operator path (SOUL/USER/*.local/credentials, fleet +// roster + agents + backlog, and anything the manifest never anticipated) is +// left byte-identical. Contract files are still reconciled (overwrite + +// backup-once). Opt-in, this also relaunches durable agents. /** Resolve the framework/ directory bundled in the installed package. */ export function resolveBundledFrameworkRoot(): string {