fix(framework): detect installed tool drift (#1194) (#1195)

Co-authored-by: coder3 <[email protected]>

packages/mosaic/framework/tools/_lib/manifest.sh

@@ -69,7 +69,7 @@ _manifest_glob_to_ere() {
           out="$out.*"
         fi
       else
-        out="$out[^/]*"
+        out="${out}[^/]*"
       fi
     else
       case "$c" in
@@ -87,7 +87,8 @@ _manifest_compile_one() {
   local norm; norm="$(_manifest_norm "$1")"
   [[ -n "$norm" ]] || return 0
   if [[ "$norm" == *"*"* ]]; then
-    local re="^$(_manifest_glob_to_ere "$norm")\$"
+    local re
+    re="^$(_manifest_glob_to_ere "$norm")\$"
     if [[ "$2" == F ]]; then
       _MF_KIND+=(re); _MF_EXACT+=(""); _MF_RE+=("$re")
     else
@@ -183,7 +184,10 @@ _mo_matches() {
   for (( i = 0; i < n; i++ )); do
     if [[ "${_MO_KIND[i]}" == exact ]]; then
       pat="${_MO_EXACT[i]}"
-      [[ "$path" == "$pat" || "$path" == "$pat/"* ]] && return 0
+      # Operator exact entries are file carve-outs, not implicit directory
+      # prefixes. Subtree ownership must be declared explicitly as `dir/**`;
+      # otherwise one bare directory entry can hide all drift beneath it.
+      [[ "$path" == "$pat" ]] && return 0
     else
       re="${_MO_RE[i]}"
       [[ "$path" =~ $re ]] && return 0

packages/mosaic/framework/tools/_scripts/mosaic-doctor

@@ -153,6 +153,38 @@ warn_if_symlink_tree_present() {
 
 echo "[mosaic-doctor] Mosaic home: $MOSAIC_HOME"
 
+# Compare the framework tools that this CLI/package ships with the deployed
+# ~/.config copy that direct wrappers and systemd units actually execute. Doctor
+# is the right boundary: observational, operator-invoked, and already designed
+# to report drift without mutating live tooling or restarting active seats.
+framework_drift_checker="$(cd -- "$(dirname -- "$0")/../quality/scripts" && pwd)/framework-drift-check.py"
+if [[ -f "$framework_drift_checker" ]]; then
+  echo "[mosaic-doctor] Checking installed framework-tool drift..."
+  drift_timeout="${MOSAIC_DOCTOR_DRIFT_TIMEOUT_SEC:-15}"
+  if ! [[ "$drift_timeout" =~ ^[1-9][0-9]*$ ]]; then
+    warn "Invalid MOSAIC_DOCTOR_DRIFT_TIMEOUT_SEC='$drift_timeout' (expected positive integer); using 15s"
+    drift_timeout=15
+  fi
+  if command -v timeout >/dev/null 2>&1; then
+    set +e
+    timeout -s TERM -k 2 "${drift_timeout}s" \
+      python3 "$framework_drift_checker" --installed-root "$MOSAIC_HOME/tools"
+    drift_rc=$?
+    set -e
+    if [[ "$drift_rc" -eq 0 ]]; then
+      pass "Installed framework tools match shipped source"
+    elif [[ "$drift_rc" -eq 124 || "$drift_rc" -eq 137 || "$drift_rc" -eq 143 ]]; then
+      warn "CANNOT_ASSERT framework drift checker timed out after ${drift_timeout}s; continuing remaining doctor checks"
+    else
+      warn "Installed framework-tool drift detected (checker exit $drift_rc; no files changed)"
+    fi
+  else
+    warn "CANNOT_ASSERT timeout utility unavailable; refusing unbounded framework drift check and continuing remaining doctor checks"
+  fi
+else
+  warn "Framework drift checker is absent from the shipped tools tree"
+fi
+
 # Canonical Mosaic checks
 expect_file "$MOSAIC_HOME/STANDARDS.md"
 expect_file "$MOSAIC_HOME/USER.md"

packages/mosaic/framework/tools/quality/scripts/framework-drift-check.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Fail-closed comparison of deployed Mosaic tools to manifest-owned shipped tools."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+from pathlib import Path
+import stat
+import subprocess
+import sys
+
+
+def digest(path: Path) -> str:
+    value = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+
+def default_source_tools() -> Path:
+    return Path(__file__).resolve().parents[2]
+
+
+def normalize_source(path: Path) -> Path:
+    candidate = path.resolve()
+    return candidate / "tools" if (candidate / "tools").is_dir() else candidate
+
+
+def assert_traversable_directory(path: Path) -> None:
+    mode = stat.S_IMODE(path.stat(follow_symlinks=False).st_mode)
+    # At least one principal class must have both read and search. This catches
+    # mode-000 even for privileged reviewers for whom os.access() would lie.
+    if not any(mode & read and mode & execute for read, execute in ((0o400, 0o100), (0o040, 0o010), (0o004, 0o001))):
+        raise PermissionError(f"directory has no readable/searchable mode: {path}")
+
+
+def census(root: Path, *, reject_symlinks: bool) -> dict[str, Path]:
+    result: dict[str, Path] = {}
+
+    def onerror(error: OSError) -> None:
+        raise error
+
+    for current, directories, filenames in os.walk(root, topdown=True, followlinks=False, onerror=onerror):
+        current_path = Path(current)
+        assert_traversable_directory(current_path)
+        for name in directories:
+            entry = current_path / name
+            if entry.is_symlink() and reject_symlinks:
+                # A source symlink makes the shipped census incomplete. Deployed
+                # aliases are assessed later only when they occupy a required
+                # framework path; installed-only aliases remain operator state.
+                raise OSError(f"symlinked directory is not an independent census entry: {entry}")
+        for name in filenames:
+            entry = current_path / name
+            if entry.is_symlink():
+                if reject_symlinks:
+                    raise OSError(f"symlinked file is not an independent census entry: {entry}")
+                result[entry.relative_to(root).as_posix()] = entry
+                continue
+            mode = entry.stat(follow_symlinks=False).st_mode
+            if not stat.S_ISREG(mode):
+                raise OSError(f"non-regular census entry: {entry}")
+            if stat.S_IMODE(mode) & 0o444 == 0:
+                raise PermissionError(f"file has no readable mode: {entry}")
+            result[entry.relative_to(root).as_posix()] = entry
+    return result
+
+
+def classify_with_manifest(source: Path, relatives: list[str]) -> dict[str, str]:
+    framework = source.parent
+    manifest = framework / "framework-manifest.txt"
+    resolver = source / "_lib" / "manifest.sh"
+    if not manifest.is_file() or not os.access(manifest, os.R_OK):
+        raise OSError(f"ownership manifest is missing or unreadable: {manifest}")
+    if not resolver.is_file() or not os.access(resolver, os.R_OK):
+        raise OSError(f"canonical manifest resolver is missing or unreadable: {resolver}")
+    payload = "".join(f"tools/{relative}\n" for relative in relatives)
+    completed = subprocess.run(
+        ["bash", str(resolver), "classify"],
+        input=payload,
+        text=True,
+        capture_output=True,
+        check=False,
+        env={**os.environ, "MANIFEST_FILE": str(manifest)},
+    )
+    if completed.returncode != 0:
+        detail = completed.stderr.strip() or f"resolver rc={completed.returncode}"
+        raise OSError(f"ownership manifest failed canonical resolution: {detail}")
+    classified: dict[str, str] = {}
+    for line in completed.stdout.splitlines():
+        ownership, separator, manifest_path = line.partition("\t")
+        if not separator or not manifest_path.startswith("tools/") or ownership not in {"framework", "operator"}:
+            raise OSError(f"invalid canonical ownership output: {line!r}")
+        relative = manifest_path.removeprefix("tools/")
+        if relative in classified:
+            raise OSError(f"duplicate canonical ownership output: {relative}")
+        classified[relative] = ownership
+    if set(classified) != set(relatives):
+        raise OSError("canonical ownership output did not classify the complete source census")
+    return classified
+
+
+def has_symlinked_component(root: Path, relative: str) -> bool:
+    current = root
+    for component in Path(relative).parts:
+        current = current / component
+        if current.is_symlink():
+            return True
+    return False
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Detect deployed Mosaic framework-tool drift")
+    parser.add_argument("--source-root", type=Path, default=Path(os.environ["MOSAIC_FRAMEWORK_SOURCE_ROOT"]) if os.environ.get("MOSAIC_FRAMEWORK_SOURCE_ROOT") else default_source_tools())
+    parser.add_argument("--installed-root", type=Path, default=Path(os.environ.get("MOSAIC_HOME", Path.home() / ".config/mosaic")) / "tools")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args()
+
+    source = normalize_source(args.source_root)
+    installed = args.installed_root.resolve()
+    try:
+        if not source.is_dir():
+            raise OSError(f"source tools missing: {source}")
+        if not installed.is_dir():
+            raise OSError(f"installed tools missing: {installed}")
+        if source.samefile(installed):
+            raise OSError("source and installed roots identify the same filesystem object")
+        source_files = census(source, reject_symlinks=True)
+        if not source_files:
+            raise OSError("source tools census is empty")
+        ownership = classify_with_manifest(source, sorted(source_files))
+        required = sorted(relative for relative, owner in ownership.items() if owner == "framework")
+        if not required:
+            raise OSError("ownership manifest classifies zero shipped tools as framework-owned")
+        installed_files = census(installed, reject_symlinks=False)
+    except (OSError, PermissionError) as error:
+        print(f"[framework-drift] CANNOT_ASSERT {error}", file=sys.stderr)
+        return 2
+
+    in_sync: list[str] = []
+    stale: list[str] = []
+    not_installed: list[str] = []
+    unsafe_alias: list[str] = []
+    for relative in required:
+        deployed = installed / relative
+        if not deployed.is_file():
+            not_installed.append(relative)
+            continue
+        if has_symlinked_component(installed, relative):
+            unsafe_alias.append(relative)
+            continue
+        try:
+            if source_files[relative].samefile(deployed):
+                unsafe_alias.append(relative)
+            elif digest(source_files[relative]) == digest(deployed):
+                in_sync.append(relative)
+            else:
+                stale.append(relative)
+        except OSError as error:
+            print(f"[framework-drift] CANNOT_ASSERT cannot compare {relative}: {error}", file=sys.stderr)
+            return 2
+
+    source_relative = set(source_files)
+    installed_only = sorted(set(installed_files) - source_relative)
+    if args.verbose:
+        for relative in in_sync:
+            print(f"[framework-drift] IN_SYNC {relative}")
+    for relative in stale:
+        print(f"[framework-drift] STALE {relative}")
+    for relative in not_installed:
+        print(f"[framework-drift] NOT_INSTALLED {relative}")
+    for relative in unsafe_alias:
+        print(f"[framework-drift] UNSAFE_ALIAS {relative}")
+    if args.verbose:
+        for relative in installed_only:
+            print(f"[framework-drift] INSTALLED_ONLY operator-or-unknown {relative}")
+
+    print(
+        "[framework-drift] summary "
+        f"in-sync={len(in_sync)} stale={len(stale)} not-installed={len(not_installed)} "
+        f"unsafe-alias={len(unsafe_alias)} installed-only={len(installed_only)}"
+    )
+    print("[framework-drift] classification canonical framework-manifest ownership; installed-only=operator-or-unknown-preserved")
+    if stale or not_installed or unsafe_alias:
+        print("[framework-drift] FAIL deployed framework tools do not match independent shipped source; schedule a reviewed framework reseed", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

packages/mosaic/framework/tools/quality/scripts/test-framework-drift-check.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+CHECKER = Path(__file__).with_name("framework-drift-check.py")
+REAL_RESOLVER = CHECKER.parents[2] / "_lib" / "manifest.sh"
+
+
+class FrameworkDriftCheckTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.temp = tempfile.TemporaryDirectory()
+        root = Path(self.temp.name)
+        self.framework = root / "framework"
+        self.source = self.framework / "tools"
+        self.installed = root / "home" / "tools"
+        for directory in (self.source / "git", self.source / "_lib", self.installed / "git", self.installed / "_lib"):
+            directory.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(REAL_RESOLVER, self.source / "_lib" / "manifest.sh")
+        (self.source / "git" / "guard.sh").write_text("fixed\n")
+        (self.source / "git" / "new-wrapper.sh").write_text("new\n")
+        (self.source / "_lib" / "credentials.json").write_text("source-placeholder\n")
+        self.write_manifest()
+
+    def tearDown(self) -> None:
+        self.temp.cleanup()
+
+    def write_manifest(self, operator_extra: str = "") -> None:
+        (self.framework / "framework-manifest.txt").write_text(
+            "[framework]\ntools/**\n[operator]\ntools/_lib/credentials.json\n" + operator_extra
+        )
+
+    def run_check(self, *extra: str) -> subprocess.CompletedProcess[str]:
+        return subprocess.run(
+            [sys.executable, str(CHECKER), "--source-root", str(self.framework), "--installed-root", str(self.installed), *extra],
+            text=True, capture_output=True, check=False,
+            env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"},
+        )
+
+    def install_matching(self) -> None:
+        for relative in ("git/guard.sh", "git/new-wrapper.sh", "_lib/manifest.sh"):
+            shutil.copy2(self.source / relative, self.installed / relative)
+        (self.installed / "_lib" / "credentials.json").write_text("different-operator-secret\n")
+
+    def test_fails_loudly_and_classifies_stale_missing_and_installed_only(self) -> None:
+        (self.installed / "git" / "guard.sh").write_text("broken\n")
+        shutil.copy2(self.source / "_lib" / "manifest.sh", self.installed / "_lib" / "manifest.sh")
+        (self.installed / "local-helper.sh").write_text("operator\n")
+        result = self.run_check("--verbose")
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("STALE git/guard.sh", result.stdout)
+        self.assertIn("NOT_INSTALLED git/new-wrapper.sh", result.stdout)
+        self.assertIn("INSTALLED_ONLY operator-or-unknown local-helper.sh", result.stdout)
+        self.assertIn("FAIL deployed framework tools", result.stderr)
+
+    def test_passes_only_when_every_manifest_owned_source_file_matches(self) -> None:
+        self.install_matching()
+        result = self.run_check()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("stale=0 not-installed=0 unsafe-alias=0", result.stdout)
+
+    def test_exact_operator_directory_does_not_hide_framework_drift_beneath_it(self) -> None:
+        self.install_matching()
+        (self.installed / "git" / "guard.sh").write_text("drift-hidden-by-directory-entry\n")
+        self.write_manifest("tools/git\n")
+
+        result = self.run_check()
+
+        self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
+        self.assertIn("STALE git/guard.sh", result.stdout)
+
+    def test_manifest_is_required_and_policy_changes_take_effect(self) -> None:
+        self.install_matching()
+        (self.installed / "git" / "guard.sh").write_text("operator-divergence\n")
+        self.write_manifest("tools/git/guard.sh\n")
+        self.assertEqual(self.run_check().returncode, 0)
+        (self.framework / "framework-manifest.txt").unlink()
+        result = self.run_check()
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("CANNOT_ASSERT ownership manifest is missing", result.stderr)
+
+    def test_empty_and_unreadable_source_census_cannot_assert(self) -> None:
+        empty_framework = Path(self.temp.name) / "empty-framework"
+        empty_source = empty_framework / "tools"
+        empty_source.mkdir(parents=True)
+        shutil.copy2(self.framework / "framework-manifest.txt", empty_framework / "framework-manifest.txt")
+        # The canonical resolver is supplied outside the empty census solely so
+        # this probe reaches the explicit minimum-population guard.
+        result = subprocess.run([sys.executable, str(CHECKER), "--source-root", str(empty_framework), "--installed-root", str(self.installed)], text=True, capture_output=True)
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("CANNOT_ASSERT", result.stderr)
+
+        blocked = self.source / "blocked"
+        blocked.mkdir(); (blocked / "hidden.sh").write_text("hidden\n"); blocked.chmod(0)
+        try:
+            result = self.run_check()
+        finally:
+            blocked.chmod(0o700)
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("CANNOT_ASSERT", result.stderr)
+        self.assertTrue("Permission denied" in result.stderr or "no readable/searchable mode" in result.stderr)
+
+    def test_root_and_descendant_aliases_cannot_report_clean(self) -> None:
+        result = subprocess.run([sys.executable, str(CHECKER), "--source-root", str(self.framework), "--installed-root", str(self.source)], text=True, capture_output=True)
+        self.assertEqual(result.returncode, 2)
+        self.assertIn("same filesystem object", result.stderr)
+
+        shutil.copy2(self.source / "_lib" / "manifest.sh", self.installed / "_lib" / "manifest.sh")
+        shutil.rmtree(self.installed / "git")
+        (self.installed / "git").symlink_to(self.source / "git", target_is_directory=True)
+        result = self.run_check()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertTrue("symlinked directory" in result.stderr or "UNSAFE_ALIAS" in result.stdout)
+
+
+if __name__ == "__main__":
+    unittest.main()

packages/mosaic/framework/tools/quality/scripts/test-framework-drift-doctor.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Doctor must contain a stalled drift checker and continue its remaining audit.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DOCTOR="$SCRIPT_DIR/../../_scripts/mosaic-doctor"
+WORK="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/framework-drift-doctor}"
+rm -rf "$WORK"
+mkdir -p "$WORK/source/tools/quality/scripts" "$WORK/source/tools/_scripts" "$WORK/home/tools"
+cp "$DOCTOR" "$WORK/source/tools/_scripts/mosaic-doctor"
+cat > "$WORK/source/tools/quality/scripts/framework-drift-check.py" <<'PY'
+import time
+time.sleep(30)
+PY
+
+start=$(date +%s)
+set +e
+output=$(MOSAIC_HOME="$WORK/home" MOSAIC_DOCTOR_DRIFT_TIMEOUT_SEC=1 \
+  bash "$WORK/source/tools/_scripts/mosaic-doctor" --fail-on-warn 2>&1)
+rc=$?
+set -e
+elapsed=$(( $(date +%s) - start ))
+
+[[ "$rc" -ne 0 ]] || { echo "FAIL: checker timeout became doctor success" >&2; exit 1; }
+[[ "$elapsed" -lt 10 ]] || { echo "FAIL: checker hang escaped watchdog (${elapsed}s)" >&2; exit 1; }
+[[ "$output" == *"CANNOT_ASSERT framework drift checker timed out"* ]] || {
+  echo "FAIL: missing timeout CANNOT_ASSERT diagnostic" >&2; printf '%s\n' "$output" >&2; exit 1;
+}
+[[ "$output" == *"[mosaic-doctor] warnings="* ]] || {
+  echo "FAIL: doctor did not continue after checker timeout" >&2; printf '%s\n' "$output" >&2; exit 1;
+}
+
+echo "framework drift doctor watchdog regression passed"