test(lease): enforce read-only tool invariant
This commit is contained in:
Jason Woltje
@@ -25,7 +25,7 @@
|
||||
"lint": "eslint src",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run --passWithNoTests && pnpm run test:framework-shell",
|
||||
"test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/promotion_binding_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-ci-queue-wait-tristate.sh && bash framework/tools/git/test-ci-queue-wait-github-checks.sh && bash framework/tools/git/test-pr-merge-queue-branch.sh && bash framework/tools/git/test-pr-merge-head-pin.sh && bash framework/tools/git/test-pr-merge-message-field.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/woodpecker/test-terminal-green-contract.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh"
|
||||
"test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/promotion_binding_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/invariant_r_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-ci-queue-wait-tristate.sh && bash framework/tools/git/test-ci-queue-wait-github-checks.sh && bash framework/tools/git/test-pr-merge-queue-branch.sh && bash framework/tools/git/test-pr-merge-head-pin.sh && bash framework/tools/git/test-pr-merge-message-field.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/woodpecker/test-terminal-green-contract.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mosaicstack/brain": "workspace:*",
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Invariant R: a read-only carve-out can neither disappear nor be shadowed.
|
||||
|
||||
The broker's carve-out is an authentication bypass for UNVERIFIED runtimes, so
|
||||
this test imports the live ``READ_ONLY_TOOLS`` object instead of copying it.
|
||||
Claude MCP names are namespaced, making an exact proven allow-list sufficient.
|
||||
Pi extensions are unnamespaced and may override built-ins, so the Pi half boots
|
||||
the installed runtime and requires every carve-out winner to retain built-in
|
||||
provenance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
||||
PACKAGE_ROOT = Path(__file__).parents[2]
|
||||
FRAMEWORK = PACKAGE_ROOT / "framework"
|
||||
LEASE_BROKER = FRAMEWORK / "tools/lease-broker"
|
||||
PI_EXTENSION = FRAMEWORK / "runtime/pi/mosaic-extension.ts"
|
||||
sys.path.insert(0, str(LEASE_BROKER))
|
||||
daemon = importlib.import_module("daemon")
|
||||
READ_ONLY_TOOLS = daemon.READ_ONLY_TOOLS
|
||||
|
||||
# Claude Code's measured, bare built-ins that are both registered and incapable
|
||||
# of filesystem mutation or subprocess execution. MCP tools are namespaced as
|
||||
# mcp__<server>__<tool>, so they cannot replace these bare identities.
|
||||
CLAUDE_PROVEN_READ_ONLY_TOOLS: Final = frozenset({"Read", "Grep", "Glob"})
|
||||
|
||||
# W-C owns removal of these known-dead names from daemon.py. This pin is
|
||||
# deliberately two-sided: until W-C lands, each name must remain in the live
|
||||
# carve-out and absent from the installed/proven set. W-C must delete the pin in
|
||||
# the same commit that removes the daemon entries, or this suite turns red.
|
||||
KNOWN_DEAD_CLAUDE: Final = frozenset({"Ls", "Find"})
|
||||
|
||||
# W-B measured Pi 0.84.1 through getAllTools(), observed every tool_call name,
|
||||
# and cross-checked dist/core/tools/index.js:17. Keep the mutating names here so
|
||||
# a runtime registry change forces the security classification to be revisited.
|
||||
PI_VERSION: Final = "0.84.1"
|
||||
PI_PROVEN_READ_ONLY_TOOLS: Final = frozenset({"read", "grep", "find", "ls"})
|
||||
PI_MUTATING_TOOLS: Final = frozenset({"bash", "edit", "write"})
|
||||
PI_MEASURED_BUILTINS: Final = PI_PROVEN_READ_ONLY_TOOLS | PI_MUTATING_TOOLS
|
||||
|
||||
# Falsifier-only inputs. They are intentionally undocumented outside this test:
|
||||
# normal CI leaves them unset; the W-A evidence run uses them to prove that the
|
||||
# suite turns red for a nonexistent Claude carve-out or a Pi built-in override.
|
||||
CLAUDE_EXTRA_TOOL_ENV: Final = "MOSAIC_INVARIANT_R_CLAUDE_EXTRA_TOOL"
|
||||
PI_EXTRA_EXTENSION_ENV: Final = "MOSAIC_INVARIANT_R_PI_EXTRA_EXTENSION"
|
||||
|
||||
|
||||
def probe_pi_registry() -> list[dict[str, object]]:
|
||||
"""Boot Pi's real registry and return the final winning tool definitions."""
|
||||
|
||||
pi = shutil.which("pi")
|
||||
if pi is None:
|
||||
raise AssertionError("installed Pi runtime is required for Invariant R")
|
||||
|
||||
version = subprocess.run(
|
||||
[pi, "--version"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if version.returncode != 0:
|
||||
raise AssertionError(f"Pi version probe failed: {version.stderr.strip()}")
|
||||
if version.stdout.strip() != PI_VERSION:
|
||||
raise AssertionError(
|
||||
f"Pi runtime changed from measured {PI_VERSION} to {version.stdout.strip()!r}; "
|
||||
"remeasure its registry before updating Invariant R"
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
output = root / "registry.json"
|
||||
observer = root / "registry-observer.ts"
|
||||
observer.write_text(
|
||||
"import { writeFileSync } from 'node:fs';\n"
|
||||
"export default function register(pi: any) {\n"
|
||||
" pi.on('session_start', () => {\n"
|
||||
f" writeFileSync({json.dumps(str(output))}, JSON.stringify(pi.getAllTools()));\n"
|
||||
" process.exit(0);\n"
|
||||
" });\n"
|
||||
"}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
command = [
|
||||
pi,
|
||||
"--mode",
|
||||
"text",
|
||||
"--no-session",
|
||||
"--no-approve",
|
||||
"--no-context-files",
|
||||
"--no-skills",
|
||||
"--no-prompt-templates",
|
||||
"--no-extensions",
|
||||
"-e",
|
||||
str(observer),
|
||||
"-e",
|
||||
str(PI_EXTENSION),
|
||||
]
|
||||
extra_extension = os.environ.get(PI_EXTRA_EXTENSION_ENV)
|
||||
if extra_extension:
|
||||
command.extend(("-e", extra_extension))
|
||||
command.append("Invariant R registry probe")
|
||||
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, "PI_OFFLINE": "1"},
|
||||
timeout=20,
|
||||
)
|
||||
if completed.returncode != 0 or not output.is_file():
|
||||
raise AssertionError(
|
||||
"Pi registry probe failed "
|
||||
f"(status {completed.returncode}): {completed.stderr.strip()}"
|
||||
)
|
||||
value = json.loads(output.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, list) or not value:
|
||||
raise AssertionError("Pi registry probe returned no tools; control failed")
|
||||
return value
|
||||
|
||||
|
||||
class InvariantRTest(unittest.TestCase):
|
||||
def test_live_carve_out_has_only_supported_runtimes(self) -> None:
|
||||
self.assertEqual(set(READ_ONLY_TOOLS), {"claude", "pi"})
|
||||
|
||||
def test_claude_carve_out_is_registered_proven_and_pinned_for_w_c(self) -> None:
|
||||
carve_out = set(READ_ONLY_TOOLS["claude"])
|
||||
falsifier = os.environ.get(CLAUDE_EXTRA_TOOL_ENV)
|
||||
if falsifier:
|
||||
carve_out.add(falsifier)
|
||||
|
||||
self.assertLessEqual(
|
||||
KNOWN_DEAD_CLAUDE,
|
||||
carve_out,
|
||||
"W-C changed the live Claude carve-out but left its known-defect pin stale",
|
||||
)
|
||||
self.assertTrue(
|
||||
KNOWN_DEAD_CLAUDE.isdisjoint(CLAUDE_PROVEN_READ_ONLY_TOOLS),
|
||||
"a W-C dead-name pin unexpectedly exists in Claude's installed/proven tool set",
|
||||
)
|
||||
self.assertEqual(
|
||||
carve_out - KNOWN_DEAD_CLAUDE,
|
||||
set(CLAUDE_PROVEN_READ_ONLY_TOOLS),
|
||||
"every unpinned Claude carve-out must exist and be in the exact proven read-only allow-list",
|
||||
)
|
||||
|
||||
def test_pi_carve_out_resolves_to_real_unshadowed_builtins(self) -> None:
|
||||
carve_out = set(READ_ONLY_TOOLS["pi"])
|
||||
self.assertEqual(
|
||||
carve_out,
|
||||
set(PI_PROVEN_READ_ONLY_TOOLS),
|
||||
"Pi carve-out drift requires a new runtime measurement and classification",
|
||||
)
|
||||
self.assertTrue(carve_out.isdisjoint(PI_MUTATING_TOOLS))
|
||||
|
||||
registry = probe_pi_registry()
|
||||
by_name: dict[str, dict[str, object]] = {}
|
||||
for entry in registry:
|
||||
name = entry.get("name")
|
||||
if not isinstance(name, str):
|
||||
self.fail(f"Pi registry entry has no string name: {entry!r}")
|
||||
by_name[name] = entry
|
||||
|
||||
builtin_names = {
|
||||
name
|
||||
for name, entry in by_name.items()
|
||||
if isinstance(entry.get("sourceInfo"), dict)
|
||||
and entry["sourceInfo"].get("source") == "builtin"
|
||||
}
|
||||
self.assertEqual(
|
||||
builtin_names,
|
||||
set(PI_MEASURED_BUILTINS),
|
||||
"Pi's real built-in registry drifted from the positive-control W-B measurement",
|
||||
)
|
||||
|
||||
for name in sorted(carve_out):
|
||||
with self.subTest(tool=name):
|
||||
self.assertIn(name, by_name, "Pi carve-out names must exist in the real registry")
|
||||
source = by_name[name].get("sourceInfo")
|
||||
self.assertIsInstance(source, dict)
|
||||
if isinstance(source, dict):
|
||||
self.assertEqual(
|
||||
source.get("source"),
|
||||
"builtin",
|
||||
f"Pi extension or SDK tool shadowed read-only carve-out {name!r}",
|
||||
)
|
||||
self.assertEqual(source.get("path"), f"<builtin:{name}>")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user