fix(installer): fail closed on test enumeration errors

docs/reports/verification/1050-b8-redaction-control/positive-control.test.sh

@@ -2,6 +2,8 @@
 set -euo pipefail
 
 ROOT="/work"
+# shellcheck source=tools/test-enumeration-assertions.sh
+source "$ROOT/tools/test-enumeration-assertions.sh"
 TMP="$(mktemp -d "${TMPDIR:-/tmp}/mosaic-next-install-test-XXXXXX")"
 trap 'rm -rf "$TMP"' EXIT
 export TMPDIR="$TMP/runtime-tmp"
@@ -494,9 +496,9 @@ fi
 if [[ "$(cat "$TMP/canary-observed" 2>/dev/null || true)" != "$canary" ]]; then
   echo 'credential canary positive control was not exercised' >&2; exit 1
 fi
-if find "$TMPDIR" -maxdepth 1 -type f \( -name 'mosaic-phase-redacted.*' -o -name 'mosaic-post-redacted.*' \) -print -quit | grep -q .; then
-  echo 'redacted diagnostic staging file survived normal completion' >&2; exit 1
-fi
+test_assert_find_empty 'redacted diagnostic staging files' \
+  "$TMPDIR" -maxdepth 1 -type f \
+  \( -name 'mosaic-phase-redacted.*' -o -name 'mosaic-post-redacted.*' \) || exit 1
 
 printf '[test] framework nested capture redacts the same canary and URL variants\n'
 framework_test_home="$TMP/framework-redact-home"
@@ -553,9 +555,8 @@ for phase in P2 P3 P4 P5 P6 P7 P8; do
   [[ -s "$LOG" ]] || { echo "$phase fault never entered the real action path" >&2; exit 1; }
   [[ "$(tree_fingerprint "$HOME_DIR")" == "$before" ]] || { echo "$phase real rollback mismatch" >&2; exit 1; }
   grep -q "phase=$phase" "$TMP/fault-$phase.log"
-  if find "$TMP/fault-$phase" -type f -exec grep -l '"status"[[:space:]]*:[[:space:]]*"in-progress"' {} + 2>/dev/null | grep -q .; then
-    echo "$phase left an in-progress transaction" >&2; exit 1
-  fi
+  test_assert_no_file_content_match "$phase fault-state" \
+    '"status"[[:space:]]*:[[:space:]]*"in-progress"' "$TMP/fault-$phase" || exit 1
 done
 
 printf '[test] stale projection is preserved while the real fault path acquires a free OS lock\n'

docs/reports/verification/1050-c1-fix-round/01-pre-fix-positive-control.log

@@ -0,0 +1,13 @@
+[test] --next fast path pins resolved package versions
+[test] fast path failure falls back to source build
+[test] source-build failure is fatal and restores the pre-install prefix
+[test] corrupt source archive is fatal and restores the pre-install prefix
+[test] source archive with multiple extracted roots fails instead of selecting by find order
+[test] --dev source install does not require registry version resolution
+[test] explicit --ref keeps source lane and avoids @next lookup
+[test] --check --next rejects mismatched prerelease pipeline suffixes
+[test] full framework path receives P3 absolute CLI without relying on PATH
+[test] captured diagnostics redact seeded credential canary everywhere
+[test] framework nested capture redacts the same canary and URL variants
+[test] real P2-P8 actions run under fault injection and restore actual surfaces
+P2 left an in-progress transaction

docs/reports/verification/1050-c1-fix-round/02-pre-fix-permission-failure-attack.log

@@ -0,0 +1,14 @@
+[test] --next fast path pins resolved package versions
+[test] fast path failure falls back to source build
+[test] source-build failure is fatal and restores the pre-install prefix
+[test] corrupt source archive is fatal and restores the pre-install prefix
+[test] source archive with multiple extracted roots fails instead of selecting by find order
+[test] --dev source install does not require registry version resolution
+[test] explicit --ref keeps source lane and avoids @next lookup
+[test] --check --next rejects mismatched prerelease pipeline suffixes
+[test] full framework path receives P3 absolute CLI without relying on PATH
+[test] captured diagnostics redact seeded credential canary everywhere
+[test] framework nested capture redacts the same canary and URL variants
+[test] real P2-P8 actions run under fault injection and restore actual surfaces
+[test] stale projection is preserved while the real fault path acquires a free OS lock
+[test] installer next lane tests passed

docs/reports/verification/1050-c1-fix-round/03-regression-test-red.log

@@ -0,0 +1,2 @@
+[test] enumeration failure cannot mask a planted in-progress transaction
+[test] FAIL: planted in-progress transaction plus failed enumeration passed the full suite

docs/reports/verification/1050-c1-fix-round/04-post-fix-permission-failure-attack.log

@@ -0,0 +1,14 @@
+[test] --next fast path pins resolved package versions
+[test] fast path failure falls back to source build
+[test] source-build failure is fatal and restores the pre-install prefix
+[test] corrupt source archive is fatal and restores the pre-install prefix
+[test] source archive with multiple extracted roots fails instead of selecting by find order
+[test] --dev source install does not require registry version resolution
+[test] explicit --ref keeps source lane and avoids @next lookup
+[test] --check --next rejects mismatched prerelease pipeline suffixes
+[test] full framework path receives P3 absolute CLI without relying on PATH
+[test] captured diagnostics redact seeded credential canary everywhere
+[test] framework nested capture redacts the same canary and URL variants
+[test] real P2-P8 actions run under fault injection and restore actual surfaces
+find: ‘/tmp/mosaic-next-install-test-hqL7U0/fault-P2/blocked’: Permission denied
+[test] ERROR: P2 fault-state enumeration failed

docs/reports/verification/1050-c1-fix-round/README.md

@@ -0,0 +1,33 @@
+# #1050 C1 fix-round verification
+
+Frozen reviewed head before remediation: `378bc1afe3bc485adb8614897d66c5edccd4a527`.
+
+Status: **believed-fixed, pending jarvis validation**. PR #1054 is not self-merged and issue #1050 remains open.
+
+## Blocker B — fail-closed test enumeration
+
+The RED-first control used a real filesystem permission failure, not binary shadowing or PATH interception.
+
+1. A planted `{"status":"in-progress"}` file in a readable P2 fault tree made the complete real walk fail the frozen suite at `P2 left an in-progress transaction` (`01-pre-fix-positive-control.log`, exit 1).
+2. The same planted defect beneath a target-owned mode-0100 directory made real `find` report a permission failure. The frozen suite erased the producer failure and exited 0 with `installer next lane tests passed` (`02-pre-fix-permission-failure-attack.log`).
+3. The committed regression control initially failed because the child full-suite attack still exited 0 (`03-regression-test-red.log`).
+4. After remediation, the same child full-suite input exits 1 and names `[test] ERROR: P2 fault-state enumeration failed` (`04-post-fix-permission-failure-attack.log`). The ordinary full suite remains green.
+
+`tools/test-enumeration-assertions.sh` now captures each complete NUL-delimited population and checks the producer status before asserting absence. Content checks inspect the captured population and distinguish “no match” from a read error. The shared fail-closed implementation covers:
+
+- `tools/install-next-lane.test.sh`: redacted staging-file cleanup and fault-state transaction scan;
+- `tools/verified-installer-fetch.test.sh`: temporary-download cleanup;
+- `tools/install-state-machine.test.sh`: symlink-target non-mutation;
+- `docs/reports/verification/1050-b8-redaction-control/positive-control.test.sh`: both copied counterparts.
+
+No assertion was loosened. A1, A2, upgrade-guard, source-root, the species-2 sweep, #869, and expected-RED verdict rows remain outside this remediation.
+
+## Blocker A — installer digest
+
+The stale sidecar value was replaced with the exact `sha256sum` record for `tools/install.sh`:
+
+```text
+e59cb441a2f37ae9150f8eae470238e9d858a1816df93343d9784a6796676096  install.sh
+```
+
+Local `sha256sum -c tools/install.sh.sha256` and the workflow's exact expected/actual equality both pass. The immutable provider-fetch arm at the new `${CI_COMMIT_SHA}` is recorded in the freeze artifact after push; local equality alone is not treated as sufficient evidence.

docs/reports/verification/1050-c1-fix-round/documentation-checklist.md

@@ -0,0 +1,22 @@
+# Documentation completion checklist — #1050 C1 fix round
+
+## Required artifacts
+
+- [x] `docs/PRD.md` exists; #1050 C1 requirements remain current and unchanged.
+- [x] User guide: not applicable; no user-facing installer behavior changed.
+- [x] Admin guide: not applicable; no operator procedure or deployment behavior changed.
+- [x] Developer guide: existing `docs/guides/installer-state-machine.md` already defines the fail-closed and immutable-remote contracts; this round adds verification evidence without changing the contract.
+- [x] OpenAPI and endpoint index: not applicable; no API changed.
+- [x] Sitemap: not applicable; no navigation changed.
+
+## API and structural coverage
+
+- [x] API schema/auth/error coverage: not applicable; no endpoint changed.
+- [x] Guide book indexes: not applicable; no guide page was added or moved.
+- [x] Root hygiene preserved; all new artifacts are under `docs/reports/verification/1050-c1-fix-round/` and the active scratchpad remains under `docs/scratchpads/`.
+
+## Review and publishing
+
+- [x] Verification documentation is in the same logical change set as the shell-test remediation.
+- [x] Independent code and security reviews found no documentation blocker.
+- [x] Canonical evidence remains in-repo. No external publishing action was requested or performed.

docs/scratchpads/1050-install-state-machine-red-fixture.md

@@ -52,6 +52,7 @@ Implement C1 from the canonical greenfield-install PRD v2: a transactional P0–
 - [x] Debian/glibc checkout fixture now packages the complete current checkout, verifies its digest in-container, and reaches the expected attributable RED without host inheritance. CI compares its exact final phase map/reasons to `tools/fixtures/greenfield-expected-red.tsv`; the fixture remains red while the detector job is green only on an exact match.
 - [ ] Reviews complete. Reviews 80 (`rev-security-02`) and 81 (`rev-974`) requested changes at `3934e03f`; their eight non-overlapping detector findings are being remediated red-first. Current remediation adds canonical-image portability, absolute P3 CLI propagation, exact expected-RED schema/cardinality, passwd-HOME binding, created-path owner/mode policy, real-action P2–P8 fault injection, verified non-empty remote installer execution, and seeded secret-canary/redacted diagnostics. Both old verdicts become void when the remediation head moves and require fresh independent review.
 - [x] Successor remediation for review 90 is RED-first and recorded in `docs/reports/verification/1050-successor-remediation/`: the manifest now binds the complete supported final P0 reason; P4 rejects an incomplete created-path walk instead of discarding `find` failure; and the TERM no-exit control is independent of filesystem enumeration order while retaining a proven RED mutation. Pipeline 2224's 32/2 result was a path-order-sensitive control, not evidence that the resume bug's premise became stale. The enumeration-class sweep additionally replaced order-dependent `find | head -1` source-root selection with a checked complete inventory requiring exactly one extracted root.
+- [x] C1 fix round for reviews 92/93: Blocker B was completed first. RED was reproduced before implementation: a planted `{"status":"in-progress"}` record makes the complete real walk fail the suite at P2, while the same planted defect beneath a target-owned mode-0100 directory makes real `find` fail and the frozen suite falsely exit 0 with `installer next lane tests passed`. One shared helper now captures and checks the complete NUL-delimited population before testing absence across the primary and copied harness sites; grep no-match is distinct from read failure. The same attack child drives the full suite RED with a named enumeration error, while ordinary native and root/container runs pass. Blocker A then regenerated the exact installer sidecar; provider-fetch validation remains the required post-push bar. Working estimate: 16K tokens; no external hard cap; 60% context is the stop/report gate.
 
 ## Risks / blockers
 
@@ -82,3 +83,6 @@ Implement C1 from the canonical greenfield-install PRD v2: a transactional P0–
 - Comparator controls pass for verdict drift, unexpected exit, manifest shrink, missing phases, duplicate rows, unknown cases, and unknown kinds. Verified-fetch controls pass for successful execution and failed/empty/digest-mismatch rejection.
 - `bash tools/e2e-install-test.sh --lane next --source checkout --git present` returns the required expected RED in clean Debian/glibc as uid 1001: installer P0/P1/P2/P3/P7 PASS; P4/P5/P6/P8 and P9 blocking; no `Done.` claim; checkout archive digest pinned and current framework installer exercised. `tools/verify-greenfield-expected-red.sh` converts that expected detector result into a green CI assertion and fails on any unreviewed verdict drift.
 - Earlier repository gates passed: `pnpm typecheck`, `pnpm lint`, `pnpm format:check`, upgrade manifest/rollback/durable-snapshot/migration suites, and focused `@mosaicstack/mosaic` tests with an isolated npm prefix. Full exact-remediation rerun is required before push.
+- Review-93 RED evidence at frozen `378bc1a`: isolated positive-control full-suite exit `1` with `P2 left an in-progress transaction`; isolated permission-failure attack full-suite exit `0` with final `installer next lane tests passed`. No binary shadowing or PATH interception was used; the failure came from a real target-owned mode-0100 directory.
+- Fix-round GREEN: `pnpm test:installer`, native next-lane, root/`ci-base:latest` next-lane, state-machine, verified-fetch, Bash syntax, ShellCheck, `pnpm typecheck`, `pnpm lint`, `pnpm format:check`, and `git diff --check` pass. The direct attack child exits 1 and names `P2 fault-state enumeration failed`. All 81 Mosaic Vitest files / 1508 tests pass under an isolated npm prefix; the wider framework-shell chain reaches the pre-existing #973 Bash-line-number gate and exits 97, matching the known host-specific condition rather than this delta. Codex code review approved at 0.93 confidence with zero findings; Codex security review reported no risk at 0.96 confidence with zero findings.
+- Sidecar GREEN locally: `sha256sum -c tools/install.sh.sha256` and the workflow's expected/actual comparison both resolve `e59cb441a2f37ae9150f8eae470238e9d858a1816df93343d9784a6796676096`. This is not substituted for the required immutable provider-fetch arm at the pushed head.

tools/install-next-lane.test.sh

@@ -2,6 +2,8 @@
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck source=tools/test-enumeration-assertions.sh
+source "$ROOT/tools/test-enumeration-assertions.sh"
 TMP="$(mktemp -d "${TMPDIR:-/tmp}/mosaic-next-install-test-XXXXXX")"
 trap 'rm -rf "$TMP"' EXIT
 export TMPDIR="$TMP/runtime-tmp"
@@ -15,6 +17,31 @@ STATE="$TMP/state"
 LOG="$TMP/npm.log"
 mkdir -p "$FAKE_BIN" "$HOME_DIR" "$STATE"
 
+if [[ "${MOSAIC_TEST_ENUMERATION_FAILURE_CHILD:-0}" != "1" ]]; then
+  printf '[test] enumeration failure cannot mask a planted in-progress transaction\n'
+  set +e
+  if [[ "$(/usr/bin/id -u)" -eq 0 ]]; then
+    enumeration_control_output="$(
+      su -s /bin/bash nobody -c \
+        "TMPDIR=/tmp MOSAIC_TEST_ENUMERATION_FAILURE_CHILD=1 bash '$0'" 2>&1
+    )"
+    enumeration_control_status=$?
+  else
+    enumeration_control_output="$(MOSAIC_TEST_ENUMERATION_FAILURE_CHILD=1 bash "$0" 2>&1)"
+    enumeration_control_status=$?
+  fi
+  set -e
+  printf '%s\n' "$enumeration_control_output" > "$TMP/enumeration-failure-control.log"
+  if [[ "$enumeration_control_status" -eq 0 ]]; then
+    echo '[test] FAIL: planted in-progress transaction plus failed enumeration passed the full suite' >&2
+    exit 1
+  fi
+  grep -qF '[test] ERROR: P2 fault-state enumeration failed' "$TMP/enumeration-failure-control.log" || {
+    echo '[test] FAIL: failed fault-state enumeration was not named' >&2
+    exit 1
+  }
+fi
+
 # Model the supported non-root/glibc target explicitly even when this harness
 # itself runs as root in Alpine/BusyBox CI.
 cat > "$FAKE_BIN/id" <<'FAKE_ID'
@@ -518,9 +545,9 @@ fi
 if [[ "$(cat "$TMP/canary-observed" 2>/dev/null || true)" != "$canary" ]]; then
   echo 'credential canary positive control was not exercised' >&2; exit 1
 fi
-if find "$TMPDIR" -maxdepth 1 -type f \( -name 'mosaic-phase-redacted.*' -o -name 'mosaic-post-redacted.*' \) -print -quit | grep -q .; then
-  echo 'redacted diagnostic staging file survived normal completion' >&2; exit 1
-fi
+test_assert_find_empty 'redacted diagnostic staging files' \
+  "$TMPDIR" -maxdepth 1 -type f \
+  \( -name 'mosaic-phase-redacted.*' -o -name 'mosaic-post-redacted.*' \) || exit 1
 
 printf '[test] framework nested capture redacts the same canary and URL variants\n'
 framework_test_home="$TMP/framework-redact-home"
@@ -577,9 +604,15 @@ for phase in P2 P3 P4 P5 P6 P7 P8; do
   [[ -s "$LOG" ]] || { echo "$phase fault never entered the real action path" >&2; exit 1; }
   [[ "$(tree_fingerprint "$HOME_DIR")" == "$before" ]] || { echo "$phase real rollback mismatch" >&2; exit 1; }
   grep -q "phase=$phase" "$TMP/fault-$phase.log"
-  if find "$TMP/fault-$phase" -type f -exec grep -l '"status"[[:space:]]*:[[:space:]]*"in-progress"' {} + 2>/dev/null | grep -q .; then
-    echo "$phase left an in-progress transaction" >&2; exit 1
+  if [[ "${MOSAIC_TEST_ENUMERATION_FAILURE_CHILD:-0}" == "1" && "$phase" == "P2" ]]; then
+    mkdir -p "$TMP/fault-$phase/blocked"
+    printf '{"status":"in-progress"}\n' > "$TMP/fault-$phase/blocked/planted-in-progress.json"
+    chmod 0666 "$TMP/fault-$phase/blocked/planted-in-progress.json"
+    chmod 0100 "$TMP/fault-$phase/blocked"
+    trap 'chmod 0700 "$TMP/fault-P2/blocked" 2>/dev/null || true; rm -rf "$TMP"' EXIT
   fi
+  test_assert_no_file_content_match "$phase fault-state" \
+    '"status"[[:space:]]*:[[:space:]]*"in-progress"' "$TMP/fault-$phase" || exit 1
 done
 
 printf '[test] stale projection is preserved while the real fault path acquires a free OS lock\n'

tools/install-state-machine.test.sh

@@ -8,6 +8,8 @@
 set -uo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck source=tools/test-enumeration-assertions.sh
+source "$ROOT/tools/test-enumeration-assertions.sh"
 TMP="$(mktemp -d "${TMPDIR:-/tmp}/mosaic-install-state-test.XXXXXX")"
 trap 'rm -rf "$TMP"' EXIT
 failures=0
@@ -396,7 +398,8 @@ set -e
 grep -q '^\[P0\] FAIL:.*unsafe context' "$TMP/symlink-target.log" \
   && pass_case 'symlinked rollback parent was rejected by P0' \
   || fail_case 'symlinked rollback parent lacked an attributable P0 failure'
-[[ -z "$(find "$symlink_outside" -mindepth 1 -print -quit)" ]] || fail_case 'symlink target was mutated'
+test_assert_find_empty 'symlink target mutation check' "$symlink_outside" -mindepth 1 \
+  || fail_case 'symlink target was mutated or could not be enumerated'
 
 printf '[test] case: journal initialization failure is fatal before mutation\n'
 journal_home="$TMP/journal-failure/home"

tools/install.sh.sha256

@@ -1 +1 @@
-4cd391b0974d3cce6c2a98455420d45bc2a04cb624e3c4bf43a813b8e28693e6  install.sh
+e59cb441a2f37ae9150f8eae470238e9d858a1816df93343d9784a6796676096  install.sh

tools/test-enumeration-assertions.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Fail-closed filesystem-enumeration assertions for shell test harnesses.
+
+# Usage: test_assert_find_empty <label> <find arguments...>
+test_assert_find_empty() {
+  local label="$1"
+  shift
+  local inventory
+
+  inventory="$(mktemp "${TMPDIR:-/tmp}/mosaic-test-find.XXXXXX")" || {
+    printf '[test] ERROR: %s inventory allocation failed\n' "$label" >&2
+    return 2
+  }
+
+  if ! find "$@" -print0 > "$inventory"; then
+    rm -f "$inventory"
+    printf '[test] ERROR: %s enumeration failed\n' "$label" >&2
+    return 2
+  fi
+
+  if [[ -s "$inventory" ]]; then
+    rm -f "$inventory"
+    printf '[test] FAIL: %s was not empty\n' "$label" >&2
+    return 1
+  fi
+
+  rm -f "$inventory"
+  return 0
+}
+
+# Usage: test_assert_no_file_content_match <label> <extended-regex> <find roots/options...>
+test_assert_no_file_content_match() {
+  local label="$1"
+  local pattern="$2"
+  shift 2
+  local inventory path grep_status result=0
+
+  inventory="$(mktemp "${TMPDIR:-/tmp}/mosaic-test-find.XXXXXX")" || {
+    printf '[test] ERROR: %s inventory allocation failed\n' "$label" >&2
+    return 2
+  }
+
+  if ! find "$@" -type f -print0 > "$inventory"; then
+    rm -f "$inventory"
+    printf '[test] ERROR: %s enumeration failed\n' "$label" >&2
+    return 2
+  fi
+
+  while IFS= read -r -d '' path; do
+    grep_status=0
+    grep -Eq -- "$pattern" "$path" || grep_status=$?
+    if [[ "$grep_status" -eq 0 ]]; then
+      result=1
+      break
+    fi
+    if [[ "$grep_status" -ne 1 ]]; then
+      result=2
+      break
+    fi
+  done < "$inventory"
+
+  rm -f "$inventory"
+  if [[ "$result" -eq 1 ]]; then
+    printf '[test] FAIL: %s contained a forbidden match\n' "$label" >&2
+    return 1
+  fi
+  if [[ "$result" -eq 2 ]]; then
+    printf '[test] ERROR: %s content inspection failed\n' "$label" >&2
+    return 2
+  fi
+  return 0
+}

tools/verified-installer-fetch.test.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck source=tools/test-enumeration-assertions.sh
+source "$ROOT/tools/test-enumeration-assertions.sh"
 TMP="$(mktemp -d "${TMPDIR:-/tmp}/mosaic-fetch-contract.XXXXXX")"
 trap 'rm -rf "$TMP"' EXIT
 FAKE_BIN="$TMP/bin"; mkdir -p "$FAKE_BIN"
@@ -41,7 +43,7 @@ empty_sha="$(printf '' | sha256sum | awk '{print $1}')"
 mkdir -p "$TMP/downloads"
 output="$(TMPDIR="$TMP/downloads" PATH="$FAKE_BIN:$PATH" bash "$ROOT/tools/verified-installer-fetch.sh" fixture://ok "$ok_sha" -- marker)"
 [[ "$output" == 'executed:marker' ]]
-[[ -z "$(find "$TMP/downloads" -mindepth 1 -print -quit)" ]]
+test_assert_find_empty 'verified-installer temporary downloads' "$TMP/downloads" -mindepth 1
 printf '[test] PASS: digest-pinned fetched artifact executes and its temporary body is removed\n'
 
 for row in 'fixture://empty empty-body' 'fixture://failed failed-fetch'; do