fred
10a1f82031
ci/woodpecker/pr/ci Pipeline was canceled
Review finding from scooby: this PR added a failure branch the harness structurally could not reach. The fake tmux answered `has-session` only for `=_holder:0.0`, so every non-holder agent landed in the session-is-gone branch no matter what — the `elif` (tmux still reports the session, no pane PID after the retries) had zero coverage and no way to get any. That is the same shape as the bug this PR exists to fix, one layer down: a code path shipped green where the gate that should measure it cannot. Less severe, because the branch fails closed at exit 69 rather than reporting success — but "the harness can't reach it" is the sentence that precedes the next silent regression, so it gets closed here rather than filed. `MOSAIC_TEST_HELD_SESSIONS` lets a case name targets the shim should also answer for. It answers them only AFTER `new-session`, and that detail is the whole trick: the launcher asks `has-session` about the same name twice — once at line 255 where a yes means "already running, exit 0", and once at 417 where a yes means "the session survived". A shim answering yes to both short-circuits at the first and never reaches the branch under test. It would have looked like coverage while measuring the idempotency path. Both failure modes were measured, not reasoned about: - toggle absent (the old shim): `code=pane-did-not-survive` — the case lands on the wrong branch, which is exactly the unreachability being reported. - toggle answering unconditionally: launcher exits 0 via the idempotency short-circuit — "launcher reported success over a session with no resolvable pane PID". - toggle gated on new-session: `code=pane-pid-unresolved`, exit 69. The case also asserts the diagnostic is not `pane-did-not-survive` and does not mention the heartbeat, so the two pane faults cannot collapse into one message. Gates: bash -n · launcher harness rc=0 · test-fleet-units.sh (real tmux) rc=0 · fleet specs 342 passed. Refs #1241.
542 lines
23 KiB
Bash
Executable File
542 lines
23 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd)
|
|
START="$SCRIPT_DIR/start-agent-session.sh"
|
|
INTERACTION_START="$SCRIPT_DIR/start-interaction-service.sh"
|
|
ROOT=$(mktemp -d)
|
|
FAKE_BIN=$(mktemp -d)
|
|
TMUX_CALLS=$(mktemp)
|
|
trap 'rm -rf "$ROOT" "$FAKE_BIN" "$TMUX_CALLS"' EXIT
|
|
|
|
fail() {
|
|
echo "FAIL: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
cat > "$FAKE_BIN/tmux" <<'SHIM'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
printf '%s\0' "$@" >> "${MOSAIC_TEST_TMUX_CALLS:?}"
|
|
args=("$@")
|
|
index=0
|
|
if [ "${args[0]:-}" = -L ]; then index=2; fi
|
|
case "${args[$index]:-}" in
|
|
has-session)
|
|
# The holder always answers. MOSAIC_TEST_HELD_SESSIONS lets a case add
|
|
# other targets that should answer too — without it there is no way to
|
|
# model "tmux still reports the session" for a non-holder agent, and the
|
|
# launcher's pane-pid-unresolved branch is unreachable from this harness.
|
|
#
|
|
# A listed target answers only AFTER new-session, because the launcher asks
|
|
# this question twice about the same name: once before launching, where a
|
|
# yes means "already running, nothing to do, exit 0", and once after, where
|
|
# a yes means "the session survived". A shim that answered yes to both
|
|
# would short-circuit at the first and never reach the branch under test —
|
|
# it would look like coverage and measure the idempotency path instead.
|
|
for argument in "${args[@]}"; do
|
|
[ "$argument" = '=_holder:0.0' ] && exit 0
|
|
case " ${MOSAIC_TEST_HELD_SESSIONS:-} " in
|
|
*" $argument "*)
|
|
if tr '\0' '\n' < "${MOSAIC_TEST_TMUX_CALLS:?}" | grep -qxF new-session; then
|
|
exit 0
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
exit 1
|
|
;;
|
|
show-environment)
|
|
printf '%s\n' \
|
|
"HOME=${MOSAIC_TEST_HOME:?}" \
|
|
'PATH=/usr/bin:/bin' \
|
|
"PWD=${MOSAIC_TEST_HOME:?}" \
|
|
"MOSAIC_FLEET_OWNER=${MOSAIC_TEST_FLEET_OWNER:?}" \
|
|
'MOSAIC_TMUX_HOLDER=_holder' \
|
|
'MOSAIC_TMUX_SOCKET=mosaic-test'
|
|
exit 0
|
|
;;
|
|
list-panes) printf '%s\n' "${MOSAIC_TEST_PANE_PID:-}"; exit 0 ;;
|
|
new-session)
|
|
if [ "${MOSAIC_TEST_EXECUTE_PANE:-}" = 1 ]; then
|
|
for ((index = 0; index < ${#args[@]}; index++)); do
|
|
if [ "${args[$index]}" = /usr/bin/env ]; then
|
|
"${args[@]:$index}"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
exit 0
|
|
;;
|
|
*) exit 0 ;;
|
|
esac
|
|
SHIM
|
|
chmod +x "$FAKE_BIN/tmux"
|
|
|
|
cat > "$FAKE_BIN/mosaic" <<'SHIM'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
env -0 > "${MOSAIC_HOME:?}/fleet/pane-environment"
|
|
SHIM
|
|
chmod +x "$FAKE_BIN/mosaic"
|
|
|
|
# The runtime the rosters below name. The launcher resolves it against PANE_PATH
|
|
# before spawning (#1241), so it has to exist somewhere the pane would find it —
|
|
# not merely on the launcher's own PATH.
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$FAKE_BIN/pi"
|
|
chmod +x "$FAKE_BIN/pi"
|
|
|
|
# PANE_PATH is derived partly from `npm config get prefix`. Left to the real npm
|
|
# it would splice whatever the host has installed into the path under test, and
|
|
# the missing-binary cases below would pass or fail by accident of the machine.
|
|
cat > "$FAKE_BIN/npm" <<'SHIM'
|
|
#!/usr/bin/env bash
|
|
printf '%s\n' "${MOSAIC_TEST_NPM_PREFIX:-/nonexistent}"
|
|
SHIM
|
|
chmod +x "$FAKE_BIN/npm"
|
|
|
|
# PANE_PATH always ends in the system path. A host that installs these there can
|
|
# not measure the missing-binary cases at all, and a green run would mean
|
|
# nothing — so say so instead of passing.
|
|
for host_binary in mosaic pi; do
|
|
if PATH=/usr/local/bin:/usr/bin:/bin command -v "$host_binary" >/dev/null 2>&1; then
|
|
fail "host provides '$host_binary' in the system path; missing-binary cases are not measurable here"
|
|
fi
|
|
done
|
|
|
|
write_generated() {
|
|
local home="$1"
|
|
local agent="$2"
|
|
mkdir -p "$home/fleet/agents" "$home/fleet/run"
|
|
chmod 700 "$home" "$home/fleet" "$home/fleet/agents" "$home/fleet/run"
|
|
printf '123e4567-e89b-12d3-a456-426614174000\n' > "$home/fleet/run/holder-owner"
|
|
chmod 600 "$home/fleet/run/holder-owner"
|
|
cat > "$home/fleet/agents/$agent.env.generated" <<EOF
|
|
MOSAIC_AGENT_NAME=$agent
|
|
MOSAIC_AGENT_CLASS=code
|
|
MOSAIC_AGENT_RUNTIME=pi
|
|
MOSAIC_AGENT_MODEL=openai-codex/gpt-5.6-sol
|
|
MOSAIC_AGENT_REASONING=high
|
|
MOSAIC_AGENT_TOOL_POLICY=code
|
|
MOSAIC_AGENT_WORKDIR=$home/work
|
|
MOSAIC_TMUX_SOCKET=mosaic-test
|
|
EOF
|
|
chmod 600 "$home/fleet/agents/$agent.env.generated"
|
|
mkdir -p "$home/work"
|
|
install_pane_binaries "$home"
|
|
}
|
|
|
|
# `$PANE_HOME/.npm-global/bin` is one of the prefixes the launcher folds into
|
|
# PANE_PATH, so this is the pane's own view of "installed", distinct from the
|
|
# launcher's PATH. Tests that need a binary *absent* remove it from here.
|
|
install_pane_binaries() {
|
|
local pane_home="$1"
|
|
mkdir -p "$pane_home/.npm-global/bin"
|
|
local binary
|
|
for binary in mosaic pi; do
|
|
ln -sf "$FAKE_BIN/$binary" "$pane_home/.npm-global/bin/$binary"
|
|
done
|
|
}
|
|
|
|
run_start() {
|
|
local home="$1"
|
|
local agent="$2"
|
|
HOME="$home" PATH="$FAKE_BIN:$PATH" MOSAIC_TEST_TMUX_CALLS="$TMUX_CALLS" \
|
|
MOSAIC_TEST_PANE_PID="${MOSAIC_TEST_PANE_PID:-}" \
|
|
MOSAIC_TEST_HELD_SESSIONS="${MOSAIC_TEST_HELD_SESSIONS:-}" \
|
|
MOSAIC_TEST_HOME="$home" \
|
|
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
|
MOSAIC_HOME="$home" "$START" "$agent"
|
|
}
|
|
|
|
# Valid generated data launches only the fixed runtime argument array. It never
|
|
# reads an agent-command string or constructs a bash -c pane payload.
|
|
HOME_VALID="$ROOT/valid"
|
|
AGENT_VALID="coder0"
|
|
write_generated "$HOME_VALID" "$AGENT_VALID"
|
|
# A live pane PID is part of what "valid launch" means. Until #1241 this case
|
|
# ran with none, so the suite's one success path was itself a dead pane the
|
|
# launcher reported as fine.
|
|
MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_VALID" "$AGENT_VALID"
|
|
valid_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
|
echo "$valid_args" | grep -qF new-session || fail "valid generated projection did not reach tmux"
|
|
echo "$valid_args" | grep -qF 'mosaic' || fail "fixed mosaic launcher command missing"
|
|
echo "$valid_args" | grep -qF 'yolo' || fail "fixed yolo launcher command missing"
|
|
echo "$valid_args" | grep -qF 'pi' || fail "roster runtime missing"
|
|
if echo "$valid_args" | grep -qF 'bash -c'; then
|
|
fail "launcher constructed a shell command payload"
|
|
fi
|
|
|
|
# The pane must start through an absolute clean-environment boundary. Its
|
|
# runtime command remains an argv vector, but no holder/session environment
|
|
# control variable can pass through the pane command.
|
|
echo "$valid_args" | grep -qxF '/usr/bin/env' || fail "pane does not use absolute env"
|
|
echo "$valid_args" | grep -qxF -- '-i' || fail "pane environment is not cleared"
|
|
|
|
# The generated-file parent is a security boundary too: even a private regular
|
|
# file is untrusted if its parent can be replaced or written by another user.
|
|
# Validation must happen before fake tmux receives even a has-session call.
|
|
: > "$TMUX_CALLS"
|
|
HOME_UNSAFE_PARENT="$ROOT/unsafe-parent"
|
|
write_generated "$HOME_UNSAFE_PARENT" "coder-parent"
|
|
chmod 777 "$HOME_UNSAFE_PARENT/fleet/agents"
|
|
if output=$(run_start "$HOME_UNSAFE_PARENT" coder-parent 2>&1); then
|
|
fail "generated file under a world-writable parent was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before unsafe parent rejection"
|
|
echo "$output" | grep -qF 'code=unsafe-permissions' || fail "unsafe parent diagnostic missing"
|
|
|
|
: > "$TMUX_CALLS"
|
|
HOME_SYMLINK_PARENT="$ROOT/symlink-parent"
|
|
write_generated "$HOME_SYMLINK_PARENT" "coder-symlink-parent"
|
|
mv "$HOME_SYMLINK_PARENT/fleet/agents" "$HOME_SYMLINK_PARENT/private-agents"
|
|
ln -s "$HOME_SYMLINK_PARENT/private-agents" "$HOME_SYMLINK_PARENT/fleet/agents"
|
|
if output=$(run_start "$HOME_SYMLINK_PARENT" coder-symlink-parent 2>&1); then
|
|
fail "generated file under a symlinked parent was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before symlinked parent rejection"
|
|
echo "$output" | grep -qF 'code=unsafe-directory' || fail "symlinked parent diagnostic missing"
|
|
|
|
# Every managed ancestor is a boundary: MOSAIC_HOME, fleet, and agents. A
|
|
# symlink or group/world-writable ancestor must fail before environment parsing,
|
|
# workdir creation, or tmux effects. The malformed local input proves parsing
|
|
# was not reached when the ancestor rejection is reported.
|
|
assert_managed_ancestor_rejected() {
|
|
local ancestor="$1"
|
|
local hazard="$2"
|
|
local home="$ROOT/managed-${ancestor//\//-}-${hazard}"
|
|
local agent="coder-managed-${ancestor//\//-}-${hazard}"
|
|
local node
|
|
write_generated "$home" "$agent"
|
|
printf 'MOSAIC_AGENT_COMMAND=must-not-be-parsed\n' > "$home/fleet/agents/$agent.env.local"
|
|
chmod 600 "$home/fleet/agents/$agent.env.local"
|
|
rm -rf "$home/work"
|
|
|
|
case "$ancestor" in
|
|
MOSAIC_HOME) node="$home" ;;
|
|
MOSAIC_HOME/fleet) node="$home/fleet" ;;
|
|
MOSAIC_HOME/fleet/agents) node="$home/fleet/agents" ;;
|
|
*) fail "unknown managed ancestor: $ancestor" ;;
|
|
esac
|
|
|
|
if [ "$hazard" = symlink ]; then
|
|
local target="${node}-target"
|
|
mv "$node" "$target"
|
|
ln -s "$target" "$node"
|
|
else
|
|
chmod 777 "$node"
|
|
fi
|
|
|
|
: > "$TMUX_CALLS"
|
|
if output=$(run_start "$home" "$agent" 2>&1); then
|
|
fail "${hazard} $ancestor was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before $hazard $ancestor rejection"
|
|
[ ! -e "$home/work" ] || fail "workdir was created before $hazard $ancestor rejection"
|
|
echo "$output" | grep -qF "code=unsafe-" || fail "managed ancestor diagnostic missing"
|
|
if echo "$output" | grep -qF 'key=MOSAIC_AGENT_COMMAND'; then
|
|
fail "environment parsing ran before $hazard $ancestor rejection"
|
|
fi
|
|
}
|
|
|
|
for managed_ancestor in MOSAIC_HOME MOSAIC_HOME/fleet MOSAIC_HOME/fleet/agents; do
|
|
assert_managed_ancestor_rejected "$managed_ancestor" symlink
|
|
assert_managed_ancestor_rejected "$managed_ancestor" group-world-writable
|
|
done
|
|
|
|
# A local file cannot shadow any roster-derived generated key. Validation must
|
|
# happen before fake tmux receives even a has-session call.
|
|
: > "$TMUX_CALLS"
|
|
HOME_SHADOW="$ROOT/shadow"
|
|
write_generated "$HOME_SHADOW" "coder1"
|
|
printf 'MOSAIC_AGENT_RUNTIME=codex\n' > "$HOME_SHADOW/fleet/agents/coder1.env.local"
|
|
chmod 600 "$HOME_SHADOW/fleet/agents/coder1.env.local"
|
|
if output=$(run_start "$HOME_SHADOW" coder1 2>&1); then
|
|
fail "generated-key shadow was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before generated-key shadow rejection"
|
|
echo "$output" | grep -qF 'key=MOSAIC_AGENT_RUNTIME' || fail "shadow diagnostic omitted key"
|
|
echo "$output" | grep -qF 'sha256=' || fail "shadow diagnostic omitted hash"
|
|
if echo "$output" | grep -qF 'codex'; then
|
|
fail "shadow diagnostic leaked value"
|
|
fi
|
|
|
|
# Arbitrary command compatibility is quarantined/rejected as data. Diagnostics
|
|
# may name the key and hash but must never echo the privileged command text.
|
|
: > "$TMUX_CALLS"
|
|
HOME_COMMAND="$ROOT/command"
|
|
write_generated "$HOME_COMMAND" "coder2"
|
|
COMMAND_VALUE='mosaic yolo codex --dangerous'
|
|
printf 'MOSAIC_AGENT_COMMAND=%s\n' "$COMMAND_VALUE" > "$HOME_COMMAND/fleet/agents/coder2.env.local"
|
|
chmod 600 "$HOME_COMMAND/fleet/agents/coder2.env.local"
|
|
if output=$(run_start "$HOME_COMMAND" coder2 2>&1); then
|
|
fail "arbitrary command override was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before command rejection"
|
|
echo "$output" | grep -qF 'key=MOSAIC_AGENT_COMMAND' || fail "command diagnostic omitted key"
|
|
echo "$output" | grep -qF 'sha256=' || fail "command diagnostic omitted hash"
|
|
if echo "$output" | grep -qF "$COMMAND_VALUE"; then
|
|
fail "command diagnostic leaked command value"
|
|
fi
|
|
|
|
# Group/world-readable local input is not trusted even when its syntax is safe.
|
|
: > "$TMUX_CALLS"
|
|
HOME_PERMS="$ROOT/perms"
|
|
write_generated "$HOME_PERMS" "coder3"
|
|
printf 'MOSAIC_RUNTIME_BIN=/opt/mosaic/bin\n' > "$HOME_PERMS/fleet/agents/coder3.env.local"
|
|
chmod 644 "$HOME_PERMS/fleet/agents/coder3.env.local"
|
|
if output=$(run_start "$HOME_PERMS" coder3 2>&1); then
|
|
fail "world-readable local input was accepted"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before permissions rejection"
|
|
echo "$output" | grep -qF 'code=unsafe-permissions' || fail "permission diagnostic missing"
|
|
|
|
# A unit/holder-like clean bootstrap must yield a pane with trusted HOME and
|
|
# computed PATH only. The pane command itself must not carry loader, shell
|
|
# control, arbitrary sentinel, or stale bootstrap variables.
|
|
: > "$TMUX_CALLS"
|
|
HOME_PANE_BOUNDARY="$ROOT/pane-boundary/.config/mosaic"
|
|
write_generated "$HOME_PANE_BOUNDARY" "coder-pane-boundary"
|
|
PANE_TRUSTED_HOME="${HOME_PANE_BOUNDARY%/.config/mosaic}"
|
|
PANE_STALE_HOME="$ROOT/stale-home"
|
|
PANE_STALE_PATH="$ROOT/stale-bin"
|
|
PANE_BASH_ENV="$ROOT/pane-boundary.bash-env"
|
|
printf 'MOSAIC_RUNTIME_BIN=%s\n' "$FAKE_BIN" > \
|
|
"$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
|
|
chmod 600 "$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
|
|
LD_PRELOAD='/not/loaded/by-clean-bootstrap.so' \
|
|
BASH_ENV="$PANE_BASH_ENV" \
|
|
MOSAIC_UNTRUSTED_SENTINEL='must-not-reach-pane' \
|
|
HOME="$PANE_STALE_HOME" \
|
|
PATH="$PANE_STALE_PATH" \
|
|
/usr/bin/env -i \
|
|
"HOME=$PANE_TRUSTED_HOME" \
|
|
"PATH=$FAKE_BIN:/usr/bin:/bin" \
|
|
"MOSAIC_HOME=$HOME_PANE_BOUNDARY" \
|
|
"MOSAIC_TEST_TMUX_CALLS=$TMUX_CALLS" \
|
|
"MOSAIC_TEST_HOME=$PANE_TRUSTED_HOME" \
|
|
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
|
MOSAIC_TEST_EXECUTE_PANE=1 \
|
|
"MOSAIC_TEST_PANE_PID=$$" \
|
|
"$START" coder-pane-boundary
|
|
pane_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
|
echo "$pane_args" | grep -qxF "HOME=$PANE_TRUSTED_HOME" || \
|
|
fail "pane did not restore trusted HOME"
|
|
echo "$pane_args" | grep -qF "HOME=$PANE_STALE_HOME" && \
|
|
fail "pane inherited stale HOME"
|
|
echo "$pane_args" | grep -qF "$PANE_STALE_PATH" && fail "pane inherited stale PATH"
|
|
for blocked in LD_PRELOAD= BASH_ENV= MOSAIC_UNTRUSTED_SENTINEL=; do
|
|
echo "$pane_args" | grep -qF "$blocked" && fail "pane inherited $blocked"
|
|
done
|
|
|
|
after_pane_env=$(printf '%s\n' "$pane_args" | grep -n -m1 -F '/usr/bin/env' | cut -d: -f1)
|
|
[ -n "$after_pane_env" ] || fail "pane command did not use absolute env"
|
|
printf '%s\n' "$pane_args" | tail -n +"$after_pane_env" | grep -qxF -- '-i' || \
|
|
fail "pane command did not clear its environment"
|
|
pane_environment=$(tr '\0' '\n' < "$HOME_PANE_BOUNDARY/fleet/pane-environment")
|
|
echo "$pane_environment" | grep -qxF "HOME=$PANE_TRUSTED_HOME" || \
|
|
fail "runtime pane did not receive trusted HOME"
|
|
echo "$pane_environment" | grep -qF "$PANE_STALE_PATH" && fail "runtime pane received stale PATH"
|
|
for blocked in LD_PRELOAD= BASH_ENV= MOSAIC_UNTRUSTED_SENTINEL=; do
|
|
echo "$pane_environment" | grep -qF "$blocked" && fail "runtime pane received $blocked"
|
|
done
|
|
|
|
write_interaction_generated() {
|
|
local home="$1"
|
|
local agent="$2"
|
|
mkdir -p "$home/fleet/agents" "$home/fleet/run" "$home/work"
|
|
chmod 700 "$home" "$home/fleet" "$home/fleet/agents" "$home/fleet/run"
|
|
printf '123e4567-e89b-12d3-a456-426614174000\n' > "$home/fleet/run/holder-owner"
|
|
chmod 600 "$home/fleet/run/holder-owner"
|
|
cat > "$home/fleet/agents/$agent.env.generated" <<EOF
|
|
MOSAIC_AGENT_NAME=$agent
|
|
MOSAIC_AGENT_CLASS=operator-interaction
|
|
MOSAIC_AGENT_RUNTIME=pi
|
|
MOSAIC_AGENT_MODEL=openai/gpt-5.6-sol
|
|
MOSAIC_AGENT_REASONING=high
|
|
MOSAIC_AGENT_TOOL_POLICY=operator-interaction
|
|
MOSAIC_AGENT_WORKDIR=$home/work
|
|
MOSAIC_TMUX_SOCKET=mosaic-test
|
|
EOF
|
|
chmod 600 "$home/fleet/agents/$agent.env.generated"
|
|
}
|
|
|
|
run_interaction() {
|
|
local home="$1"
|
|
local agent="$2"
|
|
HOME="$home" PATH="$FAKE_BIN:$PATH" MOSAIC_TEST_TMUX_CALLS="$TMUX_CALLS" \
|
|
MOSAIC_TEST_HOME="$home" \
|
|
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
|
MOSAIC_HOME="$home" "$INTERACTION_START" "$agent"
|
|
}
|
|
|
|
write_heartbeat_local() {
|
|
local home="$1"
|
|
local agent="$2"
|
|
mkdir -p "$home/run"
|
|
cat > "$home/fleet/agents/$agent.env.local" <<EOF
|
|
MOSAIC_HEARTBEAT_RUN_DIR=$home/run
|
|
MOSAIC_HEARTBEAT_INTERVAL=1
|
|
EOF
|
|
chmod 600 "$home/fleet/agents/$agent.env.local"
|
|
}
|
|
|
|
wait_for_sidecar_status() {
|
|
local file="$1"
|
|
for _retry in $(seq 1 30); do
|
|
grep -qF 'status=ok' "$file" 2>/dev/null && return 0
|
|
sleep 0.1
|
|
done
|
|
fail "heartbeat sidecar did not resume after native marker became stale or absent"
|
|
}
|
|
|
|
# A fresh Pi-native marker is authoritative: the shell sidecar may start but
|
|
# must not overwrite Pi's busy/ok/model heartbeat. It must resume only when
|
|
# the marker is stale or absent.
|
|
HOME_NATIVE_FRESH="$ROOT/native-fresh"
|
|
write_generated "$HOME_NATIVE_FRESH" "coder-native-fresh"
|
|
write_heartbeat_local "$HOME_NATIVE_FRESH" "coder-native-fresh"
|
|
FRESH_HB="$HOME_NATIVE_FRESH/run/coder-native-fresh.hb"
|
|
printf 'ts=native\npid=1\nstatus=busy\nmodel=authoritative-model\n' > "$FRESH_HB"
|
|
touch "$FRESH_HB.native"
|
|
MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_NATIVE_FRESH" coder-native-fresh
|
|
sleep 0.3
|
|
fresh_content=$(cat "$FRESH_HB")
|
|
[ "$fresh_content" = 'ts=native
|
|
pid=1
|
|
status=busy
|
|
model=authoritative-model' ] || fail "fresh native heartbeat was overwritten"
|
|
|
|
HOME_NATIVE_STALE="$ROOT/native-stale"
|
|
write_generated "$HOME_NATIVE_STALE" "coder-native-stale"
|
|
write_heartbeat_local "$HOME_NATIVE_STALE" "coder-native-stale"
|
|
STALE_HB="$HOME_NATIVE_STALE/run/coder-native-stale.hb"
|
|
printf 'ts=native\npid=1\nstatus=busy\nmodel=stale-model\n' > "$STALE_HB"
|
|
touch -d '10 seconds ago' "$STALE_HB.native"
|
|
MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_NATIVE_STALE" coder-native-stale
|
|
wait_for_sidecar_status "$STALE_HB"
|
|
|
|
HOME_NATIVE_ABSENT="$ROOT/native-absent"
|
|
write_generated "$HOME_NATIVE_ABSENT" "coder-native-absent"
|
|
write_heartbeat_local "$HOME_NATIVE_ABSENT" "coder-native-absent"
|
|
ABSENT_HB="$HOME_NATIVE_ABSENT/run/coder-native-absent.hb"
|
|
printf 'ts=native\npid=1\nstatus=busy\nmodel=absent-model\n' > "$ABSENT_HB"
|
|
MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_NATIVE_ABSENT" coder-native-absent
|
|
wait_for_sidecar_status "$ABSENT_HB"
|
|
|
|
# The interaction wrapper delegates to the shared strict parser before applying
|
|
# its pinned policy, so malformed projection data wins over profile diagnostics.
|
|
: > "$TMUX_CALLS"
|
|
HOME_INTERACTION_MALFORMED="$ROOT/interaction-malformed"
|
|
write_interaction_generated "$HOME_INTERACTION_MALFORMED" "interaction-malformed"
|
|
printf 'UNTRUSTED_BOOTSTRAP=value\n' >> "$HOME_INTERACTION_MALFORMED/fleet/agents/interaction-malformed.env.generated"
|
|
if output=$(run_interaction "$HOME_INTERACTION_MALFORMED" interaction-malformed 2>&1); then
|
|
fail "interaction wrapper accepted malformed generated data"
|
|
fi
|
|
[ ! -s "$TMUX_CALLS" ] || fail "tmux ran before interaction strict-parser rejection"
|
|
echo "$output" | grep -qF 'code=unknown-key' || fail "interaction did not use shared strict parser first"
|
|
|
|
# A syntactically valid but policy-incompatible projection reaches the pinned
|
|
# interaction policy check only after strict parsing and never starts tmux.
|
|
: > "$TMUX_CALLS"
|
|
HOME_INTERACTION_POLICY="$ROOT/interaction-policy"
|
|
write_interaction_generated "$HOME_INTERACTION_POLICY" "interaction-policy"
|
|
perl -0pi -e 's/MOSAIC_AGENT_RUNTIME=pi/MOSAIC_AGENT_RUNTIME=codex/' \
|
|
"$HOME_INTERACTION_POLICY/fleet/agents/interaction-policy.env.generated"
|
|
if output=$(run_interaction "$HOME_INTERACTION_POLICY" interaction-policy 2>&1); then
|
|
fail "interaction wrapper accepted a policy-incompatible projection"
|
|
fi
|
|
interaction_policy_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
|
echo "$interaction_policy_args" | grep -qF 'new-session' && \
|
|
fail "interaction pinned-policy rejection created a tmux session"
|
|
echo "$output" | grep -qF 'operator interaction service requires runtime pi' || \
|
|
fail "interaction pinned-policy check did not follow strict parsing"
|
|
|
|
# #1241. The pane runs `mosaic yolo <runtime>` against PANE_PATH. A binary
|
|
# missing from that path is a launch failure, and it has to be named before the
|
|
# session is created — after it, the diagnostic dies with the pane.
|
|
assert_missing_pane_binary_rejected() {
|
|
local binary="$1"
|
|
local home="$ROOT/missing-$binary"
|
|
local agent="coder-missing-$binary"
|
|
write_generated "$home" "$agent"
|
|
rm -f "$home/.npm-global/bin/$binary"
|
|
|
|
: > "$TMUX_CALLS"
|
|
local output
|
|
if output=$(MOSAIC_TEST_PANE_PID=$$ run_start "$home" "$agent" 2>&1); then
|
|
fail "launch succeeded with '$binary' absent from the pane PATH"
|
|
fi
|
|
echo "$output" | grep -qF 'code=missing-binary' || fail "missing '$binary' diagnostic missing"
|
|
echo "$output" | grep -qF "'$binary'" || fail "missing-binary diagnostic did not name $binary"
|
|
if tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session; then
|
|
fail "launcher created a session it knew would die ($binary absent)"
|
|
fi
|
|
}
|
|
|
|
assert_missing_pane_binary_rejected mosaic
|
|
assert_missing_pane_binary_rejected pi
|
|
|
|
# #1241. tmux destroys a session when its pane command exits, so no pane PID a
|
|
# second after new-session means the runtime died on startup. This used to be a
|
|
# WARNING about the heartbeat sidecar followed by exit 0 — three layers above it
|
|
# then reported a fleet that was not running.
|
|
: > "$TMUX_CALLS"
|
|
HOME_DEAD_PANE="$ROOT/dead-pane"
|
|
write_generated "$HOME_DEAD_PANE" "coder-dead-pane"
|
|
if output=$(MOSAIC_TEST_PANE_PID='' run_start "$HOME_DEAD_PANE" coder-dead-pane 2>&1); then
|
|
fail "launcher reported success over a pane that did not survive"
|
|
fi
|
|
echo "$output" | grep -qF 'code=pane-did-not-survive' || fail "dead-pane diagnostic missing"
|
|
if echo "$output" | grep -qiF 'heartbeat'; then
|
|
fail "dead pane is still being reported as a heartbeat-sidecar problem"
|
|
fi
|
|
tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session || \
|
|
fail "dead-pane case did not reach the launch it is measuring"
|
|
|
|
# #1241, the other way a pane fails. Above, tmux destroyed the session and
|
|
# has-session said so. Here the session is still there and no PID comes back
|
|
# after the retries — a different fault (the pane is alive but unusable, or
|
|
# tmux is answering inconsistently) that an operator has to be told apart from
|
|
# a runtime that died on startup.
|
|
#
|
|
# This case exists because the branch that handles it shipped with nothing able
|
|
# to reach it: the shim answered has-session only for the holder, so every
|
|
# non-holder agent landed in the session-is-gone branch no matter what. A
|
|
# defensive branch nothing exercises is the same shape as the bug this whole
|
|
# change is about, one layer down.
|
|
: > "$TMUX_CALLS"
|
|
HOME_NO_PID="$ROOT/pane-no-pid"
|
|
write_generated "$HOME_NO_PID" "coder-no-pid"
|
|
if output=$(MOSAIC_TEST_PANE_PID='' MOSAIC_TEST_HELD_SESSIONS='=coder-no-pid:0.0' \
|
|
run_start "$HOME_NO_PID" coder-no-pid 2>&1); then
|
|
fail "launcher reported success over a session with no resolvable pane PID"
|
|
fi
|
|
echo "$output" | grep -qF 'code=pane-pid-unresolved' || \
|
|
fail "session-present/no-PID was not reported as pane-pid-unresolved: $output"
|
|
if echo "$output" | grep -qF 'code=pane-did-not-survive'; then
|
|
fail "a session tmux still reports was diagnosed as a destroyed session"
|
|
fi
|
|
if echo "$output" | grep -qiF 'heartbeat'; then
|
|
fail "an unresolvable pane PID is still being reported as a heartbeat-sidecar problem"
|
|
fi
|
|
|
|
# Exact stop derives the socket exclusively from the validated generated
|
|
# projection and ignores an ambient socket supplied by the caller.
|
|
: > "$TMUX_CALLS"
|
|
HOME_STOP="$ROOT/stop"
|
|
write_generated "$HOME_STOP" "coder-stop"
|
|
HOME="$HOME_STOP" PATH="$FAKE_BIN:$PATH" MOSAIC_TEST_TMUX_CALLS="$TMUX_CALLS" \
|
|
MOSAIC_TEST_HOME="$HOME_STOP" \
|
|
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
|
MOSAIC_HOME="$HOME_STOP" MOSAIC_TMUX_SOCKET=ambient-socket "$START" --stop coder-stop
|
|
stop_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
|
echo "$stop_args" | grep -qxF 'mosaic-test' || fail "exact stop did not use the validated generated socket"
|
|
echo "$stop_args" | grep -qxF 'kill-session' || fail "exact stop did not request session termination"
|
|
echo "$stop_args" | grep -qxF '=coder-stop' || fail "exact stop did not exact-match the generated agent name"
|
|
if echo "$stop_args" | grep -qF 'ambient-socket'; then
|
|
fail "exact stop trusted an ambient socket"
|
|
fi
|
|
|
|
echo 'ok - start-agent-session generated environment boundary'
|