jason.woltjeandClaude Opus 5.5
4b7405b86d
Adds ten cc-7 cases (Darkwing's six wrapper cases plus allow, block, SIGTERM-ignoring gate and inner timeout above the hook timeout), amends rely-on lines 2 to 4, removes the stale cc-1d-block-bare evidence, adds compare.sh, reads SDK_ENTRY from the environment, fixes the cc-5c row and the Pi exit-code sentence. Co-Authored-By: Claude Opus 5.5 <[email protected]>
116 lines
7.3 KiB
Bash
Executable File
116 lines
7.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Row S0 probe matrix runner. Usage: probe.sh <case>... | all
|
|
# Each case gets a fresh run directory under $S0_RUNS with its own HOME and
|
|
# working directory, a clean environment (env -i: no tokens), and a network
|
|
# namespace whose only interface is loopback, where the mock model listens.
|
|
# The probed action is creating work/target.txt.
|
|
set -u
|
|
here=$(dirname "$(readlink -f "$0")")
|
|
RUNS=${S0_RUNS:-$HOME/filbert-scratch/s0/runs}
|
|
PI_CLI=/mnt/storage/src/mosaic-stack/node_modules/@earendil-works/pi-coding-agent/dist/bundle/cli.js
|
|
CLAUDE_BIN=$HOME/.local/share/claude/versions/2.1.289
|
|
SDK_ENTRY=${SDK_ENTRY:-$HOME/filbert-scratch/s0/sdk/node_modules/@anthropic-ai/claude-agent-sdk/sdk.mjs}
|
|
MOCK_URL=http://127.0.0.1:47123
|
|
|
|
launch() { # launch <limit-s> <env...> -- <cmd...>
|
|
local limit=$1 envs=()
|
|
shift
|
|
while [ "$1" != -- ]; do envs+=("$1"); shift; done
|
|
shift
|
|
printf '%q ' "$@" > "$run/command.txt"; echo >> "$run/command.txt"
|
|
printf '%s\n' "${envs[@]}" > "$run/env.txt"
|
|
env -i PATH=/usr/bin:/bin LANG=C.UTF-8 HOME="$run/home" GATE_LOG="$run/gate.log" MOCK_LOG="$run/mock.jsonl" MOCK_SCRIPT="$script" "${envs[@]}" \
|
|
unshare -rn sh -c 'ip link set lo up && exec unshare -U --map-user=1000 --map-group=1000 "$0" "$@"' \
|
|
"$here/inner.sh" "$run" "$limit" -- "$@"
|
|
echo "== $case"; cat "$run/outcome.txt"
|
|
}
|
|
|
|
pi() { # pi <limit> <gate-mode|-> <tools> [extra pi args...]
|
|
local limit=$1 mode=$2 tools=$3
|
|
shift 3
|
|
mkdir -p "$run/home/.pi/agent"
|
|
printf '%s\n' '{"providers":{"probe":{"baseUrl":"'$MOCK_URL'","api":"anthropic-messages","apiKey":"probe-dummy","models":[{"id":"probe-model"}]}}}' > "$run/home/.pi/agent/models.json"
|
|
launch "$limit" PI_CODING_AGENT_DIR="$run/home/.pi/agent" GATE_MODE="$mode" -- \
|
|
node "$PI_CLI" -p --mode json --provider probe --model probe-model --no-session --no-extensions --no-skills --no-context-files --offline --tools "$tools" "$@" probe
|
|
}
|
|
|
|
cc() { # cc <limit> <hook-command|-> <hook-timeout|-> [extra claude args...]
|
|
local limit=$1 hook=$2 htimeout=$3
|
|
shift 3
|
|
if [ "$hook" = - ]; then echo '{}' > "$run/settings.json"
|
|
elif [ "$htimeout" = - ]; then printf '{"hooks":{"PreToolUse":[{"matcher":"%s","hooks":[{"type":"command","command":"%s"}]}]}}\n' "${matcher:-Write}" "$hook" > "$run/settings.json"
|
|
else printf '{"hooks":{"PreToolUse":[{"matcher":"%s","hooks":[{"type":"command","command":"%s","timeout":%s}]}]}}\n' "${matcher:-Write}" "$hook" "$htimeout" > "$run/settings.json"; fi
|
|
launch "$limit" "${cc_env[@]}" -- \
|
|
"$CLAUDE_BIN" -p probe --output-format stream-json --verbose --model claude-sonnet-4-5 --settings "$run/settings.json" --strict-mcp-config --no-session-persistence "$@"
|
|
}
|
|
|
|
sdk() { # sdk <limit> <hook-mode> [hook-timeout]
|
|
launch "$1" "${cc_env[@]}" HOOK_MODE="$2" HOOK_TIMEOUT="${3:-}" CLAUDE_BIN="$CLAUDE_BIN" SDK_ENTRY="$SDK_ENTRY" -- node "$here/sdk-probe.mjs"
|
|
}
|
|
|
|
one() {
|
|
case=$1 run=$RUNS/$1 matcher=Write
|
|
[ -n "$1" ] && rm -rf "$run"
|
|
mkdir -p "$run/home" "$run/work"
|
|
local tgt=$run/work/target.txt
|
|
local pw='[{"name":"write","input":{"path":"target.txt","content":"probe\n"}}]'
|
|
local pb='[{"name":"bash","input":{"command":"echo probe > target.txt"}}]'
|
|
local cw='[{"name":"Write","input":{"file_path":"'$tgt'","content":"probe\n"}}]'
|
|
local cb='[{"name":"Bash","input":{"command":"echo probe > target.txt","description":"probe"}}]'
|
|
cc_env=(CLAUDE_CONFIG_DIR="$run/home/.claude" ANTHROPIC_BASE_URL=$MOCK_URL ANTHROPIC_API_KEY=probe-dummy CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 DISABLE_TELEMETRY=1 DISABLE_AUTOUPDATER=1 DISABLE_ERROR_REPORTING=1)
|
|
local g=$here/claude-gate.sh ct=(--tools Read,Write,Bash --allowedTools Read,Write,Bash)
|
|
case $1 in
|
|
pi-0-nogate) script=$pw; pi 60 - read,write,bash ;;
|
|
pi-1-block) script=$pw; pi 60 block read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-2-throw) script=$pw; pi 60 throw read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-2b-throw-string) script=$pw; pi 60 throw-string read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-2c-exit) script=$pw; pi 60 exit read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-3-missing) script=$pw; pi 60 block read,write,bash -e "$here/no-such-gate.ts" ;;
|
|
pi-3b-broken) script=$pw; pi 60 block read,write,bash -e "$here/pi-gate-broken.ts" ;;
|
|
pi-4-hang) script=$pw; pi 90 hang read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-4b-slow) script=$pw; pi 60 slow read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-5-bash) script=$pb; pi 60 block read,write,bash -e "$here/pi-gate.ts" ;;
|
|
pi-5b-toollimit) script=$pb; pi 60 - read,write ;;
|
|
pi-5c-toollimit-write) script=$pw; pi 60 - read,bash ;;
|
|
cc-0-nogate) script=$cw; cc 90 - - "${ct[@]}" ;;
|
|
cc-1-block) script=$cw; cc 90 "$g block" - "${ct[@]}" ;;
|
|
cc-1b-deny-json) script=$cw; cc 90 "$g deny-json" - "${ct[@]}" ;;
|
|
cc-1c-block-bypass) script=$cw; cc 90 "$g block" - --tools Read,Write,Bash --permission-mode bypassPermissions ;;
|
|
cc-1d-bash-hook) script=$cb matcher=Bash; cc 90 "$g block" - "${ct[@]}" ;;
|
|
cc-1e-bash-hook-bare) script=$cb matcher=Bash; cc 90 "$g block" - "${ct[@]}" --bare ;;
|
|
cc-2-crash) script=$cw; cc 90 "$g crash" - "${ct[@]}" ;;
|
|
cc-2c-crash-events) script=$cw; cc 90 "$g crash" - "${ct[@]}" --include-hook-events ;;
|
|
cc-2b-crash3) script=$cw; cc 90 "$g crash3" - "${ct[@]}" ;;
|
|
cc-3-missing) script=$cw; cc 90 "$here/no-such-gate.sh block" - "${ct[@]}" ;;
|
|
cc-3b-noexec) script=$cw; cc 90 "$here/claude-gate-noexec.sh block" - "${ct[@]}" ;;
|
|
cc-4-hang) script=$cw; cc 90 "$g hang" 3 "${ct[@]}" ;;
|
|
cc-4c-hang-default) script=$cw; cc 150 "$g hang" - "${ct[@]}" ;;
|
|
cc-4d-hang700-default) script=$cw; cc 900 "$g hang700" - "${ct[@]}" ;;
|
|
cc-4b-hang-events) script=$cw; cc 90 "$g hang" 3 "${ct[@]}" --include-hook-events ;;
|
|
cc-5-bash) script=$cb; cc 90 "$g block" - "${ct[@]}" ;;
|
|
cc-5b-toollimit) script=$cb; cc 90 - - --tools Read,Write --allowedTools Read,Write ;;
|
|
cc-5c-disallowed) script=$cb; cc 90 - - --tools Read,Write,Bash --allowedTools Read,Write --disallowedTools Bash ;;
|
|
sdk-6a-deny) script=$cw; sdk 120 deny ;;
|
|
sdk-6-throw) script=$cw; sdk 120 throw ;;
|
|
sdk-6b-hang) script=$cw; sdk 120 hang 3 ;;
|
|
sdk-6c-hang-default) script=$cw; sdk 900 hang ;;
|
|
# Wrapped command hooks (round 2, from Darkwing's review of round 1):
|
|
# "|| exit 2" turns any gate failure into a block.
|
|
cc-7a-allow-wrap) script=$cw; cc 90 "$g allow || exit 2" - "${ct[@]}" ;;
|
|
cc-7b-block-wrap) script=$cw; cc 90 "$g block || exit 2" - "${ct[@]}" ;;
|
|
cc-7c-deny-json-wrap) script=$cw; cc 90 "$g deny-json || exit 2" - "${ct[@]}" ;;
|
|
cc-7d-crash-wrap) script=$cw; cc 90 "$g crash || exit 2" - "${ct[@]}" ;;
|
|
cc-7e-missing-wrap) script=$cw; cc 90 "$here/no-such-gate.sh block || exit 2" - "${ct[@]}" ;;
|
|
cc-7f-noexec-wrap) script=$cw; cc 90 "$here/claude-gate-noexec.sh block || exit 2" - "${ct[@]}" ;;
|
|
cc-7g-hang-wrap) script=$cw; cc 90 "timeout -k 1 2 $g hang || exit 2" 10 "${ct[@]}" ;;
|
|
cc-7h-hangterm-wrap) script=$cw; cc 90 "timeout -k 1 2 $g hangterm || exit 2" 10 "${ct[@]}" ;;
|
|
cc-7i-hangterm-wrap-nokill) script=$cw; cc 90 "timeout 2 $g hangterm || exit 2" 10 "${ct[@]}" ;;
|
|
cc-7j-hang-wrap-inner-above) script=$cw; cc 90 "timeout -k 1 20 $g hang || exit 2" 3 "${ct[@]}" ;;
|
|
*) echo "unknown case $1" >&2; return 2 ;;
|
|
esac
|
|
}
|
|
|
|
cases=("$@")
|
|
[ "${1:-}" = all ] && cases=($(grep -oE '^ [a-z0-9-]+\)' "$0" | tr -d ' )'))
|
|
for c in "${cases[@]}"; do one "$c"; done
|