jason.woltjeandClaude Sonnet 5.5
587e64a27f
Packet under agents/researcher/work/i1-proxy-test: private-netns harness, stand-in upstream, report, evidence. Stand-in keys only; running proxy untouched. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
223 lines
13 KiB
Bash
Executable File
223 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Row 96 (#1590) test body. Runs INSIDE a private network namespace started by
|
|
# run-all.sh (unshare -rn), so only a loopback interface exists: nothing here can reach
|
|
# the running proxy on the host's 127.0.0.1:8317 or leave the machine.
|
|
# Do not run it directly; use run-all.sh.
|
|
set -u
|
|
PKT=$(cd "$(dirname "$0")" && pwd)
|
|
REPO=$(cd "$PKT/../../../.." && pwd)
|
|
STATE=$PKT/state
|
|
OUT=$PKT/out
|
|
PROXY_BIN=$HOME/.local/opt/EasyCLIProxyAPI/cpa-core/cli-proxy-api
|
|
CLAUDE_BIN=${MOSAIC_CLAUDE_BIN:-$HOME/.local/bin/claude}
|
|
PROXY_PORT=18317
|
|
UP_PORT=18400
|
|
URL=http://127.0.0.1:$PROXY_PORT
|
|
MGMT_KEY=standin-mgmt-key-0001
|
|
BOOTSTRAP_KEY=standin-bootstrap-key
|
|
UPSTREAM_KEY=standin-upstream-key-0001
|
|
MODEL=claude-sonnet-5-5
|
|
CLIENT_TIMEOUT=${CLIENT_TIMEOUT:-100}
|
|
UP_PID=""
|
|
PX_PID=""
|
|
|
|
note() { printf '%s %s\n' "$(date -u +%H:%M:%SZ)" "$*" | tee -a "$OUT/run.log"; }
|
|
|
|
stop_all() {
|
|
[ -z "$PX_PID" ] || kill "$PX_PID" 2>/dev/null
|
|
[ -z "$UP_PID" ] || kill "$UP_PID" 2>/dev/null
|
|
wait 2>/dev/null
|
|
}
|
|
trap stop_all EXIT
|
|
|
|
# --- processes -------------------------------------------------------------------------
|
|
start_upstream() {
|
|
node "$PKT/stand-in-upstream.mjs" "$UP_PORT" "$STATE/upstream.jsonl" "$UPSTREAM_KEY" "$STATE/ws/probe.txt" >> "$STATE/upstream.out" 2>&1 &
|
|
UP_PID=$!
|
|
}
|
|
start_proxy() {
|
|
# cwd is the state dir and HOME is inside it, so any relative or ~ path the proxy
|
|
# uses lands under the packet. Empty environment: no ambient variable reaches it.
|
|
( cd "$STATE" && exec env -i HOME="$STATE/home" PATH=/usr/bin:/bin "$PROXY_BIN" -config "$STATE/config.yaml" -local-model ) >> "$STATE/proxy.out" 2>&1 &
|
|
PX_PID=$!
|
|
}
|
|
wait_port() { # port
|
|
for _ in $(seq 1 50); do
|
|
(exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && return 0
|
|
sleep 0.2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# --- management API --------------------------------------------------------------------
|
|
# mgmt LABEL METHOD PATH [JSON] -> prints "HTTP <code>" and the body; one line per call
|
|
# in out/mgmt-calls.log so the report can count steps.
|
|
mgmt() {
|
|
local label=$1 method=$2 path=$3 data=${4:-}
|
|
local args=(-s -m 10 -X "$method" -H "Authorization: Bearer $MGMT_KEY" -o "$STATE/mgmt.body" -w '%{http_code}')
|
|
[ -z "$data" ] || args+=(-H 'Content-Type: application/json' --data "$data")
|
|
local code
|
|
code=$(curl "${args[@]}" "$URL$path")
|
|
printf '%s %s %s -> %s %s\n' "$label" "$method" "$path" "$code" "$(head -c 300 "$STATE/mgmt.body")" | tee -a "$OUT/mgmt-calls.log" >/dev/null
|
|
echo "HTTP $code $(head -c 400 "$STATE/mgmt.body")"
|
|
}
|
|
|
|
# snap_queue NAME : copy the usage queue (events are kept in memory for 60 s only) to
|
|
# out/usage-queue.NAME.json. Not logged as a management step.
|
|
snap_queue() {
|
|
# A read of the queue removes what it returns, and returns one event per call, so drain
|
|
# it: read until the reply is an empty array, then merge into one JSON array.
|
|
: > "$STATE/queue.parts"
|
|
for _ in $(seq 1 60); do
|
|
curl -s -m 10 -H "Authorization: Bearer $MGMT_KEY" -o "$STATE/queue.part" "$URL/v8/management/observability/usage/queue"
|
|
[ "$(tr -d ' \n' < "$STATE/queue.part")" != "[]" ] || break
|
|
cat "$STATE/queue.part" >> "$STATE/queue.parts"; echo >> "$STATE/queue.parts"
|
|
done
|
|
node -e 'const fs=require("fs");const ev=fs.readFileSync(process.argv[1],"utf8").split("\n").filter(Boolean).flatMap(l=>{try{return JSON.parse(l)}catch{return []}});fs.writeFileSync(process.argv[2],JSON.stringify(ev))' "$STATE/queue.parts" "$OUT/usage-queue.$1.json"
|
|
}
|
|
|
|
# probe LABEL KEY|- : one plain Messages request straight to the proxy. "-" sends no key.
|
|
probe() {
|
|
local label=$1 key=$2
|
|
local args=(-s -m 20 -o "$STATE/probe.body" -w '%{http_code}' -H 'Content-Type: application/json' -H 'anthropic-version: 2023-06-01')
|
|
[ "$key" = "-" ] || args+=(-H "x-api-key: $key")
|
|
local code
|
|
code=$(curl "${args[@]}" --data "{\"model\":\"$MODEL\",\"max_tokens\":32,\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}]}" "$URL/v1/messages")
|
|
printf '%s key=%s -> HTTP %s %s\n' "$label" "$key" "$code" "$(head -c 300 "$STATE/probe.body")" | tee -a "$OUT/probes.log"
|
|
snap_queue "$label"
|
|
}
|
|
|
|
# --- clients ---------------------------------------------------------------------------
|
|
# client LABEL claude|pi KEY|- REQUEST
|
|
# Runs the repo's own adapter with an environment limited to the session allowlist
|
|
# (packages/seat/src/session.mjs ENV_ALLOW) plus the adapter's MOSAIC_* inputs and the
|
|
# PATH the session builds. Output, exit code and the slice of the upstream log for this run go
|
|
# to out/<LABEL>.*.
|
|
client() {
|
|
local label=$1 kind=$2 key=$3 request=$4
|
|
local home=$STATE/home-$label sess=$STATE/sess-$label
|
|
mkdir -p "$home" "$sess"
|
|
local common=(PATH="$REPO/node_modules/.bin:/usr/bin:/bin" HOME="$home" USER=agent LOGNAME=agent LANG=C.UTF-8 TERM=dumb
|
|
ANTHROPIC_BASE_URL="$URL" MOSAIC_REQUEST="$request" MOSAIC_SYSTEM_PROMPT_FILE="$STATE/prompt.txt" MOSAIC_WORKSPACE="$STATE/ws" MOSAIC_SESSION_DIR="$sess")
|
|
local before; before=$(wc -l < "$STATE/upstream.jsonl" 2>/dev/null || echo 0)
|
|
local rc
|
|
if [ "$kind" = claude ]; then
|
|
local auth=(); [ "$key" = "-" ] || auth=(ANTHROPIC_AUTH_TOKEN="$key")
|
|
timeout -k 2 "${TMO:-$CLIENT_TIMEOUT}" env -i "${common[@]}" "${auth[@]}" MOSAIC_CLAUDE_BIN="$CLAUDE_BIN" MOSAIC_MODEL="$MODEL" MOSAIC_TOOLS=Read \
|
|
MOSAIC_CLAUDE_SETTINGS="$STATE/claude-settings.json" MOSAIC_CLAUDE_MCP_CONFIG="$STATE/claude-mcp.json" \
|
|
sh "$REPO/adapters/claude/adapter.sh" > "$OUT/$label.stdout" 2> "$OUT/$label.stderr" < /dev/null
|
|
else
|
|
local auth=(); [ "$key" = "-" ] || auth=(ANTHROPIC_API_KEY="$key")
|
|
# kind "pi" adds the one thing pi needs beyond the allowlist: a models.json under the
|
|
# session HOME that points the built-in anthropic provider at the proxy (pi docs
|
|
# models.md "Overriding Built-in Providers"). kind "pi-env" is the allowlist alone.
|
|
if [ "$kind" = pi ]; then
|
|
mkdir -p "$home/.pi/agent"
|
|
printf '{"providers":{"anthropic":{"baseUrl":"%s"}}}\n' "$URL" > "$home/.pi/agent/models.json"
|
|
fi
|
|
timeout -k 2 "${TMO:-$CLIENT_TIMEOUT}" env -i "${common[@]}" "${auth[@]}" PI_PROVIDER=anthropic PI_MODEL="$MODEL" MOSAIC_TOOLS=read \
|
|
sh "$REPO/adapters/pi/adapter.sh" > "$OUT/$label.stdout" 2> "$OUT/$label.stderr" < /dev/null
|
|
fi
|
|
rc=$?
|
|
echo "$rc" > "$OUT/$label.rc"
|
|
snap_queue "$label"
|
|
sed -n "$((before + 1)),\$p" "$STATE/upstream.jsonl" > "$OUT/$label.upstream.jsonl" 2>/dev/null
|
|
note "client $label ($kind, key=$key): exit=$rc, upstream requests=$(wc -l < "$OUT/$label.upstream.jsonl"), stdout: $(head -c 160 "$OUT/$label.stdout" | tr '\n' ' ')"
|
|
}
|
|
|
|
# --- setup -----------------------------------------------------------------------------
|
|
mkdir -p "$OUT" "$STATE/home" "$STATE/ws"
|
|
: > "$OUT/run.log"; : > "$OUT/mgmt-calls.log"; : > "$OUT/probes.log"
|
|
ip link set lo up || { echo "cannot bring lo up" >&2; exit 1; }
|
|
note "network inside the namespace: $(ip -o addr show | awk '{print $2 ":" $4}' | tr '\n' ' ')"
|
|
echo "probe-file-content: STAND-IN-PROBE-7f3a" > "$STATE/ws/probe.txt"
|
|
echo 'You are a test agent. Follow the request exactly.' > "$STATE/prompt.txt"
|
|
echo '{}' > "$STATE/claude-settings.json"
|
|
echo '{"mcpServers":{}}' > "$STATE/claude-mcp.json"
|
|
sed -e "s#@PROXY_PORT@#$PROXY_PORT#; s#@MGMT_KEY@#$MGMT_KEY#; s#@BOOTSTRAP_KEY@#$BOOTSTRAP_KEY#; s#@STATE@#$STATE#; s#@UPSTREAM_PORT@#$UP_PORT#; s#@UPSTREAM_KEY@#$UPSTREAM_KEY#" \
|
|
"$PKT/config.template.yaml" > "$STATE/config.yaml"
|
|
cp "$STATE/config.yaml" "$OUT/config.as-written.yaml"
|
|
|
|
start_upstream; wait_port "$UP_PORT" || { note "stand-in upstream did not start"; exit 1; }
|
|
start_proxy; wait_port "$PROXY_PORT" || { note "stand-in proxy did not start"; exit 1; }
|
|
note "stand-in upstream pid $UP_PID on 127.0.0.1:$UP_PORT; stand-in proxy pid $PX_PID on 127.0.0.1:$PROXY_PORT"
|
|
note "listeners visible inside the namespace: $(ss -ltnH | awk '{print $4}' | sort | tr '\n' ' ') (the host's 8317 is not among them)"
|
|
head -1 "$STATE/proxy.out" | tee -a "$OUT/run.log"
|
|
cp "$STATE/config.yaml" "$OUT/config.after-start.yaml"
|
|
|
|
# --- S1 issue two client keys from the command line ------------------------------------
|
|
KEY_A=sk-standin-agentA-$(openssl rand -hex 8)
|
|
KEY_B=sk-standin-agentB-$(openssl rand -hex 8)
|
|
echo "$KEY_A" > "$OUT/key-A.txt"; echo "$KEY_B" > "$OUT/key-B.txt"
|
|
note "S1 issue: GET the list, then PUT [bootstrap, A, B]"
|
|
mgmt S1 GET /v8/management/config/access/api-keys | tee -a "$OUT/run.log"
|
|
mgmt S1 PUT /v8/management/config/access/api-keys "[\"$BOOTSTRAP_KEY\",\"$KEY_A\",\"$KEY_B\"]" | tee -a "$OUT/run.log"
|
|
mgmt S1 GET /v8/management/config/access/api-keys | tee -a "$OUT/run.log"
|
|
probe S1-A-immediately "$KEY_A"
|
|
probe S1-B-immediately "$KEY_B"
|
|
cp "$STATE/config.yaml" "$OUT/config.after-issue.yaml"
|
|
|
|
# --- S2 the two clients, key A ---------------------------------------------------------
|
|
client S2-claude-A claude "$KEY_A" 'TOOL-TEST read probe.txt in your working directory and tell me what it says'
|
|
client S2-pi-env-A pi-env "$KEY_A" 'TOOL-TEST read probe.txt in your working directory and tell me what it says'
|
|
client S2-pi-A pi "$KEY_A" 'TOOL-TEST read probe.txt in your working directory and tell me what it says'
|
|
mgmt S2 GET /v8/management/observability/usage/api-keys | tee -a "$OUT/run.log"; cp "$STATE/mgmt.body" "$OUT/usage-api-keys.after-S2.json"
|
|
|
|
# --- S3 key B works too (plain streamed turn) ------------------------------------------
|
|
client S3-claude-B claude "$KEY_B" 'Say hello in one short sentence.'
|
|
|
|
# --- S4 revoke A -----------------------------------------------------------------------
|
|
note "S4 revoke A: DELETE /v0/management/api-keys?value=A"
|
|
mgmt S4 DELETE "/v0/management/api-keys?value=$KEY_A" | tee -a "$OUT/run.log"
|
|
mgmt S4 GET /v8/management/config/access/api-keys | tee -a "$OUT/run.log"
|
|
probe S4-A-after-revoke "$KEY_A"
|
|
probe S4-B-after-revoke "$KEY_B"
|
|
client S4-claude-A-revoked claude "$KEY_A" 'Say hello in one short sentence.'
|
|
client S4-pi-A-revoked pi "$KEY_A" 'Say hello in one short sentence.'
|
|
client S4-claude-B-after claude "$KEY_B" 'Say hello in one short sentence.'
|
|
mgmt S4 GET /v8/management/observability/usage/api-keys | tee -a "$OUT/run.log"; cp "$STATE/mgmt.body" "$OUT/usage-api-keys.after-S4.json"
|
|
cp "$STATE/config.yaml" "$OUT/config.after-revoke.yaml"
|
|
|
|
# --- S4b do issued keys survive a proxy restart? ----------------------------------------
|
|
note "S4b restart the stand-in proxy (same config file, same command)"
|
|
kill "$PX_PID" 2>/dev/null; wait "$PX_PID" 2>/dev/null; PX_PID=""
|
|
start_proxy; wait_port "$PROXY_PORT" || { note "stand-in proxy did not restart"; exit 1; }
|
|
probe S4b-B-after-restart "$KEY_B"
|
|
probe S4b-A-after-restart "$KEY_A"
|
|
|
|
# --- S5 unknown key and missing key ----------------------------------------------------
|
|
probe S5-unknown "sk-standin-never-issued-0000"
|
|
probe S5-missing -
|
|
TMO=330 client S5-claude-unknown claude "sk-standin-never-issued-0000" 'Say hello in one short sentence.'
|
|
client S5-pi-unknown pi "sk-standin-never-issued-0000" 'Say hello in one short sentence.'
|
|
client S5-claude-missing claude - 'Say hello in one short sentence.'
|
|
client S5-pi-missing pi - 'Say hello in one short sentence.'
|
|
|
|
# --- S6 upstream down ------------------------------------------------------------------
|
|
kill "$UP_PID" 2>/dev/null; wait "$UP_PID" 2>/dev/null; UP_PID=""
|
|
note "S6 stand-in upstream stopped"
|
|
probe S6-B-upstream-down "$KEY_B"
|
|
client S6-claude-B-upstream-down claude "$KEY_B" 'Say hello in one short sentence.'
|
|
client S6-pi-B-upstream-down pi "$KEY_B" 'Say hello in one short sentence.'
|
|
|
|
# --- collect ---------------------------------------------------------------------------
|
|
mgmt S7 GET /v8/management/observability/usage/api-keys | tee -a "$OUT/run.log"; cp "$STATE/mgmt.body" "$OUT/usage-api-keys.final.json"
|
|
# Queue read semantics: the queue is empty here (snap_queue drains it). Make three events
|
|
# (upstream is down, so each is a failed 500), then read once with ?count=100 and twice
|
|
# without. The numbers go to out/queue-read-semantics.txt.
|
|
for n in 1 2 3; do curl -s -m 10 -o /dev/null -H 'Content-Type: application/json' -H 'anthropic-version: 2023-06-01' -H "x-api-key: $KEY_B" -d '{"model":"claude-sonnet-5-5","max_tokens":8,"messages":[{"role":"user","content":"hi"}]}' "$URL/v1/messages"; done
|
|
{
|
|
echo "3 events pending. GET queue?count=100 -> $(curl -s -m 10 -H "Authorization: Bearer $MGMT_KEY" "$URL/v8/management/observability/usage/queue?count=100" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{console.log(JSON.parse(s).length+" event(s)")}catch{console.log("not JSON: "+s.slice(0,80))}})')"
|
|
echo "then plain GET queue -> $(curl -s -m 10 -H "Authorization: Bearer $MGMT_KEY" "$URL/v8/management/observability/usage/queue" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{console.log(JSON.parse(s).length+" event(s)")}catch{console.log("not JSON: "+s.slice(0,80))}})')"
|
|
echo "then plain GET queue again -> $(curl -s -m 10 -H "Authorization: Bearer $MGMT_KEY" "$URL/v8/management/observability/usage/queue" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{console.log(JSON.parse(s).length+" event(s)")}catch{console.log("not JSON: "+s.slice(0,80))}})')"
|
|
} > "$OUT/queue-read-semantics.txt"
|
|
snap_queue final
|
|
mgmt S7 GET /v8/management/observability/logs | tee -a "$OUT/run.log" >/dev/null; cp "$STATE/mgmt.body" "$OUT/logs-endpoint.final.json"
|
|
stop_all; UP_PID=""; PX_PID=""
|
|
mkdir -p "$OUT/proxy-logs"
|
|
cp "$STATE/upstream.jsonl" "$OUT/upstream.all.jsonl"
|
|
cp "$STATE/proxy.out" "$OUT/proxy.out.txt"
|
|
cp "$STATE/upstream.out" "$OUT/upstream.out.txt"
|
|
[ ! -d "$STATE/auth/logs" ] || cp -r "$STATE/auth/logs/." "$OUT/proxy-logs/"
|
|
note "inner script done; proxy log files: $(ls "$OUT/proxy-logs" | wc -l), size $(du -sk "$OUT/proxy-logs" | cut -f1) KiB"
|