Files
stack/scripts/test-release.sh
T

183 lines
10 KiB
Bash
Executable File

#!/usr/bin/env bash
# Sandboxed selftests for the release layer.
#
# Fast cases (version validation) need no Docker. State-machine cases
# (status/activate/refusal) run against a sandboxed config and therefore
# require the Docker daemon; they are skipped when it is unavailable.
set -uo pipefail
cd "$(dirname "$0")/.."
SANDBOX="$(mktemp -d)"
RELEASE_BACKUP="$(mktemp)"
cp RELEASE "$RELEASE_BACKUP"
# One exit trap: the repo RELEASE is ALWAYS restored from the backup,
# regardless of how the test run ends.
# Compose names the project after the checkout directory unless told otherwise,
# and `compose run --rm` leaves the project's <project>_default network behind.
# Every run gets its own project, and the exit trap removes that one network
# (row 305). It never runs `compose down` and never touches another network.
COMPOSE_ID="$(basename "$SANDBOX")"
COMPOSE_ID="$(printf '%s' "${COMPOSE_ID##*.}" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]')"
export COMPOSE_PROJECT_NAME="mosaic-test-release-$COMPOSE_ID"
remove_compose_network() { # absent is fine; any other failure is one stderr line, never a status
command -v docker >/dev/null 2>&1 || return 0
local net="${COMPOSE_PROJECT_NAME}_default" err
err="$(docker network rm "$net" 2>&1 >/dev/null)" && return 0
case "$err" in *"not found"*|*"No such network"*) return 0 ;; esac
echo "test-release: could not remove network $net: ${err%%$'\n'*}" >&2
return 0
}
trap 'cp "$RELEASE_BACKUP" RELEASE 2>/dev/null; rm -rf "$SANDBOX" "$RELEASE_BACKUP"; remove_compose_network' EXIT
# Compose mounts PI_AUTH_FILE. Point it at a fixture so no activation reaches
# the host's pi credential (row 244). chmod after the write: umask-proof (row 243).
{ mkdir -p "$SANDBOX/pi" && printf '{}' > "$SANDBOX/pi/auth.json" && chmod 600 "$SANDBOX/pi/auth.json"; } \
|| { echo "cannot write the pi auth fixture" >&2; exit 1; }
export PI_AUTH_FILE="$SANDBOX/pi/auth.json"
PASS=0
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
C_OK=$'\033[0;32m'; C_FAIL=$'\033[0;31m'; C_RESET=$'\033[0m'
else
C_OK=""; C_FAIL=""; C_RESET=""
fi
FAIL=0
expect_exit() {
local name="$1" expected="$2"
shift 3
local rc
"$@" >/dev/null 2>&1
rc=$?
if [ "$rc" -eq "$expected" ]; then
PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $name (exit $rc)"
else
FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $name (exit $rc, expected $expected)"
fi
}
check() {
if [ "$2" = "0" ]; then PASS=$((PASS+1)); echo "${C_OK}OK${C_RESET} $1"; else FAIL=$((FAIL+1)); echo "${C_FAIL}FAIL${C_RESET} $1"; fi
}
# ---------- fast: release identity ----------
expect_exit "valid RELEASE resolves" 0 -- bash -c 'source scripts/common.sh && load_release'
printf 'garbage\n' > RELEASE
expect_exit "invalid RELEASE exits 1" 1 -- bash -c 'source scripts/common.sh && load_release'
mv RELEASE "$SANDBOX/RELEASE.hidden"
expect_exit "missing RELEASE exits 1" 1 -- bash -c 'source scripts/common.sh && load_release'
cp "$RELEASE_BACKUP" RELEASE
bash -c 'source scripts/common.sh && load_release' >/dev/null 2>&1
bash -c 'source scripts/common.sh && load_release && case "$MOSAIC_IMAGE_TAG" in mosaic-poc-agent:*-r'"$(cat RELEASE)"') exit 0;; *) exit 1;; esac' >/dev/null 2>&1
check "valid RELEASE leaves image tag consistent with version" $?
# ---------- fast: the reason line when the health check gave no run id (lane health-reason) ----------
# The runner's last refusal line wins over notes, loses its prefix and is redacted: no host path and
# no key-shaped value. A line holding a value from the environment is withheld whole.
cat > "$SANDBOX/runner-err.txt" <<EOF
run-task: note: the session is new
mosaic-task: cannot fork: source session dir not found: $SANDBOX/sessions/x token=ghp_FIXTURE0123456789
mosaic-task: worker credential not checked: compose.yaml can't be read
EOF
REASON="$(node scripts/health-reason.mjs "$SANDBOX/runner-err.txt")"
case "$REASON" in "cannot fork: source session dir not found: "*) ok=0 ;; *) ok=1 ;; esac
[ "$ok" = 0 ] && ! printf '%s' "$REASON" | grep -qF "$SANDBOX" && ! printf '%s' "$REASON" | grep -qF 'ghp_FIXTURE' \
&& check "reason with no run id is the runner's refusal, redacted" 0 \
|| { check "reason with no run id is the runner's refusal, redacted" 1; printf '%s\n' "$REASON" >&2; }
printf 'mosaic-task: cannot start: quietharbourlamp refused\n' > "$SANDBOX/runner-err2.txt"
REASON="$(FIXTURE_TOKEN=quietharbourlamp node scripts/health-reason.mjs "$SANDBOX/runner-err2.txt")"
[ -z "$REASON" ] && check "reason holding an environment secret is withheld" 0 \
|| { check "reason holding an environment secret is withheld" 1; printf '%s\n' "$REASON" >&2; }
# ---------- sandboxed state machine (Docker required) ----------
if docker info >/dev/null 2>&1; then
mkdir -p "$SANDBOX/data"
# The mock adapter: no credential and no provider call in the gate (row 244,
# ruling A, sage-1914). This suite proves the release state machine, image and
# Compose path, not that the pi image reaches the provider.
cat > "$SANDBOX/config.json" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash","adapter":"mock"}}
EOF
export MOSAIC_CONFIG="$SANDBOX/config.json"
expect_exit "status safe on empty state" 0 -- scripts/release.sh status
[ ! -e "$SANDBOX/data/state/active.json" ] \
&& check "status created no pointer" 0 || check "status created no pointer" 1
# The mock gives the right marker, so only the fault's wrong expectation refuses.
expect_exit "fault-injected activation refuses" 1 -- env MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK scripts/release.sh activate --fault-injection
[ ! -e "$SANDBOX/data/state/active.json" ] \
&& check "refused activation wrote no pointer" 0 || check "refused activation wrote no pointer" 1
if [ -f "$SANDBOX/data/state/activation-log.jsonl" ]; then
node -e '
const fs = require("fs");
const lines = fs.readFileSync(process.argv[1], "utf8").split("\n").filter(Boolean);
if (lines.length !== 1) process.exit(1);
const e = JSON.parse(lines[0]);
process.exit(e.event === "refused" && e.release && e.imageTag && e.at ? 0 : 1);
' "$SANDBOX/data/state/activation-log.jsonl"
check "refusal logged exactly once with valid fields" $?
else
check "refusal logged exactly once with valid fields" 1
fi
# The refusal says which run failed and why, from the run record (lane health-reason).
env MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK scripts/release.sh activate --fault-injection >"$SANDBOX/fault-out.txt" 2>&1
FAULT_RUN="$(sed -n 's/^release: health check run: \(r-[A-Za-z0-9][A-Za-z0-9._-]*\)$/\1/p' "$SANDBOX/fault-out.txt")"
[ -n "$FAULT_RUN" ] && [ -f "$SANDBOX/data/runs/$FAULT_RUN/result.json" ] \
&& grep -qx 'release: reason: the response did not match the expected text' "$SANDBOX/fault-out.txt" \
&& [ "$(tail -n 1 "$SANDBOX/fault-out.txt")" = "release: refusing activate: health check failed" ] \
&& ! grep -qF "$SANDBOX" "$SANDBOX/fault-out.txt" \
&& check "refused activation prints its run id and reason, no host path" 0 \
|| { check "refused activation prints its run id and reason, no host path" 1; cat "$SANDBOX/fault-out.txt" >&2; }
# A pi config whose provider the container has no credential for refuses before a run record;
# the reason is the runner's refusal line.
cat > "$SANDBOX/pi-config.json" <<EOF
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash","adapter":"pi"}}
EOF
env -u ZAI_API_KEY -u ANTHROPIC_API_KEY -u COMPOSE_FILE -u COMPOSE_ENV_FILES MOSAIC_CONFIG="$SANDBOX/pi-config.json" \
scripts/release.sh activate >"$SANDBOX/cred-out.txt" 2>&1
grep -qx 'release: health check gave no run id' "$SANDBOX/cred-out.txt" \
&& grep -q '^release: reason: the worker container has no credential for provider zai (config execution.provider); it has credentials for: none\.' "$SANDBOX/cred-out.txt" \
&& [ "$(tail -n 1 "$SANDBOX/cred-out.txt")" = "release: refusing activate: health check failed" ] \
&& check "activation with no worker credential prints the refusal as its reason" 0 \
|| { check "activation with no worker credential prints the refusal as its reason" 1; cat "$SANDBOX/cred-out.txt" >&2; }
expect_exit "healthy activation succeeds" 0 -- env MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK scripts/release.sh activate
node -e '
const fs = require("fs");
const p = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
process.exit(p.pointerVersion === 1 && p.release && p.imageTag && p.activatedAt ? 0 : 1);
' "$SANDBOX/data/state/active.json"
check "pointer written with valid fields" $?
expect_exit "repeat activation succeeds (log grows)" 0 -- env MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK scripts/release.sh activate
LINES=$(grep -c '' "$SANDBOX/data/state/activation-log.jsonl")
[ "$LINES" -ge 3 ] && check "log is append-only across activations" 0 || check "log is append-only across activations" 1
expect_exit "rollback without previous refuses" 1 -- scripts/release.sh rollback
else
echo "skip state-machine cases (docker daemon unavailable)"
fi
# The compose network goes with the run (row 305). Present before the remove
# proves compose used this run's project name; absent after proves the trap's
# cleanup removes it (mos-1445).
if docker info >/dev/null 2>&1; then
NET="${COMPOSE_PROJECT_NAME}_default"
BEFORE="$(docker network ls -q --filter "name=^${NET}\$")" && [ -n "$BEFORE" ] \
&& remove_compose_network \
&& AFTER="$(docker network ls -q --filter "name=^${NET}\$")" && [ -z "$AFTER" ] \
&& check "compose network made under the run's project and removed after the run" 0 \
|| check "compose network made under the run's project and removed after the run" 1
else
echo "skip compose network check (docker daemon unavailable)"
fi
echo
echo "selftest: $PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]