Files
stack/packages/mosaic/framework/tools/git/test-issue-comment-readback.sh
Hermes Agent 10fdd49e32
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
fix(tools): bound Gitea read-back to this write; verify approve/reject state (#865)
Review remediation for two correctness holes in the #865 fix:

BLOCKER 1 — issue-comment.sh read-back was body-only across all history:
if `tea comment` silently no-opped (the #865 bug) while an identically
bodied comment already existed from a prior run, the read-back matched the
OLD comment and falsely reported success. Now record the pre-write maximum
comment id as a boundary and require a comment with id > boundary AND exact
body match; monotonic Gitea ids make id > boundary mean "created by this
write". Fails closed otherwise.

BLOCKER 2 — pr-review.sh approve/reject trusted tea's exit code for the
review STATE (same never-trust-exit-zero defect class as #865). Removed the
TODO deferral and added a real bounded read-back: record the max review id
before `tea pr approve`/`reject`, then require a review with id > boundary,
the expected state (APPROVED / REQUEST_CHANGES), and commit_id equal to the
PR's current head. Fails closed if absent.

Tests: extended test-pr-review-gitea-comment.sh to model and assert the new
review-state read-back (guardrails preserved, assertions added). Added
test-issue-comment-readback.sh proving the pre-existing-identical-body
false positive now fails closed and a genuinely new comment verifies.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-21 18:21:02 -05:00

217 lines
6.8 KiB
Bash
Executable File

#!/usr/bin/env bash
# Regression harness for issue-comment.sh top-level `tea comment` invocation and
# its BOUNDED read-back verification (#865).
#
# The #865 bug: `tea issue comment ...` (a nonexistent subcommand on tea
# v0.11.1) silently no-ops and exits 0, so a comment is never posted. A naive
# read-back that matches ANY historical comment by body would falsely report
# success whenever an identically-bodied comment already exists from a prior
# run. This harness proves the wrapper:
# 1. uses the top-level `tea comment` form (never `tea issue comment`);
# 2. records the pre-write maximum comment id as a boundary and requires a
# strictly-newer comment on read-back, so a pre-existing identical body
# does NOT satisfy verification (fails closed);
# 3. reports success only when a genuinely new comment (id > boundary) with
# the exact body appears.
#
# The `tea` stub NEVER creates a comment (it mimics the silent no-op); the
# "server" comment state is modeled entirely by the curl stub's responses, so
# the fresh-success vs. no-op distinction is driven purely by whether the
# post-write read-back surfaces a new id.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/issue-comment-readback}"
REPO_DIR="$WORK_DIR/repo"
BIN_DIR="$WORK_DIR/bin"
TEA_LOG="$WORK_DIR/tea.log"
CURL_LOG="$WORK_DIR/curl.log"
OUTPUT_FILE="$WORK_DIR/output.log"
CREDENTIALS_FILE="$WORK_DIR/credentials.json"
CALLS_FILE="$WORK_DIR/comment_calls"
cleanup() {
rm -rf "$WORK_DIR"
}
trap cleanup EXIT
mkdir -p "$REPO_DIR" "$BIN_DIR"
git -C "$REPO_DIR" init -q
git -C "$REPO_DIR" remote add origin https://git.mosaicstack.dev/mosaicstack/stack.git
ISSUE_NUMBER=7
API_BASE="https://git.mosaicstack.dev/api/v1/repos/mosaicstack/stack"
BODY='durable "note" -- marker'
CONFIGURED_GITEA_URL="https://git.mosaicstack.dev" python3 - "$CREDENTIALS_FILE" <<'PY'
import json
import os
import sys
with open(sys.argv[1], "w", encoding="utf-8") as credentials:
json.dump({
"gitea": {
"mosaicstack": {
"url": os.environ["CONFIGURED_GITEA_URL"],
"token": "test-only-placeholder",
}
}
}, credentials)
PY
# tea stub: resolves the login list, and treats `tea comment ...` as a silent
# no-op (exit 0 without creating anything) to mimic the real failure mode.
cat > "$BIN_DIR/tea" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$*" >> "$ISSUE_COMMENT_TEA_LOG"
if [[ "$*" == "login list --output json" ]]; then
printf '%s\n' '[{"name":"mosaicstack","url":"https://git.mosaicstack.dev"}]'
exit 0
fi
# The wrapper must use the TOP-LEVEL `tea comment` form; the broken
# `tea issue comment` subcommand must never be invoked.
if [[ "$*" == issue\ comment* ]]; then
echo "wrapper invoked nonexistent 'tea issue comment' subcommand" >&2
exit 90
fi
if [[ "$*" == comment\ * ]]; then
# Mimic tea v0.11.1: exit 0. Whether a comment actually lands is modeled
# by the curl stub's post-write read-back response, not here.
exit 0
fi
echo "Unexpected tea command: $*" >&2
exit 92
SH
chmod +x "$BIN_DIR/tea"
# curl stub: serves GET .../issues/7/comments. First call = pre-write boundary,
# second call = post-write read-back. The boundary always contains a
# pre-existing comment (id 50) whose body is IDENTICAL to the one under test,
# which is exactly the condition a body-only match would trip over.
cat > "$BIN_DIR/curl" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
output_file=""
method="GET"
url=""
while [[ $# -gt 0 ]]; do
case "$1" in
-o) output_file="$2"; shift 2 ;;
-w|-H) shift 2 ;;
-X) method="$2"; shift 2 ;;
-d|--data) shift 2 ;;
-s|-S|-sS) shift ;;
http://*|https://*) url="$1"; shift ;;
*) shift ;;
esac
done
printf '%s %s\n' "$method" "$url" >> "$ISSUE_COMMENT_CURL_LOG"
write_response() {
local status="$1" body="$2"
[[ -n "$output_file" ]] || exit 96
printf '%s' "$body" > "$output_file"
printf '%s' "$status"
}
if [[ "$method" == "GET" && "$url" == "$ISSUE_COMMENT_API_BASE/issues/7/comments" ]]; then
calls_file="$ISSUE_COMMENT_CALLS"
if [[ -f "$calls_file" ]]; then
# post-write read-back
if [[ "$ISSUE_COMMENT_TEST_MODE" == "fresh-success" ]]; then
response=$(ISSUE_COMMENT_BODY="$ISSUE_COMMENT_EXPECTED_BODY" python3 - <<'PY'
import json
import os
body = os.environ["ISSUE_COMMENT_BODY"]
print(json.dumps([
{"id": 50, "body": body},
{"id": 60, "body": body},
]))
PY
)
else
# no-op: nothing new landed; the pre-existing id-50 comment remains.
response=$(ISSUE_COMMENT_BODY="$ISSUE_COMMENT_EXPECTED_BODY" python3 - <<'PY'
import json
import os
body = os.environ["ISSUE_COMMENT_BODY"]
print(json.dumps([{"id": 50, "body": body}]))
PY
)
fi
else
: > "$calls_file"
response=$(ISSUE_COMMENT_BODY="$ISSUE_COMMENT_EXPECTED_BODY" python3 - <<'PY'
import json
import os
body = os.environ["ISSUE_COMMENT_BODY"]
print(json.dumps([{"id": 50, "body": body}]))
PY
)
fi
write_response 200 "$response"
else
echo "Unexpected curl request: $method $url" >&2
exit 97
fi
SH
chmod +x "$BIN_DIR/curl"
run_comment() {
local mode="$1"
: > "$TEA_LOG"
: > "$CURL_LOG"
: > "$OUTPUT_FILE"
rm -f "$CALLS_FILE"
(
cd "$REPO_DIR"
PATH="$BIN_DIR:$PATH" \
MOSAIC_CREDENTIALS_FILE="$CREDENTIALS_FILE" \
ISSUE_COMMENT_TEA_LOG="$TEA_LOG" \
ISSUE_COMMENT_CURL_LOG="$CURL_LOG" \
ISSUE_COMMENT_CALLS="$CALLS_FILE" \
ISSUE_COMMENT_TEST_MODE="$mode" \
ISSUE_COMMENT_EXPECTED_BODY="$BODY" \
ISSUE_COMMENT_API_BASE="$API_BASE" \
"$SCRIPT_DIR/issue-comment.sh" -i "$ISSUE_NUMBER" -c "$BODY"
) > "$OUTPUT_FILE" 2>&1
}
# Case 1: silent no-op with a pre-existing identical body must FAIL CLOSED.
if run_comment noop-preexisting; then
echo "FAIL: wrapper reported success when tea no-opped but an identical body pre-existed" >&2
cat "$OUTPUT_FILE" >&2
exit 1
fi
if grep -q 'Added and verified comment' "$OUTPUT_FILE"; then
echo "FAIL: read-back matched a pre-existing comment by body only" >&2
exit 1
fi
# The wrapper must have used the top-level form and read comments back twice
# (boundary + post-write).
grep -q "^comment 7 " "$TEA_LOG"
if grep -q '^issue comment' "$TEA_LOG"; then
echo "FAIL: wrapper used the broken 'tea issue comment' subcommand" >&2
exit 1
fi
[[ "$(grep -c "^GET $API_BASE/issues/7/comments$" "$CURL_LOG")" == "2" ]]
# Case 2: a genuinely new comment (id 60 > boundary 50) verifies successfully.
run_comment fresh-success
grep -q 'Added and verified comment on Gitea issue #7 (comment ID 60)' "$OUTPUT_FILE"
grep -q "^comment 7 " "$TEA_LOG"
echo "issue-comment.sh bounded read-back regression passed"