Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #935.
This commit is contained in:
@@ -326,6 +326,51 @@ cmd_drain() {
|
||||
grep -v '^[[:space:]]*$' "$STATE_DIR/pending.jsonl" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# _record_last_consumed UPTO — #932: at consume-truncation, record the
|
||||
# last-consumed observed_hash per (kind,id) into a STORE-OWNED durable record
|
||||
# (consumed-hashes.jsonl). This is the record the reconciler consults as its
|
||||
# THIRD accounting source: after consume truncates the pending prefix, a consumed
|
||||
# state matches NEITHER the inbox (truncated) NOR the reconciler's seen-ledger
|
||||
# (which only covers the reconciler's OWN enumerations), so the reconciler used
|
||||
# to treat it as UNACCOUNTED and re-enumerate it — one duplicate orientation wake
|
||||
# + one spurious rc=1 CRITICAL per detector-active window per cycle (safe but
|
||||
# noisy; G2a alarm-hygiene at fleet scale).
|
||||
#
|
||||
# Called BEFORE the pending prefix is dropped (it reads the entries being
|
||||
# consumed). For each entry with observed_seq <= UPTO that carries full source
|
||||
# locators (kind,id,observed_hash), keep the HIGHEST-seq hash per (kind,id) as
|
||||
# that source's last-consumed state; merge with any prior record (monotonic —
|
||||
# consume only advances, so a newer consume's seq always wins its key).
|
||||
#
|
||||
# TRUST BOUNDARY (load-bearing): this record is written ONLY here, at CONSUME of a
|
||||
# durably-enqueued entry, so its existence implies the state WAS durably
|
||||
# enqueued+consumed. It therefore structurally cannot exhibit the §5 swallow-hole
|
||||
# signature (hash-advance-WITHOUT-enqueue). It is a STORE record — the detector's
|
||||
# own hash-file stays DISTRUSTED and is never consulted for accounting.
|
||||
#
|
||||
# ADDITIVE: a new file consumed-hashes.jsonl; the existing on-disk format is
|
||||
# unchanged and read-compatible (older code ignores this file). Atomic write.
|
||||
# Best-effort: a failure here degrades to the pre-#932 safe-but-noisy behaviour
|
||||
# (the reconciler re-enumerates the consumed state once — no lost obligation),
|
||||
# never a failed consume.
|
||||
_record_last_consumed() {
|
||||
local upto="$1" existing new_records merged
|
||||
[ -f "$STATE_DIR/pending.jsonl" ] || return 0
|
||||
new_records="$(jq -c --argjson upto "$upto" '
|
||||
select((.observed_seq // -1) <= $upto)
|
||||
| select((.locators.kind // "") != "" and (.locators.id // "") != "" and (.locators.observed_hash // "") != "")
|
||||
| {kind:.locators.kind, id:.locators.id, observed_hash:.locators.observed_hash, observed_seq:.observed_seq}
|
||||
' "$STATE_DIR/pending.jsonl" 2>/dev/null || true)"
|
||||
[ -n "$new_records" ] || return 0
|
||||
existing="$(cat "$STATE_DIR/consumed-hashes.jsonl" 2>/dev/null || true)"
|
||||
merged="$(printf '%s\n%s\n' "$existing" "$new_records" | grep -v '^[[:space:]]*$' |
|
||||
jq -s -c 'group_by([.kind, .id]) | .[] | max_by(.observed_seq)' 2>/dev/null || true)"
|
||||
[ -n "$merged" ] || return 0
|
||||
if ! printf '%s\n' "$merged" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/consumed-hashes.jsonl"; then
|
||||
echo "store.sh consume: WARN — could not record last-consumed hashes (#932); the reconciler may re-enumerate the consumed state once (safe-but-noisy), no lost obligation." >&2
|
||||
fi
|
||||
}
|
||||
|
||||
cmd_consume() {
|
||||
local upto=''
|
||||
while [ $# -gt 0 ]; do
|
||||
@@ -381,6 +426,12 @@ cmd_consume() {
|
||||
k=$((k + 1))
|
||||
done
|
||||
|
||||
# #932: record the last-consumed observed_hash per (kind,id) BEFORE the pending
|
||||
# prefix is dropped (this reads the entries about to be truncated), so the
|
||||
# reconciler can recognise an already-consumed state as ACCOUNTED instead of
|
||||
# re-enumerating it. Additive, store-owned, best-effort (never fails consume).
|
||||
_record_last_consumed "$upto"
|
||||
|
||||
# Advance the consumer cursor and drop the now-consumed prefix from the
|
||||
# durable store (retain-until-CONSUMED is satisfied).
|
||||
jq -c "select(.observed_seq > $upto)" "$STATE_DIR/pending.jsonl" 2>/dev/null |
|
||||
|
||||
Reference in New Issue
Block a user